From e1af7f7125c6e2002f85dab5b260c6ee3e0b5d4e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:48:25 +0100 Subject: [PATCH 1/3] docs: replace the monolithic SQLBuild skill with a focused skill, task guides and bundled docs pages --- .github/workflows/publish.yml | 1 + scripts/docs_skill/_helpers/mdx.py | 8 +- scripts/docs_skill/_helpers/navigation.py | 38 + scripts/docs_skill/_helpers/output.py | 28 +- scripts/docs_skill/_helpers/render.py | 117 +- scripts/docs_skill/constants.py | 18 +- scripts/docs_skill/main/generate.py | 24 +- scripts/docs_skill/models.py | 11 +- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 15780 +--------------- .../skills/sqlbuild/references/docs/INDEX.md | 150 + .../sqlbuild/references/docs/benchmarks.md | 125 + .../sqlbuild/references/docs/cli/audit.md | 70 + .../sqlbuild/references/docs/cli/build.md | 132 + .../sqlbuild/references/docs/cli/check.md | 56 + .../sqlbuild/references/docs/cli/clean.md | 19 + .../sqlbuild/references/docs/cli/clone.md | 91 + .../sqlbuild/references/docs/cli/compile.md | 184 + .../sqlbuild/references/docs/cli/contract.md | 44 + .../sqlbuild/references/docs/cli/dag.md | 172 + .../sqlbuild/references/docs/cli/dbt.md | 84 + .../sqlbuild/references/docs/cli/debug.md | 71 + .../sqlbuild/references/docs/cli/diff.md | 66 + .../sqlbuild/references/docs/cli/format.md | 33 + .../sqlbuild/references/docs/cli/freshness.md | 177 + .../sqlbuild/references/docs/cli/init.md | 79 + .../sqlbuild/references/docs/cli/janitor.md | 59 + .../sqlbuild/references/docs/cli/lineage.md | 258 + .../sqlbuild/references/docs/cli/load.md | 95 + .../sqlbuild/references/docs/cli/plan.md | 89 + .../references/docs/cli/playground.md | 95 + .../sqlbuild/references/docs/cli/promote.md | 48 + .../sqlbuild/references/docs/cli/query.md | 102 + .../sqlbuild/references/docs/cli/reconcile.md | 66 + .../sqlbuild/references/docs/cli/rollback.md | 48 + .../sqlbuild/references/docs/cli/rules.md | 52 + .../sqlbuild/references/docs/cli/scenario.md | 168 + .../sqlbuild/references/docs/cli/scope.md | 413 + .../sqlbuild/references/docs/cli/seed.md | 34 + .../sqlbuild/references/docs/cli/skills.md | 105 + .../sqlbuild/references/docs/cli/state.md | 128 + .../sqlbuild/references/docs/cli/test.md | 44 + .../references/docs/concepts/adapters.md | 120 + .../docs/concepts/adapters/bigquery.md | 65 + .../docs/concepts/adapters/databricks.md | 45 + .../docs/concepts/adapters/duckdb.md | 62 + .../docs/concepts/adapters/motherduck.md | 79 + .../docs/concepts/adapters/postgres.md | 66 + .../docs/concepts/adapters/snowflake.md | 68 + .../docs/concepts/adapters/sqlserver.md | 80 + .../references/docs/concepts/audits.md | 316 + .../docs/concepts/column-lineage.md | 190 + .../references/docs/concepts/constants.md | 84 + .../constants/collections-and-rendering.md | 163 + .../adding-sqlbuild-models.md | 59 + .../concepts/dbt-compatibility/overview.md | 150 + .../concepts/dbt-compatibility/selection.md | 75 + .../docs/concepts/declaration-scopes.md | 108 + .../concepts/declaration-scopes/explorer.md | 345 + .../concepts/declaration-scopes/placement.md | 96 + .../concepts/declaration-scopes/visibility.md | 187 + .../sqlbuild/references/docs/concepts/diff.md | 242 + .../docs/concepts/enums-and-constants.md | 52 + .../references/docs/concepts/enums.md | 91 + .../docs/concepts/enums/model-contracts.md | 50 + .../references/docs/concepts/functions.md | 182 + .../references/docs/concepts/incremental.md | 395 + .../references/docs/concepts/interpolation.md | 210 + .../references/docs/concepts/macros.md | 178 + .../macros/composition-and-context.md | 139 + .../docs/concepts/model-private-values.md | 71 + .../references/docs/concepts/models.md | 57 + .../docs/concepts/models/configuration.md | 122 + .../docs/concepts/models/contracts.md | 171 + .../references/docs/concepts/models/hooks.md | 119 + .../docs/concepts/models/hooks/python.md | 105 + .../docs/concepts/models/hooks/sql.md | 177 + .../docs/concepts/models/materializations.md | 100 + .../docs/concepts/models/schemas.md | 137 + .../docs/concepts/models/type-enforcement.md | 74 + .../references/docs/concepts/observability.md | 86 + .../docs/concepts/observability/sinks.md | 89 + .../references/docs/concepts/planning.md | 150 + .../concepts/planning/cascade-propagation.md | 41 + .../planning/selection-and-staleness.md | 73 + .../concepts/planning/source-freshness.md | 42 + .../docs/concepts/project-configuration.md | 498 + .../docs/concepts/python-nodes/assets.md | 146 + .../docs/concepts/python-nodes/checks.md | 128 + .../docs/concepts/python-nodes/factories.md | 97 + .../docs/concepts/python-nodes/loaders.md | 391 + .../docs/concepts/python-nodes/overview.md | 177 + .../docs/concepts/python-nodes/providers.md | 209 + .../concepts/python-nodes/sql-references.md | 64 + .../docs/concepts/python-nodes/tasks.md | 143 + .../docs/concepts/resource-identities.md | 57 + .../references/docs/concepts/rules.md | 237 + .../rules/configuration-and-selection.md | 67 + .../concepts/rules/custom-rules/overview.md | 84 + .../rules/custom-rules/rule-context.md | 74 + .../custom-rules/testing-and-determinism.md | 64 + .../concepts/rules/execution-and-caching.md | 56 + .../concepts/rules/findings-and-exceptions.md | 77 + .../references/docs/concepts/scenarios.md | 290 + .../references/docs/concepts/seeds.md | 133 + .../references/docs/concepts/selectors.md | 171 + .../references/docs/concepts/snapshots.md | 368 + .../references/docs/concepts/sources.md | 248 + .../references/docs/concepts/testing.md | 604 + .../docs/concepts/virtual-environments.md | 117 + .../virtual-environments/adopt-detach.md | 123 + .../concepts/virtual-environments/building.md | 232 + .../concepts/virtual-environments/clone.md | 77 + .../concepts/virtual-environments/diff.md | 61 + .../concepts/virtual-environments/janitor.md | 76 + .../concepts/virtual-environments/locks.md | 60 + .../virtual-environments/promotion.md | 72 + .../virtual-environments/reconcile.md | 74 + .../concepts/virtual-environments/recovery.md | 127 + .../concepts/virtual-environments/rollback.md | 119 + .../concepts/virtual-environments/setup.md | 189 + .../references/docs/feature-comparison.md | 140 + .../skills/sqlbuild/references/docs/index.md | 254 + .../docs/integrations/dagster-reference.md | 261 + .../references/docs/integrations/dagster.md | 175 + .../references/docs/integrations/dlt.md | 226 + .../references/docs/integrations/ingestr.md | 168 + .../references/docs/integrations/rivers.md | 181 + .../sqlbuild/references/docs/quickstart.md | 279 + .../references/exploring-a-project.md | 114 + .../sqlbuild/references/incremental-models.md | 146 + .../references/macros-and-declarations.md | 133 + .../skills/sqlbuild/references/testing.md | 157 + .../sqlbuild/references/verifying-changes.md | 133 + .../cli/commands/_helpers/skills/update.py | 131 +- .../cli/commands/main/skills/_test_types.py | 2 +- .../cli/commands/main/skills/test_skills.py | 2 +- .../docs_skill/_helpers/_test_types.py | 10 + .../docs_skill/_helpers/test_render.py | 84 + .../cli/commands/main/skills/_test_types.py | 35 +- .../cli/commands/main/skills/helpers.py | 34 +- .../main/skills/test_skill_directory.py | 176 + 141 files changed, 17692 insertions(+), 15752 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/audit.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clean.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/contract.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/diff.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/promote.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/reconcile.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rollback.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/seed.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/skills.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/state.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/column-lineage.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants/collections-and-rendering.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/selection.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/explorer.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/placement.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums-and-constants.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/composition-and-context.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/source-freshness.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/overview.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/adopt-detach.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/building.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/clone.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/diff.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/janitor.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/locks.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/promotion.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/reconcile.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/recovery.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/rollback.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/setup.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/exploring-a-project.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/testing.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md create mode 100644 tests/unit/scripts/docs_skill/_helpers/test_render.py create mode 100644 tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 25115123cc..53d7ca0b9f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -80,6 +80,7 @@ jobs: exit 1 fi unzip -l dist/*.whl | grep 'scripts/dupscore/dupscore.toml' + unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md' python -m venv /tmp/sqlbuild-wheel-smoke /tmp/sqlbuild-wheel-smoke/bin/python -m pip install dist/*.whl /tmp/sqlbuild-wheel-smoke/bin/python -c "import sqlbuild._native as native; assert native.catalogue_json()" diff --git a/scripts/docs_skill/_helpers/mdx.py b/scripts/docs_skill/_helpers/mdx.py index ddd1fc5d64..bc855fe1cd 100644 --- a/scripts/docs_skill/_helpers/mdx.py +++ b/scripts/docs_skill/_helpers/mdx.py @@ -55,7 +55,7 @@ def _clean_mdx_body(body: str) -> str: stripped: str = line.strip() if _is_mdx_only_line(stripped): continue - cleaned_lines.append(_shift_markdown_heading(line)) + cleaned_lines.append(line) return normalize_blank_lines("\n".join(cleaned_lines)).strip() @@ -72,12 +72,6 @@ def _is_mdx_only_line(stripped: str) -> bool: return bool(re.fullmatch(r"]*/>", stripped)) -def _shift_markdown_heading(line: str) -> str: - if re.match(r"^#{1,6} ", line): - return f"#{line}" - return line - - def normalize_blank_lines(contents: str) -> str: """Collapse runs of blank lines to one blank line.""" diff --git a/scripts/docs_skill/_helpers/navigation.py b/scripts/docs_skill/_helpers/navigation.py index 3afc203bc4..72890b2327 100644 --- a/scripts/docs_skill/_helpers/navigation.py +++ b/scripts/docs_skill/_helpers/navigation.py @@ -4,6 +4,9 @@ from pathlib import Path from typing import Any +from scripts.docs_skill.constants import UNGROUPED_LABEL +from scripts.docs_skill.models import NavigationGroup + def list_ordered_page_paths(*, docs_root: Path) -> list[Path]: """Return existing MDX pages in navigation order followed by unlisted pages.""" @@ -47,3 +50,38 @@ def _collect_navigation_pages(node: Any) -> list[str]: pages.extend(_collect_navigation_pages(node.get(key, []))) return pages + + +def list_navigation_groups(*, docs_root: Path) -> list[NavigationGroup]: + """Return pages grouped by top-level navigation group, then unlisted pages.""" + + ordered_paths: list[Path] = list_ordered_page_paths(docs_root=docs_root) + docs_json_path: Path = docs_root / "docs.json" + top_level_nodes: list[Any] = [] + if docs_json_path.exists(): + navigation: Any = json.loads(docs_json_path.read_text(encoding="utf-8")).get( + "navigation", {} + ) + top_level_nodes = navigation.get("groups", []) if isinstance(navigation, dict) else [] + + groups: list[NavigationGroup] = [] + grouped_pages: set[str] = set() + for node in top_level_nodes: + label: str = str(node.get("group", UNGROUPED_LABEL)) if isinstance(node, dict) else "" + member_pages: set[str] = set(_collect_navigation_pages(node)) + page_paths: tuple[Path, ...] = tuple( + path + for path in ordered_paths + if path.with_suffix("").as_posix() in member_pages + and path.with_suffix("").as_posix() not in grouped_pages + ) + grouped_pages.update(path.with_suffix("").as_posix() for path in page_paths) + if page_paths: + groups.append(NavigationGroup(label=label, page_paths=page_paths)) + + remaining: tuple[Path, ...] = tuple( + path for path in ordered_paths if path.with_suffix("").as_posix() not in grouped_pages + ) + if remaining: + groups.append(NavigationGroup(label=UNGROUPED_LABEL, page_paths=remaining)) + return groups diff --git a/scripts/docs_skill/_helpers/output.py b/scripts/docs_skill/_helpers/output.py index 82a1a97a54..f036999ef0 100644 --- a/scripts/docs_skill/_helpers/output.py +++ b/scripts/docs_skill/_helpers/output.py @@ -1,11 +1,27 @@ -"""Generated skill output handling.""" +"""Generated docs reference output handling.""" from pathlib import Path +from scripts.docs_skill.constants import GENERATED_MARKER -def write_skill_markdown(*, output_path: Path, contents: str) -> None: - """Write generated skill Markdown and report its destination.""" - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(contents, encoding="utf-8") - print(f"Wrote {output_path}") +def write_reference_pages(*, output_dir: Path, pages: dict[str, str]) -> None: + """Write generated reference pages and remove stale generated pages.""" + + output_dir.mkdir(parents=True, exist_ok=True) + for relative_path, contents in pages.items(): + destination: Path = output_dir / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(contents, encoding="utf-8") + stale_paths: list[Path] = [ + path + for path in output_dir.rglob("*.md") + if path.relative_to(output_dir).as_posix() not in pages + and GENERATED_MARKER in path.read_text(encoding="utf-8") + ] + for stale_path in stale_paths: + stale_path.unlink() + for directory in sorted(output_dir.rglob("*"), key=lambda path: len(path.parts), reverse=True): + if directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() + print(f"Wrote {len(pages)} reference pages to {output_dir} ({len(stale_paths)} removed)") diff --git a/scripts/docs_skill/_helpers/render.py b/scripts/docs_skill/_helpers/render.py index 1565dc73d7..0de0ddc788 100644 --- a/scripts/docs_skill/_helpers/render.py +++ b/scripts/docs_skill/_helpers/render.py @@ -1,50 +1,101 @@ -"""Markdown rendering for SQLBuild docs skills.""" +"""Markdown rendering for bundled SQLBuild docs reference pages.""" -from pathlib import Path +import os +import re +from pathlib import Path, PurePosixPath from scripts.docs_skill._helpers.mdx import normalize_blank_lines, parse_mdx_page -from scripts.docs_skill._helpers.navigation import list_ordered_page_paths +from scripts.docs_skill._helpers.navigation import list_navigation_groups from scripts.docs_skill.constants import ( + DOCS_BASE_URL, GENERATED_MARKER, - INTRODUCTION_PAGE_NAME, - SKILL_FRONTMATTER, + INDEX_FILENAME, + TABLE_OF_CONTENTS_MIN_LINES, ) -from scripts.docs_skill.models import MdxPage +from scripts.docs_skill.models import MdxPage, NavigationGroup +_INTERNAL_LINK: re.Pattern[str] = re.compile(r"\]\((/[^)\s#]*)(#[^)\s]*)?\)") +_SECTION_HEADING: re.Pattern[str] = re.compile(r"^## (.+)$", re.MULTILINE) -def build_skill_markdown(*, docs_root: Path, include_introduction: bool = True) -> str: - """Build one Markdown skill from all ordered documentation pages.""" - page_paths: list[Path] = list_ordered_page_paths(docs_root=docs_root) - if not include_introduction: - page_paths = [ - path for path in page_paths if path.with_suffix("").as_posix() != INTRODUCTION_PAGE_NAME - ] +def build_reference_pages(*, docs_root: Path) -> dict[str, str]: + """Render every documentation page plus a grouped index, keyed by relative output path.""" - sections: list[str] = [_render_skill_header(page_paths=page_paths)] - for page_path in page_paths: - absolute_path: Path = docs_root / page_path - page: MdxPage = parse_mdx_page(absolute_path.read_text(encoding="utf-8")) - sections.append(_render_page_section(page=page, source_path=page_path)) + groups: list[NavigationGroup] = list_navigation_groups(docs_root=docs_root) + page_names: set[str] = set() + for group in groups: + page_names.update(path.with_suffix("").as_posix() for path in group.page_paths) + known_pages: frozenset[str] = frozenset(page_names) + pages: dict[str, MdxPage] = {} + rendered: dict[str, str] = {} + for group in groups: + for page_path in group.page_paths: + page_name: str = page_path.with_suffix("").as_posix() + page: MdxPage = parse_mdx_page((docs_root / page_path).read_text(encoding="utf-8")) + pages[page_name] = page + rendered[f"{page_name}.md"] = _render_page( + page=page, page_name=page_name, known_pages=known_pages + ) + rendered[INDEX_FILENAME] = _render_index(groups=groups, pages=pages) + return rendered - return normalize_blank_lines("\n\n".join(sections)).strip() + "\n" - -def _render_skill_header(*, page_paths: list[Path]) -> str: - page_list: str = "\n".join(f"- `{path.with_suffix('').as_posix()}`" for path in page_paths) +def _render_page(*, page: MdxPage, page_name: str, known_pages: frozenset[str]) -> str: + body: str = _rewrite_internal_links( + body=page.body, page_name=page_name, known_pages=known_pages + ) + description_block: str = f"> {page.description}\n\n" if page.description else "" + contents_block: str = _table_of_contents(body=body) return ( - f"{SKILL_FRONTMATTER}\n\n" - f"{GENERATED_MARKER}\n\n" - "# SQLBuild Skill\n\n" - "This file is generated from the SQLBuild documentation. Use it as the source of truth " - "for SQLBuild syntax, project structure, configuration, testing, adapters, and CLI " - f"behavior.\n\n## Included Pages\n\n{page_list}" + normalize_blank_lines( + f"{GENERATED_MARKER}\n\n# {page.title}\n\n{description_block}" + f"Online: {DOCS_BASE_URL}/{page_name}\n\n{contents_block}{body}" + ).strip() + + "\n" ) -def _render_page_section(*, page: MdxPage, source_path: Path) -> str: - description_block: str = f"\n\n{page.description}" if page.description else "" - body_block: str = f"\n\n{page.body}" if page.body else "" - return f"""## {page.title} +def _table_of_contents(*, body: str) -> str: + if body.count("\n") < TABLE_OF_CONTENTS_MIN_LINES: + return "" + headings: list[str] = _SECTION_HEADING.findall(_without_code_blocks(body)) + if not headings: + return "" + return "## Contents\n\n" + "".join(f"- {heading}\n" for heading in headings) + "\n" + + +def _without_code_blocks(body: str) -> str: + return re.sub(r"```.*?```", "", body, flags=re.DOTALL) + + +def _rewrite_internal_links(*, body: str, page_name: str, known_pages: frozenset[str]) -> str: + source_dir: PurePosixPath = PurePosixPath(page_name).parent -Source: `{source_path.as_posix()}`{description_block}{body_block}""" + def replace(match: re.Match[str]) -> str: + target: str = match.group(1).strip("/") or "index" + anchor: str = match.group(2) or "" + if target not in known_pages: + return f"]({DOCS_BASE_URL}/{target}{anchor})" + relative: str = os.path.relpath(f"{target}.md", start=source_dir.as_posix() or ".") + return f"]({PurePosixPath(relative).as_posix()}{anchor})" + + return _INTERNAL_LINK.sub(replace, body) + + +def _render_index(*, groups: list[NavigationGroup], pages: dict[str, MdxPage]) -> str: + sections: list[str] = [] + for group in groups: + lines: list[str] = [f"## {group.label}", ""] + for page_path in group.page_paths: + page_name: str = page_path.with_suffix("").as_posix() + page: MdxPage = pages[page_name] + summary: str = f" - {page.description}" if page.description else "" + lines.append(f"- [{page.title}]({page_name}.md) (`{page_name}`){summary}") + sections.append("\n".join(lines)) + return ( + f"{GENERATED_MARKER}\n\n# SQLBuild documentation index\n\n" + "Bundled copies of every page on the SQLBuild documentation site, matching the " + "installed SQLBuild version. Open only the page you need.\n\n" + + "\n\n".join(sections) + + "\n" + ) diff --git a/scripts/docs_skill/constants.py b/scripts/docs_skill/constants.py index dada3ce734..275592fe2c 100644 --- a/scripts/docs_skill/constants.py +++ b/scripts/docs_skill/constants.py @@ -1,20 +1,14 @@ -"""Stable constants for SQLBuild docs skill generation.""" +"""Stable constants for SQLBuild docs reference generation.""" import tempfile from pathlib import Path DEFAULT_REPO_URL: str = "https://github.com/chio-labs/sqlbuild-docs" DEFAULT_CLONE_DIR: Path = Path(tempfile.gettempdir()) / "sqlbuild-docs-skill-source" -DEFAULT_OUTPUT_PATH: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/SKILL.md") +DEFAULT_OUTPUT_DIR: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/references/docs") +DOCS_BASE_URL: str = "https://docs.sqlbuild.com" GENERATED_MARKER: str = "" -SKILL_DESCRIPTION: str = ( - "ALWAYS load this skill when doing ANY SQLBuild work. This includes models, tests, audits, " - "scenarios, configuration, CLI behavior, adapters, dependencies, documentation, and related " - "code." -) -SKILL_FRONTMATTER: str = f"""--- -name: sqlbuild -description: {SKILL_DESCRIPTION} ----""" -INTRODUCTION_PAGE_NAME: str = "index" +INDEX_FILENAME: str = "INDEX.md" FRONTMATTER_DELIMITER: str = "---" +TABLE_OF_CONTENTS_MIN_LINES: int = 100 +UNGROUPED_LABEL: str = "Other pages" diff --git a/scripts/docs_skill/main/generate.py b/scripts/docs_skill/main/generate.py index d56408d297..aeeee74279 100644 --- a/scripts/docs_skill/main/generate.py +++ b/scripts/docs_skill/main/generate.py @@ -1,4 +1,4 @@ -"""Focused entry for SQLBuild docs skill generation.""" +"""Focused entry for SQLBuild docs reference generation.""" from __future__ import annotations @@ -6,34 +6,34 @@ from pathlib import Path from scripts.docs_skill._helpers.clone import clone_docs_repo -from scripts.docs_skill._helpers.output import write_skill_markdown -from scripts.docs_skill._helpers.render import build_skill_markdown -from scripts.docs_skill.constants import DEFAULT_CLONE_DIR, DEFAULT_OUTPUT_PATH, DEFAULT_REPO_URL +from scripts.docs_skill._helpers.output import write_reference_pages +from scripts.docs_skill._helpers.render import build_reference_pages +from scripts.docs_skill.constants import DEFAULT_CLONE_DIR, DEFAULT_OUTPUT_DIR, DEFAULT_REPO_URL def generate_docs_skill(argv: list[str] | None = None) -> int: - """Generate and write the SQLBuild documentation skill.""" + """Generate the bundled documentation reference pages for the SQLBuild skill.""" args: argparse.Namespace = _parse_args(argv) docs_root: Path = args.docs_root or clone_docs_repo( repo_url=args.repo_url, clone_dir=args.clone_dir, ) - skill_markdown: str = build_skill_markdown( - docs_root=docs_root, - include_introduction=not args.exclude_introduction, + write_reference_pages( + output_dir=args.output_dir, + pages=build_reference_pages(docs_root=docs_root), ) - write_skill_markdown(output_path=args.output, contents=skill_markdown) return 0 def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="Clone sqlbuild-docs and convert all MDX pages into a single SKILL.md." + description=( + "Clone sqlbuild-docs and convert each MDX page into a bundled skill reference page." + ) ) parser.add_argument("--repo-url", default=DEFAULT_REPO_URL) parser.add_argument("--clone-dir", type=Path, default=DEFAULT_CLONE_DIR) parser.add_argument("--docs-root", type=Path) - parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH) - parser.add_argument("--exclude-introduction", action="store_true") + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) return parser.parse_args(argv) diff --git a/scripts/docs_skill/models.py b/scripts/docs_skill/models.py index 585e36086a..3353c058b5 100644 --- a/scripts/docs_skill/models.py +++ b/scripts/docs_skill/models.py @@ -1,6 +1,7 @@ -"""Structured models for SQLBuild docs skill generation.""" +"""Structured models for SQLBuild docs reference generation.""" from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True, slots=True) @@ -10,3 +11,11 @@ class MdxPage: title: str description: str body: str + + +@dataclass(frozen=True, slots=True) +class NavigationGroup: + """One navigation group label and its pages in documentation order.""" + + label: str + page_paths: tuple[Path, ...] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 35b96a6f79..196b14eea7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -1,237 +1,30 @@ --- name: sqlbuild -description: ALWAYS load this skill when doing ANY SQLBuild work. This includes models, tests, audits, scenarios, configuration, CLI behavior, adapters, dependencies, documentation, and related code. +description: ALWAYS load this skill when doing ANY SQLBuild work - models, sources, seeds, macros, enums, constants, tests, audits, scenarios, incremental and microbatch models, configuration, CLI, adapters, or dbt interop. Covers the verify loop (compile, plan, build), comparing data or two queries with sqb diff, inspecting macro/enum/constant visibility with sqb scope, impact analysis with sqb lineage, and where to find exact syntax. --- - +# SQLBuild -# SQLBuild Skill +SQLBuild builds SQL and Python data pipelines. It looks like dbt (SQL models, refs, sources, seeds, +selectors) but compiles and checks the whole project offline first, explains every build with a +plan, and ships tools for proving a change is correct. Use those tools; do not stop at "it ran". -This file is generated from the SQLBuild documentation. Use it as the source of truth for SQLBuild syntax, project structure, configuration, testing, adapters, and CLI behavior. +## How SQLBuild differs from dbt -## Included Pages - -- `index` -- `quickstart` -- `feature-comparison` -- `benchmarks` -- `concepts/dbt-compatibility/overview` -- `concepts/dbt-compatibility/selection` -- `concepts/dbt-compatibility/adding-sqlbuild-models` -- `concepts/project-configuration` -- `concepts/resource-identities` -- `concepts/adapters` -- `concepts/adapters/duckdb` -- `concepts/adapters/motherduck` -- `concepts/adapters/snowflake` -- `concepts/adapters/bigquery` -- `concepts/adapters/databricks` -- `concepts/adapters/postgres` -- `concepts/adapters/sqlserver` -- `concepts/sources` -- `concepts/seeds` -- `concepts/models` -- `concepts/models/materializations` -- `concepts/models/schemas` -- `concepts/models/type-enforcement` -- `concepts/models/contracts` -- `concepts/models/hooks` -- `concepts/models/hooks/sql` -- `concepts/models/hooks/python` -- `concepts/models/configuration` -- `concepts/enums` -- `concepts/constants` -- `concepts/constants/collections-and-rendering` -- `concepts/enums/model-contracts` -- `concepts/model-private-values` -- `concepts/macros` -- `concepts/macros/composition-and-context` -- `concepts/interpolation` -- `concepts/functions` -- `concepts/incremental` -- `concepts/planning` -- `concepts/planning/cascade-propagation` -- `concepts/planning/source-freshness` -- `concepts/planning/selection-and-staleness` -- `concepts/snapshots` -- `concepts/audits` -- `concepts/rules` -- `concepts/rules/configuration-and-selection` -- `concepts/rules/findings-and-exceptions` -- `concepts/rules/execution-and-caching` -- `concepts/testing` -- `concepts/scenarios` -- `concepts/selectors` -- `concepts/column-lineage` -- `concepts/diff` -- `concepts/observability` -- `concepts/observability/sinks` -- `concepts/declaration-scopes` -- `concepts/declaration-scopes/visibility` -- `concepts/declaration-scopes/placement` -- `concepts/declaration-scopes/explorer` -- `concepts/rules/custom-rules/overview` -- `concepts/rules/custom-rules/rule-context` -- `concepts/rules/custom-rules/testing-and-determinism` -- `concepts/python-nodes/overview` -- `concepts/python-nodes/loaders` -- `concepts/python-nodes/tasks` -- `concepts/python-nodes/assets` -- `concepts/python-nodes/checks` -- `concepts/python-nodes/factories` -- `concepts/python-nodes/providers` -- `concepts/python-nodes/sql-references` -- `concepts/virtual-environments` -- `concepts/virtual-environments/setup` -- `concepts/virtual-environments/building` -- `concepts/virtual-environments/promotion` -- `concepts/virtual-environments/rollback` -- `concepts/virtual-environments/adopt-detach` -- `concepts/virtual-environments/clone` -- `concepts/virtual-environments/diff` -- `concepts/virtual-environments/reconcile` -- `concepts/virtual-environments/locks` -- `concepts/virtual-environments/janitor` -- `concepts/virtual-environments/recovery` -- `integrations/dagster` -- `integrations/dagster-reference` -- `integrations/rivers` -- `integrations/dlt` -- `integrations/ingestr` -- `cli/init` -- `cli/playground` -- `cli/skills` -- `cli/compile` -- `cli/format` -- `cli/contract` -- `cli/scope` -- `cli/rules` -- `cli/plan` -- `cli/build` -- `cli/load` -- `cli/seed` -- `cli/test` -- `cli/scenario` -- `cli/audit` -- `cli/freshness` -- `cli/check` -- `cli/clone` -- `cli/diff` -- `cli/lineage` -- `cli/dag` -- `cli/query` -- `cli/debug` -- `cli/janitor` -- `cli/clean` -- `cli/dbt` -- `cli/state` -- `cli/promote` -- `cli/rollback` -- `cli/reconcile` -- `concepts/enums-and-constants` - -## Introduction - -Source: `index.mdx` - -Verify early, test properly, and deploy reversibly. SQL pipelines with the rigor of real software - free and open source. - -**Valid isn't the same as correct.** Your SQL compiles, runs, and returns rows - none of that means the number is right, and a silently-wrong number a stakeholder already trusted is the bug that actually hurts. - -SQLBuild brings software-engineering rigor to SQL pipelines: **verify early, test properly, deploy reversibly.** Catch errors before the warehouse runs them and test your logic locally - with change-aware builds and reversible deploys there when you need them, not forced on you on day one. It is a standalone framework for building SQL and Python data pipelines - free and open source. - -### Test properly - -Most data testing checks for nulls and uniqueness. That tells you a column isn't empty, not that your logic is right. SQLBuild tests the logic. - -- **Multi-model tests.** Mock your sources, assert on the model you care about, and SQLBuild resolves every intermediate model from its real SQL. One test file can cover a whole stretch of your pipeline. -- **E2E scenario tests with local replay.** Build the real model graph against coherent fixtures in isolated relations, capture the result as JSONL, and replay it locally through DuckDB. CI runs full end-to-end tests with zero warehouse credentials or compute. -- **Macro-powered mocks.** Tests are SQL, so they support macro calls - write reusable mock generators instead of copy-pasting fixtures. - -```sql --- Mock two sources, assert on the final mart. --- stg_orders and stg_payments resolve automatically from their real SQL. -TEST(); - -WITH -__source__raw__orders AS ( - @mock_orders() -), -__source__raw__payments AS ( - SELECT - 1 AS payment_id, - 1 AS order_id, - 1500 AS amount_cents, - 'credit_card' AS method -), -__expected__fact_orders AS ( - SELECT - 1 AS order_id, - 100 AS customer_id, - 1500 AS total_cents, - 'credit_card' AS payment_method -) -SELECT 1 -``` - -See [Testing](/concepts/testing) for SQL unit tests and [Scenarios](/concepts/scenarios) for local -E2E replay. - -### Verify early - -Before any model runs, SQLBuild does static analysis of your project - offline, no warehouse connection needed. - -- **Catch errors at compile.** SQL syntax, type inference, contract checks, and column lineage all run before execution. A bad reference or a type mismatch fails at compile, with an error that points at the line - not halfway through a warehouse run. -- **Turn review decisions into compiler checks.** Configure [Rules](/concepts/rules) over compiler-owned SQL, models, dependencies, contracts, tests, and project paths. -- **Format separately.** `sqb format` owns deterministic source rewriting; Rules remain diagnostic-only. -- **Fast, because it's Rust where it counts.** Static analysis runs on [Polyglot](https://github.com/tobilg/polyglot), a Rust SQL engine (MIT, 32+ dialects), so compile stays quick even on large projects. -- **Open, not paywalled.** The static analysis is part of the Apache-2.0 core - no proprietary engine, no separate login, and no paid tier gating the smart checks. - -### Plans that explain every build - -**Start simple.** By default, `sqb build` runs your full selection - the same predictable mental model as dbt, with nothing to configure. For many projects, that is all they will ever need. - -Every model, seed, function, and Python node has a versioned identity. SQLBuild uses those identities -to explain query and configuration changes, propagate upstream changes, choose incremental replay -actions, and protect scoped builds from stale dependencies. Direct mode still runs the selected -scope; change detection makes that plan explicit rather than silently changing what was selected. - -[Virtual environments](/concepts/virtual-environments) add version reuse and optional stale-driven -execution when a project needs isolated previews, promotion, and rollback. - -The planner carries that evidence across the whole graph: - -- **Models and functions:** query, configuration, and dependency identities explain local and cascaded changes. -- **Seeds:** content and load-affecting configuration identify when a seed changed. -- **Audits:** audit evidence is associated with the model version it validated. -- **Source freshness:** observable source data versions can affect downstream plans, with lag tolerance to avoid jitter. -- **Cascade propagation:** when a model does change, the signal propagates downstream, with configurable replay windows (`replay_on_change`). -- **Python nodes:** source and dependency identities appear in plans; skip/run remains user-controlled through `ctx.skip()`. - -In direct mode, this evidence is stored as append-only rows in your warehouse. Virtual environments -use their configured state backend for environment-scoped version bindings. - -### Deploy reversibly (opt-in) - -By default, SQLBuild runs in direct mode with state as append-only rows in your warehouse. When you want more, [virtual environments](/concepts/virtual-environments) add a reversibility layer on top: - -- **Instant branching, promotion, and rollback** as low-copy pointer operations. -- **Partial promotion.** Promote the models that are ready without re-running everything downstream of them - you don't have to rebuild the whole closure to ship one fix. -- **Checkpoints and reconciliation** so a bad change is something you undo, not an incident. - -Virtual environments are opt-in, not a tax you pay upfront - direct mode stays the default, so the floor stays low and you reach for them only when a workflow needs them. - -### Supported adapters - -SQLBuild works with **DuckDB, MotherDuck, Snowflake, BigQuery, Databricks, PostgreSQL, and SQL Server** today. Support for ClickHouse, Redshift, Trino, Spark, and Athena is on the way. - -DuckDB runs entirely locally, so you can try SQLBuild and run full E2E tests without any warehouse credentials. Head to the [Quickstart](/quickstart) to get a project running in minutes, or see [Adapters](/concepts/adapters) for connection setup. - -### More in SQLBuild - -#### Familiar SQL models - -Models are SQL files with a `MODEL()` header and a `SELECT`. References to other models and sources use `__ref()` and `__source()`, and configuration, schema, and audits are declared inline in the header. If you know dbt or SQLMesh, you already know the shape. +- Models are SQL files with a `MODEL (...)` header, then one query. No Jinja. +- References are `__ref("model")`, `__source("source_name")`, `__seed("seed")`. Never hardcode a + relation SQLBuild manages. +- Macros are plain Python functions called as `@macro_name("arg")`. Constants and enums are + `@const("name")` and `@enum("name").MEMBER`. +- Config, column schemas, contracts and audits live in the `MODEL()` header, not YAML. +- `sqb compile` is offline and strict: syntax, binding, types, contracts, column lineage and Rules + all run before anything touches the warehouse. Fix compile errors; do not bypass them. +- Declarations are scoped by folder. A macro, enum or constant must live in the narrowest folder + that contains every use, or compile rejects it. See `sqb scope` below. +- Plans record why each model runs (first run, query change, config change, upstream change). +- Destructive changes are gated: snapshot full refreshes, permanent-to-transient table changes and + time travel retention decreases need an explicit policy or flag. Never pass an `--allow-*` + flag without the user's say-so. ```sql -- models/marts/fact_orders.sql @@ -242,15462 +35,129 @@ MODEL ( ), ); -SELECT - o.order_id, - o.customer_id, - p.amount_cents AS total_cents, - p.method AS payment_method +SELECT o.order_id, o.customer_id, p.amount_cents AS total_cents FROM __ref("stg_orders") o JOIN __ref("stg_payments") p USING (order_id) ``` -#### Python macros, not Jinja - -- **Real Python functions:** Macros are plain Python, called with `@macro()` in SQL. Testable, debuggable, and composable with standard tooling - no Jinja, no string-templating language to wrangle. - -```python -# macros/grant_target.py -def grant_target(target): - return f"GRANT SELECT ON {target} TO analyst_role" -``` - -#### Audits that block bad data - -- **Full table builds:** SQLBuild materializes into a staging table and runs `error`-severity audits before promotion. If any fail, the swap is blocked and the production table is untouched. -- **Incremental models:** Delta-phase audits validate each batch before DML is applied. Bad data is caught before it reaches the target. -- **Measured quality:** Measurement audits evaluate values against warning/error thresholds, distinguish insufficient samples, and can retain bounded evidence and immutable result history. - -#### Incremental processing - -- **Cursor-based replay:** SQLBuild resumes by reading the highest timestamp or integer value already in the target table. If a model fails for several runs, the next successful build picks up from the last data it actually wrote, with no manual backfilling. -- **Microbatch mode:** Split large replay windows into configurable batches, each with its own audit cycle. Or process the full range in one pass, the choice is per-model. -- **Replay on change:** When a model's version identity changes, `replay_on_change` controls how much data to reprocess: `forward` (default, just run the next delta), `bounded-14d` (replay a window), or `full` (rebuild the table). - -#### Python you can read and extend - -- **The framework is Python.** Adapters, macros, hooks, providers, custom materializations, and Python nodes are all plain Python you can read and extend. (The SQL analysis underneath runs on Rust via Polyglot - see [Verify early](#verify-early) - so the code you work with stays Python while compile stays fast.) - -#### Testing in depth - -Beyond the multi-model tests shown above, SQLBuild runs end-to-end scenario tests against the real model graph, with local replay and property assertions. - -- **Real graph execution:** Define coherent fixture inputs, and SQLBuild builds the real model graph against them in isolated warehouse relations. Test end-to-end business logic across your entire pipeline, not just one model at a time. -- **Local replay without a warehouse:** Capture scenario fixtures as JSONL snapshots, then replay them locally through DuckDB. CI pipelines run full E2E tests with zero warehouse credentials or compute cost. -- **Zero-row assertions:** Write property checks that pass when no rows violate a condition. Useful for invariants like "no negative revenue" or "no duplicate customer IDs" alongside full expected-output comparisons. - -```sql -SCENARIO ( - description "Daily revenue includes only successful payments", - tags ["revenue"] -); - -WITH -__ref__stg_orders AS ( - SELECT 1 AS order_id, 10 AS customer_id, 'completed' AS status -), -__ref__stg_payments AS ( - SELECT - 1 AS payment_id, - 1 AS order_id, - 1700 AS amount_cents, - 'success' AS payment_status -), -__expected__daily_revenue AS ( - SELECT - CAST('2026-04-01' AS DATE) AS revenue_date, - 1700 AS total_revenue_cents -), -__assert__all_orders_have_payments AS ( - SELECT * FROM __ref("fact_orders") WHERE payment_amount_cents IS NULL -) -SELECT 1 -``` - -#### Python nodes - -Grow beyond warehouse-only SQL without leaving the graph. Python nodes are ordinary functions, decorated to become first-class nodes in the same DAG as your SQL models, and they run as part of `sqb build`. There are four kinds: - -- **Loaders** (`@loader`) load external data into managed sources, with incremental write strategies (table, append, delete_insert, merge), cursor-based loading, and concurrent execution. -- **Tasks** (`@task`) run Python computation or side effects as graph nodes. -- **Assets** (`@asset`) produce or observe external artifacts, with optional columns and lineage. -- **Checks** (`@check`) validate tasks, assets, and loaders, and run during `sqb build` or on their own with `sqb check`. - -```python -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_orders(ctx: LoaderContext): - if ctx.current_cursor_value is None: - return fetch_all_orders() - return fetch_orders_since(ctx.current_cursor_value) -``` - -SQL models never depend on Python nodes - the only path from Python into SQL is a loader populating a source - so the SQL graph stays fully analyzable on its own. Nodes can also be generated programmatically with `@factory`. See [Python Nodes](/concepts/python-nodes/overview). - -#### Multi-target workflows - -- **Data diffs:** Compare schemas and row-level data between targets (or virtual environments) with `sqb diff prod:dev`. - -- **Zero-copy cloning:** Branch targets instantly with `sqb clone` without duplicating data. -- **Deferred references:** Resolve a production namespace with `--defer-to` through the active target's connection while building in dev. -- **No manifest required:** Clone, diff, and defer work directly against live targets. No `manifest.json` generation, no artifact management, no stale state. - -#### Extensibility - -- **User-defined functions:** SQL and Python UDFs managed as part of your project. Functions participate in the DAG - definition changes trigger rebuilds of dependent models. Table functions provide predicate-pushdown-friendly alternatives to final-layer views. -- **Custom materializations:** Write materialization logic in Python with full framework integration - including audit hooks, schema change signals, and query change detection. - -```python -def materialize(ctx: MaterializationContext) -> MaterializationResult: - stale = find_untracked_partitions(ctx) - - for partition in stale: - ctx.adapter.create_table_as(ctx.connection, target=staging, sql=partition_sql) - ctx.run_audits(staging) # same audit guarantees as built-in types - ctx.execute_sql(f"INSERT INTO {ctx.destination} SELECT * FROM {staging}") - - return MaterializationResult(relation=ctx.destination) -``` -- **Path-between selectors:** `--select fact_orders~daily_activity_rollup` selects every model on the shortest path between two nodes, with optional upstream/downstream expansion. - -### What's next - -- **Broader adapter support** - ClickHouse, Redshift, Trino, Spark, Athena - -### Quick links - - Get a project running locally in minutes. - Reference for SQLBuild commands and flags. - Understand models, incremental strategies, audits, and selectors. - E2E tests with local DuckDB replay. - -## Quickstart - -Source: `quickstart.mdx` - -Get a SQLBuild project running locally with DuckDB in under a minute. - -This guide walks you through creating and running a complete transformation project - views, tables, incremental models, audits, tests, and a custom materialization - all running locally against DuckDB. No external data setup or warehouse credentials required. - -### Prerequisites - -- Python 3.12+ -- SQLBuild installed: `uv pip install sqlbuild` or `pip install sqlbuild` - -Confirm the installed release with `sqb --version`. - -### 1. Create the playground - -```bash -sqb playground waffle-shop -cd waffle-shop -``` - -This creates a self-contained waffle shop project with everything you need to explore SQLBuild. - -The playground creates the shared `sqlbuild_project.toml`; that is sufficient for this -quickstart. In a team project, you can additionally create a gitignored -`sqlbuild_local.toml` to select your usual target and override your personal development -schema or credentials without changing the committed project config. See -[Project Configuration](/concepts/project-configuration#sqlbuild_local-toml). - -### 2. Plan - -Preview what SQLBuild will do: - -```bash -sqb plan -``` - -``` -Plan ready (13 selected) - -First run (12) - stg_customers view - stg_payments view - stg_orders view - daily_order_partitioned partition_tracked (custom) - daily_revenue table - dim_customers table - fact_orders table - customer_status_snapshot merge (timestamp) - hourly_order_activity delete_insert (timestamp, microbatch) - daily_activity_rollup delete_insert (timestamp, microbatch) - hourly_activity_with_daily_context delete_insert (timestamp, microbatch) - order_status_index delete_insert (integer) - -Seeds (1) - waffle_types -``` - -Source data is defined as inline expressions in `sources/raw.yml`, so SQLBuild resolves everything at compile time - no seeding step needed. - -### 3. Build - -Execute the full build: - -```bash -sqb --no-color build -``` - -SQLBuild materializes all models in DAG order. SQL unit tests run before their target models. Error-severity audits run against a staging table before promotion - if any fail, the production table is untouched: - -``` -Execution sqb build (concurrency: 1) - - 1/13 seed waffle_types OK 0.09s - 2/13 view stg_customers OK 0.05s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - audit not_null (email) PASS - 3/13 view stg_payments OK 0.02s - audit not_null (payment_id) PASS - audit unique (payment_id) PASS - audit not_null (order_id) PASS - 4/13 view stg_orders OK 0.03s - test test_stg_orders PASS - audit not_null (order_id) PASS - audit unique (order_id) PASS - audit not_null (customer_id) PASS - audit accepted_values (status) PASS - 5/13 custom daily_order_partitioned OK 0.17s - audit expression_is_true PASS 4/4 - audit not_null (order_date) PASS 4/4 - 6/13 table daily_revenue OK 0.05s - audit expression_is_true PASS - 7/13 table dim_customers OK 0.05s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - 8/13 table fact_orders OK 0.04s - audit not_null (order_id) PASS - 9/13 table customer_status_snapshot (merge) OK 0.04s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - 10/13 table hourly_order_activity (delete_insert) OK 0.16s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS - audit (f) not_null (activity_hour) PASS - 11/13 table order_status_index (delete_insert) OK 0.03s - audit not_null (order_id) PASS - audit unique (order_id) PASS - 12/13 table daily_activity_rollup (delete_insert) OK 0.08s - audit (d) expression_is_true PASS 2/2 - audit (d) not_null (activity_day) PASS 2/2 - audit (f) expression_is_true PASS - audit (f) not_null (activity_day) PASS - 13/13 table hourly_activity_with_daily_context (delete_insert) OK 0.17s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS - audit (f) not_null (activity_hour) PASS - -Completed successfully. -PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) -``` - -Notice `audit (d)` and `audit (f)` on incremental models - these are delta-phase and final-phase audits. Delta audits validate each batch before DML is applied; final audits run against the target after promotion. - -### 4. Verify - -Run the plan again to see the steady-state: - -```bash -sqb plan -``` - -``` -Plan ready (13 selected) - -Normal (12) - 3 view - 3 table - 3 delete_insert (timestamp, microbatch) - 1 partition_tracked (custom) - 1 merge (timestamp) - 1 delete_insert (integer) - -Seeds (1) - waffle_types -``` - -All models now show `Normal` instead of `First run`. - -### 5. Common operations - -```bash -# Rebuild a specific model -sqb build --select daily_revenue - -# Rebuild models under a path -sqb build --select path:models/marts - -# Full refresh of everything -sqb build --full-refresh - -# Run tests only -sqb test - -# Run audits only -sqb audit - -# Inspect lineage -sqb lineage fact_orders --direction both - -# Compile and check contracts -sqb compile -``` - -### What you just built - -#### Model DAG - -``` -raw__customers ──> stg_customers ──> dim_customers -raw__orders ────> stg_orders ────> fact_orders ──> customer_status_snapshot -raw__payments ──> stg_payments ──> │ order_status_index - │ hourly_order_activity ──> daily_activity_rollup - │ hourly_activity_with_daily_context - └──> daily_revenue - └──> daily_order_partitioned -``` - -#### Materialization types - -The project demonstrates several materialization strategies: - -- **Views** (`stg_customers`, `stg_orders`, `stg_payments`) - lightweight staging layer -- **Tables** (`fact_orders`, `dim_customers`, `daily_revenue`) - full table rebuilds -- **Merge incremental** (`customer_status_snapshot`) - upsert with timestamp cursor -- **Delete/insert incremental** (`hourly_order_activity`, `daily_activity_rollup`, `hourly_activity_with_daily_context`) - microbatch processing with configurable batch sizes -- **Integer cursor incremental** (`order_status_index`) - non-timestamp incremental -- **Custom materialization** (`daily_order_partitioned`) - Python-defined partition-tracked strategy - -See [Incremental](/concepts/incremental) for details on cursor-based strategies and microbatch processing. - -#### Project structure - -``` -waffle-shop/ - sqlbuild_project.toml # project config - sources/ - raw.yml # source declarations (inline expression data) - seeds/ - waffle_types.csv # seed data - lookups.yml # seed declarations - functions/ - sql/ - udf__is_completed_order.sql # scalar SQL UDF - table_fn__customer_orders.sql # table function - schemas/ - orders/ - order.sql # reusable model column schema - models/ - staging/ - stg_customers.sql # view (audits declared in MODEL() header) - stg_orders.sql # view - stg_payments.sql # view - marts/ - fact_orders.sql # table - dim_customers.sql # table - daily_revenue.sql # table - daily_order_partitioned.sql # custom materialization - hourly_order_activity.sql # microbatch incremental - hourly_activity_with_daily_context.sql # microbatch incremental - daily_activity_rollup.sql # microbatch incremental - intermediate/ - customer_status_snapshot.sql # merge incremental - order_status_index.sql # integer cursor incremental - audits/ - generic/ - expression_is_true.sql # custom generic audit - tests/ - unit/ - test_stg_orders.sql # model unit test - test_fact_orders.sql # multi-model test - test_daily_revenue_chain.sql # multi-model test - test_line_total_cents_macro.sql # macro test - scenarios/ - daily_revenue_minimal.sql # E2E scenario test - daily_revenue_multi_order.sql # E2E scenario test - macros/ - currency.py # cents_to_dollars macro - datetime.py # adapter-portable timestamp_trunc - materializations/ - partition_tracked.py # custom materialization -``` - -### Next steps - -- [Models](/concepts/models) - understand `MODEL()` headers and materialization types -- [Rules](/concepts/rules) - add compile-time SQL, architecture, and model-shape checks -- [Functions](/concepts/functions) - SQL UDFs, Python UDFs, and table functions -- [Incremental](/concepts/incremental) - learn about cursor-based incremental strategies -- [Audits](/concepts/audits) - configure data quality checks -- [Testing](/concepts/testing) - write SQL unit tests and multi-model tests with macro support -- [Column Lineage](/concepts/column-lineage) - trace individual columns through your pipeline -- [CLI Reference](/cli/build) - full command reference - -## Feature Comparison - -Source: `feature-comparison.mdx` - -Feature comparison between SQLBuild, dbt, and SQLMesh. - -SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common ground but differ in design philosophy and feature focus. - -### Feature comparison - -#### Testing - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Multi-model tests | Chain across multiple models | YAML-stub, single model | CTE-based, single model | -| Typed parameterized unit tests | Independent named cases with adapter-rendered scalar values | No | No | -| Macros as test helpers | Tests are SQL - macros work as reusable fixture generators | No (YAML stubs) | No | -| E2E scenario tests | Fixture worlds with real graph execution | No | No | -| Local E2E replay | Capture from warehouse, replay in DuckDB | No | No | -| Macro / UDF / table function tests | `TEST(mode macro)`, `TEST(mode udf)`, or `TEST(mode table_fn)` | No | No | -| Zero-row assertions | `__assert__` CTEs in tests and scenarios | No | No | -| Failure diagnostics | Bounded, redacted unexpected and missing row samples in text and JSON | Adapter/tool dependent | Row diffs | - -#### Audits - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Built-in audits | not_null, unique, accepted_values, relationships | not_null, unique, accepted_values, relationships | Extensive (statistical, string pattern, etc.) | -| Blocking audits | Block promotion from staging table | Tests run after materialization | Audits gate plan application; run-time audits execute after the interval is materialized | -| Delta/interval-scoped audits | Per-microbatch audit cycle before DML | No | Audit query filtered to processed intervals for time-range models | -| Measurement audits | Thresholds, minimum samples, bounded evidence, and immutable result history | Package/custom test patterns | Custom audits | -| Audit factories | Typed Python factories generate equivalent reviewed audit instances | Macros/packages | Python audit definitions | - -#### Compilation - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| SQL analysis | Offline syntax, binding, type inference, semantic validation, and lineage (Polyglot) | dbt Core: none; dbt Fusion engine: compile-time (proprietary; built on Apache-2.0 dbt Core v2) | Compile-time (SQLGlot) | -| Focused compilation | Full graph integrity with deep analysis limited to selection and required upstream closure | Selected compilation | Selected planning | -| Column-level lineage | Compile-time, fast and rich modes | dbt Core: post-hoc via docs; dbt Fusion engine: compile-time | Compile-time | -| Column contract validation | Compile-time inference plus runtime enforcement with `contract enforced` | YAML schema contracts at runtime | Schema contracts via plan | -| Existing-schema contracts | Online read-only diff and safe repository generation | Codegen/packages | External schema tooling | -| Rules and formatting | Compiler-integrated native and custom diagnostics, reasoned suppressions, and separate canonical formatting | External tools | Built-in formatter plus external linting | -| Compiler-integrated Rules | Native and custom rules over compiler-owned SQL, models, dependencies, contracts, tests, and project paths | Project conventions through packages and external tooling | Built-in audits and external linting | -| SQL transpilation | For local E2E replay into DuckDB | No | For cross-dialect model execution | -| Python macros | `@macro()` syntax | No (Jinja only) | SQLMesh macro syntax | -| Compiler-enforced declaration scopes | Project, descendant-public, exact-owner-private, and model-private tiers with offline `sqb scope` inspection | No lexical declaration scopes | No lexical declaration scopes | -| Jinja support | No (Python macros instead) | Yes (core templating) | Yes | - -#### Incremental - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Incremental strategies | append, delete_insert, merge, SCD Type 2 | append, delete_insert, merge, snapshots | delete_insert (time-range), merge (unique-key), SCD Type 2, partition | -| Microbatch execution | Watermark and rolling-window strategies, per-batch audits, limits, and opt-in concurrency | Microbatch | Batch size support | -| Interval progress | Sequential runs derive progress from target/input cursors; concurrent runs coordinate with append-only facts | No | Tracks intervals in a state store | -| SCD Type 2 models | Timestamp and check strategies, historical input, hard deletes | Snapshots (timestamp and check strategies) | `SCD_TYPE_2` model kind (timestamp and check strategies) | - -#### Planning and change detection - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Stale-driven virtual builds | Optional `--changes-only` execution over VDE version bindings | dbt State (paid) | Version hash comparison | -| Warehouse-native state | Append-only tables in the warehouse; no external state database | manifest.json artifacts | Requires external state store (SQLite/PostgreSQL) | -| Source freshness | `sqb freshness` with adapter/column/sql strategies, lag tolerance, and CI gating | `dbt source freshness` | No dedicated freshness command; `signals` gate model evaluation until external data is ready | -| Reuse across environments | Virtual environments reuse fingerprint-matched physical tables across environments (shared physical storage) | dbt State clone (paid) | Virtual environments reuse fingerprint-matched physical tables across environments (shared physical storage) | -| Cascade propagation | Topological walk with `replay_on_change` policy inheritance and override | No cascade control | Cascades through version hashes | - -#### Environments - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Virtual environments | Pointer swaps with hash-based version reuse (opt-in) | No | Pointer swaps, no compute cost | -| Data diffs | Full row-level data comparison across targets or virtual environments | No | Table diff | -| Zero-copy cloning | `sqb clone` | No | No | - -#### Models - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| SQL models | `MODEL()` header with inline config | Jinja-templated SQL + YAML sidecar | `MODEL` DDL | -| Python models | Coming soon | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | -| Custom materializations | Python with full framework hooks | Jinja-based | Python-based custom model kinds | -| Lifecycle hooks | Typed inline SQL, reusable parameterized SQL resources, and Python hooks with compile-time validation and `HookContext` | Jinja pre/post hooks | Python pre/post hooks | - -#### Python nodes - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Tasks | `@task` - Python computation as DAG nodes | No | No | -| Assets | `@asset` - external artifact production/observation | No | No | -| Checks | `@check` - Python validation of tasks, assets, and loaders | No | No | -| Factories | `@factory` - programmatic node generation | No | No | -| Providers | Shared runtime services with name-based injection into nodes and hooks | No | No | - -#### dbt interoperability - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| dbt compatibility | Reads dbt manifests, coordinates dbt and SQLBuild selection, and supports SQLBuild models downstream | N/A | Jinja compatibility layer plus own macro system | - -#### Sources - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Source loaders | Python `@loader` functions with table/append/delete_insert/merge strategies | No (external to dbt) | No (external to SQLMesh) | -| Declarative ingestion | dlt and ingestr integrations - YAML-only source config, no Python | No | No | -| Auto-load during builds | Managed sources loaded before dependent models | No | No | -| Source deferral | `--defer-sources-to` reads source data from another target | No | No | - -#### Other - -| Feature | SQLBuild | dbt | SQLMesh | -|---------|----------|-----|---------| -| Reference syntax | `__ref()` - parses as valid SQL | `{{ ref() }}` - Jinja template | `model_name` with dependency tracking | -| Adapters | DuckDB, MotherDuck, Snowflake, BigQuery, Databricks, PostgreSQL, SQL Server | 30+ (community adapters) | DuckDB, Snowflake, BigQuery, Databricks, Spark, Redshift, Postgres, Trino, MySQL | -| State requirements | Stateless by default | manifest.json + target/ | Requires state store (local database or PostgreSQL for production) | -| Playground | `sqb playground` | Clone example repo | Example project | -| AI agent skills | General guidance with `sqb skills`; Rules guidance with `sqb rules skills` | No | No | -| Execution observability | Immutable schema-versioned lifecycle facts, invocation-local ordering, orchestration context, and typed project sinks | Events/artifacts plus external observability | Plans, state, and external observability | - -### Where each tool fits - -| Tool | Best for | -|------|----------| -| **SQLBuild** | Rigor-first SQL pipelines: compile-time verification, pre-promotion audit gating, multi-model tests, and local E2E replay by default. Opt into change-aware builds and warehouse-native state when full rebuilds get expensive, plus ingestion, Python nodes, and virtual environments as the project grows. | -| **dbt** | The most widely adopted SQL transformation framework with the largest adapter and community ecosystem. | -| **SQLMesh** | State-managed pipelines with virtual environments, interval tracking, and cross-dialect transpilation. | - -### Not yet in SQLBuild - -- **Broader adapter support** - ClickHouse, Redshift, Trino, Spark, Athena - -## Benchmarks - -Source: `benchmarks.mdx` - -End-to-end compiler and Rules performance on deterministic, production-shaped projects. - -SQLBuild benchmarks run the real `sqb compile` command. They measure complete compiler workflows, -including project discovery, parsing, dependency resolution, SQL expansion and analysis, contracts, -tests, Rules, cache publication, and artifact generation. - -The benchmark projects are generated so each run is deterministic and reproducible. Their shape is -not arbitrary: the reference workload was calibrated from a real working SQLBuild project, including -its resource ratios, graph structure, SQL-size distribution, tests, audits, macros, functions, -sources, seeds, and hooks. - -The generated projects contain neutral names and data. They do not contain source code or business -data from the reference project. - -### Production-shaped workload - -The reference compiler workload reproduces the measured characteristics of a working SQLBuild -project: - -| Characteristic | Observed project | Generated benchmark | -|---|---:|---:| -| Models | 976 | 976 | -| Sources | 232 | 232 | -| Seeds | 46 | 46 | -| SQL functions | 23 | 23 | -| Python macros | 12 | 12 | -| Attached audits | 731 | 700 | -| Native SQL test cases | 130 | 130 | -| SQL hooks | 2 | 2 | -| Execution layers | 54 | 54 | -| Authored model SQL | 6.05 MB | about 6.4 MB | -| Median model size | 1.9 KB | 1.9 KB | -| 75th-percentile model size | 4.5 KB | 4.5 KB | -| 90th-percentile model size | 10.9 KB | 11.0 KB | -| 95th-percentile model size | 15.2 KB | 15.2 KB | -| 99th-percentile model size | 48.2 KB | 48.0 KB | -| Largest model | 519 KB | 520 KB | - -The generated dependency graph combines a 54-layer spine with bounded model chains. The workload -also includes path defaults, reusable schemas, enforced contracts, attached audits, SQL hooks, -source and seed references, SQL functions, composed Python macros, CTEs, derived tables, assertions, -repeated test targets, and test fixtures ranging from 40 to 200 rows. - -This preserves the characteristics that exercise the compiler without publishing or depending on -one specific project. - -### Compiler performance - -On the production-shaped 976-model workload, a required CI run measured: - -| Compile path | End-to-end `sqb compile` time | -|---|---:| -| First compile | 3.21s | -| Unchanged recompile | 0.60s | -| Recompile after a leaf-model edit | 0.78s | -| Recompile after a central-model edit | 0.67s | -| Recompile after a SQL-test edit | 0.70s | -| Recompile after a Python-macro edit | 0.94s | -| Recompile after a project-configuration edit | 3.56s | - -Model, test, and macro edits reuse unaffected compiler work. The project-configuration scenario -deliberately changes an input with broad impact and therefore performs wider recomputation. - -#### Model and SQL-volume scaling - -A separate compiler profile increases model count and authored SQL volume while retaining the -measured model-size distribution: - -| Models | Authored model SQL | First compile | Unchanged recompile | -|---:|---:|---:|---:| -| 3,000 | 18–25 MB | 2.37s | 1.84s | -| 10,000 | 60–80 MB | 7.50s | 5.09s | - -This profile isolates compiler scaling by model count and SQL volume. The production-shaped profile -above is the broader test of sources, seeds, functions, macros, tests, audits, hooks, contracts, and -artifact generation occurring together. - -### Rules performance - -The Rules benchmark scales the same production-shaped generator to 5,000 models. The resulting -project contains: - -| Characteristic | Generated workload | -|---|---:| -| Models | 5,000 | -| Sources | 1,189 | -| Seeds | 236 | -| SQL functions | 118 | -| Python macros | 61 | -| Native SQL tests | 666 | -| Attached audits | 3,586 | -| SQL hooks | 2 | -| Authored model SQL | 31.3 MB | -| Authored SQL-test definitions | 6.4 MB | - -On this project, 20 custom Rules complete their first compile in just over 10 seconds. An unchanged -compile completes in under six seconds. - -Even with 100 custom Rules, an unchanged compile completes in just over seven seconds. - -| Compile | 20 Custom Rules | 100 Custom Rules | -|---|---:|---:| -| First compile (no cache) | 10.83s | 12.19s | -| Unchanged compile | 5.81s | 7.24s | -| After editing 1% of models | 7.43s | 10.02s | - -Each measurement executes the complete `sqb compile` command. The edit scenario changes 50 of the -5,000 models before recompiling. - -Because most Rules in this profile evaluate once per model, the 20- and 100-Rule configurations -perform roughly 100,000 and 500,000 Rule evaluations respectively. - -Increasing the custom Rule pack from 20 to 100 multiplies the number of Rule evaluations by five, -while the unchanged compile median increases from 5.81 to 7.24 seconds. - -#### Rule implementation changes - -Changing a Rule or one of its imported helpers participates in cache invalidation. - -In the 5,000-model, 100-Rule profile, editing one custom Rule invalidated exactly 5,001 evaluations -while preserving 495,001 cached results. The complete compile finished in a 9.89-second median. - -CI also verifies exact invalidation for model edits, SQL-test edits, macro edits, project -configuration, directly observed project inputs, custom Rule source, and imported custom helpers. A -cache result with an unexpected hit or miss count fails the benchmark. - -### Methodology - -Compiler and Rules benchmarks run on an 8-vCPU, 32 GB Linux CI runner class using Python 3.12 and -Polyglot 0.9.2. - -The Rules results shown above are the median of five required-CI runs. Allocated CPUs across those -runs were AMD EPYC or Intel Xeon processors. - -The Rules benchmark output records: - -- complete CLI elapsed time; -- compiler phase timings; -- Rule cache hits and misses; -- Rule cache size; -- peak resident memory; -- workload resource counts; -- SQLBuild, Python, and Polyglot versions; -- every raw timing and memory sample. - -The required Rules CI profile enforces profile-specific latency, cache-size, peak-memory, timeout, -and aggregate compiler budgets. Cache hit and miss expectations are exact. - -These measurements cover compilation and static analysis. They do not measure warehouse query -execution. - -### Reproduce the benchmarks - -Run the five-iteration Rules publication profile: - -```bash -uv run python -m scripts.benchmark_rules \ - --models 5000 \ - --iterations 5 \ - --output rules-5000.json -``` - -Run the bounded profile used by required CI: - -```bash -uv run python -m scripts.benchmark_rules_ci \ - --output rules-performance-ci.json \ - --summary-output rules-performance-ci.md -``` - -Run the compiler performance guards: - -```bash -uv run pytest \ - tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_performance.py \ - -n auto \ - --dist loadfile -``` - -The benchmark generators, guard thresholds, and CI workflow are maintained in the public -[SQLBuild repository](https://github.com/chio-labs/sqlbuild). +## The working loop -## Using SQLBuild with dbt +1. **Orient.** Read `sqlbuild_project.toml` (and `sqlbuild_local.toml` if present) and look at + neighbouring models to match layout, naming and style. Use `sqb lineage` and `sqb scope` rather + than guessing how pieces connect. +2. **Look at the data.** `sqb query "SELECT ..." --limit 20` runs SQL on the active target. Profile + inputs (counts, nulls, key uniqueness) before writing joins. +3. **Compile.** `sqb compile --select ` after every edit. It is offline and fast. +4. **Plan.** `sqb plan --select ` shows exactly what a build will do and why. Read it before + building, especially for incremental models, full refreshes and anything destructive. +5. **Test.** `sqb test --select ` runs SQL unit tests without building. +6. **Build.** `sqb build --select ` runs tests, builds, and runs audits in DAG order. + Use `--defer-to ` to read unselected upstream models from another target instead of + rebuilding them. +7. **Verify.** Prove the result is right with `sqb diff` (below), not only that it built. -Source: `concepts/dbt-compatibility/overview.mdx` +When something fails, run `sqb debug` first. It separates configuration, authentication and +connection problems from command-specific errors. `sqb debug --no-connection` checks config only. -Run dbt and SQLBuild side by side with coordinated selection and SQLBuild models downstream. +## Reach for these tools -Use the dbt compatibility bridge to coordinate dbt selections with SQLBuild-owned models downstream of their outputs. +These are SQLBuild's highest-leverage features and the ones most often missed. Use them whenever +the trigger applies. -SQLBuild reads the dbt manifest and drives the `dbt` CLI as a subprocess. dbt remains responsible for compiling and executing dbt-owned models; SQLBuild statically analyzes and executes only SQLBuild-owned models downstream. `sqb dbt` runs in direct mode, so change-aware execution and virtual environments are not supported by the bridge. +| When you need to... | Use | +|---|---| +| Check a refactor did not change results, or compare dev with prod | `sqb diff prod:dev --full --select ` | +| Compare two arbitrary queries (old vs new logic, before vs after a fix, two tables) | `sqb diff --left-query "..." --right-query "..." --key ` | +| Know what a model can see: which macros, enums and constants are visible, used, or unavailable | `sqb scope model:` | +| Fix "unknown macro/enum/constant", or decide where a declaration should live | `sqb scope model: --explain macro:` | +| Preview whether moving a file breaks declaration visibility | `sqb scope model: --as-path ` | +| Find what a model depends on, or what breaks if it changes | `sqb lineage --direction both` | +| Trace where a column comes from, or who consumes it before renaming or dropping it | `sqb lineage . --direction downstream` | +| See which intermediate models a SQL test will really execute | `sqb test --select --inspect` | +| Inspect the full graph as data | `sqb dag --json`, `sqb plan --json`, `sqb compile --json` | +| Adopt an existing warehouse table's columns into a contract | `sqb contract generate --from --select ` | +| Try syntax safely in a throwaway DuckDB project | `sqb playground /tmp/sqb-sandbox` | -### Start with your existing dbt project +### Data diff and query diff -From inside your dbt project, run a `sqb dbt` command. Scope dbt work with familiar `--select` values, or omit selection to plan the whole project. +`sqb diff` is the main verification tool. Use it after any logic change. ```bash -sqb dbt plan --select path:models/marts -``` - -The first time you do this, SQLBuild bootstraps itself. If there is no `sqlbuild_project.toml`, it reads your `dbt_project.yml` and profile and creates a minimal twin project in a `sqlbuild_project/` directory next to your dbt project. It reuses your dbt profile for the warehouse connection, so there are no separate credentials to configure. - -``` -my-workspace/ - analytics/ # your existing dbt project, untouched - dbt_project.yml - models/ - target/ - manifest.json - sqlbuild_project/ # created by SQLBuild - sqlbuild_project.toml # points at the dbt project, no models of its own -``` - -The generated `sqlbuild_project.toml` looks like this: - -```toml -name = "analytics" -adapter = "snowflake" -default_target = "dev" - -[dbt] -project_dir = "../analytics" -profiles_dir = "/Users/you/.dbt" -target_path = "../analytics/target" -target = "dev" - -[connections.dbt_dev] -source = "dbt_profile" -profile = "analytics" -target = "dev" - -[targets.dev] -connection = "dbt_dev" -database = "ANALYTICS" -schema = "analytics" -``` - -`source = "dbt_profile"` on the named connection tells SQLBuild to connect using your dbt -profile, so it talks to the same warehouse dbt does. The target references that connection and -remains authoritative for its database and schema. - -`sqb dbt build --select path:models/marts` compiles the project, resolves the selection, runs the selected dbt models, then runs any SQLBuild models you have added against the dbt outputs. - -### How it works - -1. SQLBuild runs `dbt compile` to produce a `manifest.json` with model metadata -2. SQLBuild reads the manifest to understand dbt model names and their qualified warehouse tables -3. SQLBuild resolves your `--select`/`--exclude` against dbt by running `dbt ls`, so dbt-native selectors like `state:modified` and `package:` are evaluated by dbt itself, not reimplemented -4. `sqb dbt plan/run/build` orchestrates the run: dbt executes the selected dbt work -5. (Optional) any SQLBuild models you have added run last, against the dbt outputs - -Each step calls the `dbt` CLI directly: `dbt compile` for the manifest, `dbt ls` for selection, and `dbt build`/`dbt run` for execution. - -#### Flags - -`sqb dbt plan/run/build` declare the common flags directly. Anything declared goes **before** a `--` separator; any other raw dbt flag goes **after** it and is forwarded verbatim. A flag placed on the wrong side errors rather than silently reaching dbt. +# Model diff between two targets; needs FROM:TO, --select and exactly one mode. +# --full and --bounded need the model's unique_key. +sqb diff prod:dev --full --select fact_orders +sqb diff prod:dev --schema-only --select path:models/marts +sqb diff prod:dev --bounded 14d --select hourly_order_activity -```bash -sqb dbt build --select path:models/marts --full-refresh +# Query diff: compare any two queries on the active target (or --target). +sqb diff \ + --left-query "SELECT order_id, total_cents FROM analytics.fact_orders" \ + --right-query-file /tmp/new_fact_orders.sql \ + --key order_id --left-label current --right-label proposed ``` -Declared flags are routed to the right place automatically: - -| Flag | Routed to | Notes | -| ---- | --------- | ----- | -| `--select` / `--exclude` | dbt | dbt resolves selection (see [Selection](/concepts/dbt-compatibility/selection)). | -| `--vars` | dbt **and** SQLBuild | The same vars feed dbt's compile and SQLBuild's own variable resolution, so both sides see identical values. | -| `--full-refresh` | dbt **and** SQLBuild | Requests a full rebuild on both sides. dbt and native SQLBuild models independently apply their model-level `full_refresh` setting, including `false` opt-outs. | -| `--threads` | dbt (`--threads`) and SQLBuild (`--concurrency`) | | -| `--target` / `--project-dir` / `--profiles-dir` / `--profile` / `--target-path` | dbt | Standard dbt locators. | +Query diff requires `--key` (repeat it for composite keys) or `--unkeyed` for exact row-multiset +comparison. Queries are raw warehouse SQL: use physical relation names, not `__ref()`. Exit codes: +`0` identical, `1` differences, `2` incomplete or error. Read +[references/verifying-changes.md](references/verifying-changes.md) for tolerances, excluded +columns, type mismatches and JSON output. -##### `--vars` +## Must-know behaviour by area -`--vars` accepts the same JSON object dbt accepts, and SQLBuild passes it to **both** the underlying `dbt` invocation and its own variable resolution. That means a value referenced as `@@my_var` in SQLBuild model SQL (see [Interpolation](/concepts/interpolation)) and as `{{ var('my_var') }}` in a dbt model both resolve to the value you passed. CLI vars take precedence over project and local config vars. You do not need to declare vars twice. +**Incremental models.** Cursor-based (`cursor`, `cursor_type`, `cursor_grain`, `cursor_inputs`). +SQLBuild computes the replay window and automatically filters every input listed in +`cursor_inputs` to that window (in watermark microbatch models, inputs with the `filter` role), so +do not hand-write that filter. Unlisted inputs are read in full. Use `__cursor_start()` / +`__cursor_end()` only when SQL needs the bounds explicitly. Microbatch mode +(`incremental_mode microbatch` with `microbatch_strategy watermark` or `rolling_window`) splits the +window into batches, each with its own audits. Always `sqb plan` an incremental change before building. Read +[references/incremental-models.md](references/incremental-models.md). -```bash -sqb dbt build --select path:models/marts --vars '{"my_var": 1}' -``` +**Tests.** SQL unit tests live under `tests/unit/`. A `TEST();` file mocks inputs with +`__source__`, `__ref__`, `__seed__` CTEs, states expected output with +`__expected__`, adds zero-row checks with `__assert__`, and ends with `SELECT 1`. +One test can span many models: mock the sources, assert on the final model, and every intermediate +model runs from its real SQL. Read [references/testing.md](references/testing.md) for fixtures, +parameterised cases, scenarios, audits and Rules. -##### Forwarding other dbt flags +**Audits** validate built data and block bad data: `error` audits run on staging data before the +swap for tables, and on each delta before DML for incremental models. Attach built-ins such as +`not_null` and `unique` in the column schema. -For any native dbt flag SQLBuild does not declare, put it after `--` and it is passed straight to the `dbt` invocation untouched: +**Macros, enums and constants.** Macros are Python functions returning SQL strings; arguments are +Python literals, and SQL expressions are passed as quoted strings. Public functions become macros; +prefix helpers with `_`. Macros work in model SQL, SQL hooks, tests, scenarios, audits and SQL +functions, but not in ordinary `MODEL()` config fields. Placement decides visibility. Read +[references/macros-and-declarations.md](references/macros-and-declarations.md). -```bash -sqb dbt build -- --log-level debug -``` +**Snapshots** (SCD type 2) use `timestamp` or `check` strategies; full refreshes are policy-gated. -### Configuration +**Rules** are compile-time project checks (`sqb rules list`, `sqb rules show `). `sqb format` +rewrites sources deterministically; `sqb format --check` only reports. -The auto-generated project above is editable, and you can write `sqlbuild_project.toml` by hand. The `[dbt]` block (shown in the generated project above) accepts: +**dbt projects.** `sqb dbt plan|run|build --select ...` drives dbt for dbt-owned models and runs +SQLBuild models downstream of them. -| Field | Description | -|-------|-------------| -| `project_dir` | Path to the dbt project root (where `dbt_project.yml` lives) | -| `profiles_dir` | Path to the directory containing `profiles.yml` | -| `target_path` | Path to dbt's `target/` directory (where `manifest.json` is written) | -| `target` | dbt target name override (optional) | - -Paths can be absolute or relative to the SQLBuild project root. - -### Prerequisites - -- dbt must be installed and available on `PATH` as `dbt` -- Both projects must target the same warehouse and schema/database context - -SQLBuild uses your own `dbt` install; it does not bundle or install dbt. If your `dbt` is not reachable as a bare `dbt` on `PATH` (for example, you run it via `uv`, `poetry`, or a wrapper), set the `DBT_EXECUTABLE` environment variable to the executable SQLBuild should call. - -SQLBuild runs `dbt compile` automatically as part of `sqb dbt plan/run/build` to produce the manifest. You do not need to compile the dbt project manually. - -### Debugging - -`sqb dbt debug` runs both projects' diagnostics: `dbt debug` (verifying the dbt project config and warehouse connection) followed by `sqb debug` (verifying the SQLBuild project config and connection). - -```bash -sqb dbt debug -``` - -### On this topic - -- [Selection](/concepts/dbt-compatibility/selection) - how `--select` and `--exclude` route work across both graphs. -- [Adding SQLBuild models](/concepts/dbt-compatibility/adding-sqlbuild-models) - optionally write SQLBuild models, tests, audits, and scenarios downstream of dbt. - -## Selection - -Source: `concepts/dbt-compatibility/selection.mdx` - -How select and exclude flags route work across the dbt and SQLBuild graphs. - -The `sqb dbt` commands use `--select` and `--exclude` to scope what runs. Selectors work across both dbt and SQLBuild, with the system determining which side owns each selector and how to route work. - -### SQLBuild-recognized selectors - -These selectors match SQLBuild models directly: - -| Selector | Example | Behavior | -|----------|---------|----------| -| Model name | `fact_orders` | Selects that SQLBuild model. Auto-includes its immediate dbt upstream dependencies. | -| Leading `+` | `+fact_orders` | Selects the model plus walks upstream through both SQLBuild and dbt models. | -| Trailing `+` | `fact_orders+` | Selects the model plus all downstream SQLBuild models. | -| Both `+` | `+fact_orders+` | Full upstream (including dbt) and downstream expansion. | -| Tag | `tag:nightly` | Selects all SQLBuild models with that tag. Auto-includes dbt dependencies. | -| Tag with `+` | `+tag:nightly` | Tag match plus upstream expansion through the combined graph. | -| Path | `path:models/marts` | Selects SQLBuild models under that project-relative directory - the same syntax as dbt's `path:` selector. | -| Path with `+` | `+path:models/marts` | Path match plus upstream/downstream expansion. | - -When a SQLBuild model is selected, its immediate dbt upstream dependencies are always included so dbt can build the tables that SQLBuild models read from. - -### dbt-only selectors - -Selectors that SQLBuild does not recognize (like `state:modified`, `package:stripe`, `source:stripe.charges`) are passed to `dbt ls` to resolve: - -| Selector | Example | Behavior | -|----------|---------|----------| -| Without `+` | `state:modified` | dbt-only work. No SQLBuild models selected. | -| With trailing `+` | `state:modified+` | SQLBuild runs `dbt ls` to find which dbt models match, then walks downstream into SQLBuild territory. | -| With both `+` | `+state:modified+` | Same downstream expansion, plus upstream dbt expansion. | - -This means you can use dbt-native selectors like `state:modified+` to trigger rebuilds of SQLBuild models that depend on changed dbt models. If `dbt ls` returns no matching models, no SQLBuild work is triggered. - -### Exclude - -`--exclude` removes matching SQLBuild models from the final selection: - -```bash -sqb dbt build --select fact_orders+ --exclude tag:nightly -``` - -### Examples - -```bash -# Build a specific SQLBuild model and its dbt dependencies -sqb dbt build --select downstream_orders - -# Build everything downstream of a SQLBuild model -sqb dbt build --select downstream_orders+ - -# Build a SQLBuild model with full upstream dbt chain -sqb dbt build --select +downstream_orders - -# Build SQLBuild models downstream of modified dbt models -sqb dbt build --select state:modified+ - -# Build all SQLBuild models tagged "nightly" with their dbt dependencies -sqb dbt build --select tag:nightly - -# Build SQLBuild models under a path -sqb dbt build --select path:models/marts -``` - -### Execution order - -For `sqb dbt run` and `sqb dbt build`: - -1. **dbt runs** - a single `dbt run/build` command executes with the user's selectors merged with any additional dbt models required by selected SQLBuild models -2. **SQLBuild runs** - selected SQLBuild models execute against the now-built dbt tables - -## Adding SQLBuild models - -Source: `concepts/dbt-compatibility/adding-sqlbuild-models.mdx` - -Grow into SQLBuild's own models, tests, audits, and scenarios downstream of your dbt project. - -The dbt compatibility bridge does not require SQLBuild models. You can optionally add SQLBuild models, tests, audits, and scenarios downstream of dbt outputs. - -This is purely additive. Your dbt models stay in dbt, and the layout gains a `models/` directory in the SQLBuild project. - -``` -my-workspace/ - analytics/ # your existing dbt project, untouched - dbt_project.yml - models/ - target/ - manifest.json - sqlbuild_project/ # created by SQLBuild - sqlbuild_project.toml - models/ - marts/downstream_orders.sql # references dbt models via __dbt_ref - tests/ - unit/test_downstream_orders.sql -``` - -### Referencing dbt models - -SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")`: - -```sql -MODEL ( - tags [finance], - columns (order_id (audits [not_null])), -); - -SELECT order_id FROM __dbt_ref("analytics", "fact_orders") -``` - -This resolves to the qualified warehouse table name from the dbt manifest (e.g. `analytics.fact_orders`). The dbt model becomes an upstream dependency in the combined graph, so SQLBuild models downstream of it build against its output. - -SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: - -```sql -MODEL (tags [marts]); - -SELECT order_id FROM __ref("downstream_orders") -``` - -### Tests, audits, and scenarios - -SQLBuild models added downstream of dbt get SQLBuild's full validation surface: - -- **Unit tests** can mock dbt model dependencies with `__dbt_ref__` fixture CTEs, so you can test a SQLBuild model without a warehouse connection or a compiled dbt manifest. See [Testing](/concepts/testing). -- **Audits** declared inline in the `MODEL()` header block promotion when they fail, the same as in a standalone SQLBuild project. See [Audits](/concepts/audits). -- **Scenarios** run the combined graph against fixture inputs for end-to-end checks. See [Scenarios](/concepts/scenarios). - -For the full model authoring reference (materializations, incremental strategies, contracts, and more), see [Models](/concepts/models). - -## Project Configuration - -Source: `concepts/project-configuration.mdx` - -Configure your SQLBuild project with sqlbuild_project.toml and sqlbuild_local.toml. - -SQLBuild projects are configured with two files in the project root: - -- **`sqlbuild_project.toml`** - shared project configuration, committed to version control -- **`sqlbuild_local.toml`** - local developer overrides, gitignored - -Macro, constant, and enum visibility is not configured in either file. Declaration scopes use filesystem conventions; see [Declaration Scopes](/concepts/declaration-scopes). - -Most projects need only one committed `sqlbuild_project.toml`. Define shared targets such -as `dev` and `prod` there, including clone policies and team-wide defaults. Do not maintain -separate complete project files for each environment. - -Create `sqlbuild_local.toml` only when a developer or execution environment needs different -target selection, credentials, schemas, adapter settings, or variables. SQLBuild loads it -automatically and merges its explicitly configured values over the shared project config. - -```text -project/ - sqlbuild_project.toml # committed: shared targets and behavior - sqlbuild_local.toml # gitignored: this developer's overrides -``` - -Add the local file to `.gitignore`: - -```gitignore -sqlbuild_local.toml -``` - -### sqlbuild_project.toml - -A complete example: - -```toml -name = "waffle_shop" -adapter = "duckdb" -default_target = "dev" - -[connections.local] -database = "waffle_shop_control.duckdb" - -[settings] -default_audit_severity = "warn" - -[defaults] -materialized = "table" - -[constants] -collection_rendering = "value_list" - -[targets.prod] -connection = "local" -schema = "prod" - -[targets.dev] -connection = "local" -schema = "dev" - -[path_defaults.staging] -materialized = "view" -``` - -#### Required fields - -| Field | Description | -|-------|-------------| -| `name` | Project name. Used in fingerprint tracking and manifest generation. | -| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, or `sqlserver`. See [Adapters](/concepts/adapters). | -| `default_target` | Name of the target to build against when none is selected (see [Targets](#targets)). | - -#### Named connections - -Define reusable connections under `[connections.]`, then reference one by name from -each target. Connections own endpoint, authentication, and compute settings. Targets own the -authoritative `database`, `schema`, variables, and operational policy. - -```toml -[connections.local] -database = "my_project.duckdb" - -[targets.dev] -connection = "local" -schema = "dev" -``` - -Multiple targets can reuse one connection while keeping separate namespaces and policies: - -```toml -[connections.warehouse] -account = "my_org-my_account" -user = "${ENV:SNOWFLAKE_USER}" -password = "${ENV:SNOWFLAKE_PASSWORD}" -warehouse = "TRANSFORM_WH" - -[targets.prod] -connection = "warehouse" -database = "ANALYTICS" -schema = "PROD" - -[targets.dev] -connection = "warehouse" -database = "ANALYTICS" -schema = "DEV_ALICE" -``` - -A `database` or `schema` present in a named connection is connection/session metadata only; -it does not satisfy the mandatory namespace strategy for a named target. Put the target's -authoritative database and schema on `[targets.]`. SQLBuild validates connection -references while loading configuration, without opening a warehouse connection, and reports -an unknown `targets..connection` name as an offline configuration error. - -For migration only, SQLBuild still maps legacy `[connection]` to an implicit connection and -legacy `[targets..connection]` blocks to target-specific implicit connections. These -forms are compatibility syntax, not the canonical format for new or updated projects. - -### Targets - -A target is a named build context - the database and schema you build into, plus execution policy (for example `dev` and `prod`). Each target references a named connection and can configure: - -| Field | Description | -|-------|-------------| -| `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | -| `database` | Database for all models in this target | -| `connection` | Name from `[connections.]` used for endpoint, authentication, and compute | -| `vars` | Target-specific project variables | -| `defer_sources_to` | Target name to read managed source data from (see [Loaders](/concepts/python-nodes/loaders#source-deferral)) | -| `clone` | Clone policy (see below) | - -```toml -[connections.warehouse] -account = "my_org-my_account" -warehouse = "TRANSFORM_WH" - -[targets.prod] -connection = "warehouse" -database = "analytics" -schema = "analytics_prod" -loader_schema = "raw_prod" -defer_sources_to = "prod" - -[targets.dev] -connection = "warehouse" -database = "analytics" -schema = "analytics_dev" -loader_schema = "raw_dev" -defer_sources_to = "prod" - -[targets.staging] -connection = "warehouse" -database = "analytics" -schema = "staging" - -``` - -Managed loader writes use the active target's `loader_schema`, falling back to its model -`schema`. Managed source reads use the target named by `defer_sources_to`, or the active -target itself when deferral is omitted. In the example, `load --target dev` writes to -`raw_dev`, while models built in dev read from `raw_prod`. - -SQLBuild rejects targets on the same warehouse/database when their managed loader writes -resolve to the same schema. Two targets may read the same schema through deferral, but they -cannot both own loader writes there. - -#### Selecting a target - -The active target is determined by (in order of precedence): - -1. `--target` on the command line (highest priority) -2. `sqlbuild_local.toml` `target` field -3. `default_target` in `sqlbuild_project.toml` -4. No target (models build to the default schema) - -A typical developer keeps shared target definitions in `sqlbuild_project.toml` and selects -their normal target once in the optional local file: - -```toml -# sqlbuild_local.toml -target = "dev" - -[targets.dev] -connection = "warehouse" -schema = "dev_alice" -loader_schema = "raw_alice" - -[connections.warehouse] -user = "alice" -password = "${ENV:SNOWFLAKE_PASSWORD}" -``` - -Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` -still takes precedence for that invocation. - -#### Clone policies - -Targets can declare whether they allow cloning to or from: - -```toml -[targets.prod] -schema = "prod" - -[targets.prod.clone] -allow_as_clone_origin = true -allow_as_clone_destination = false - -[targets.dev] -schema = "dev" - -[targets.dev.clone] -allow_as_clone_origin = false -allow_as_clone_destination = true -``` - -Both policies default to `false`. `sqb clone --from prod --to dev` requires `allow_as_clone_origin = true` on `prod` and `allow_as_clone_destination = true` on `dev`. - -### Defaults - -Project-wide model defaults. Any field you can set in a `MODEL()` header can be set here as a default: - -```toml -[defaults] -materialized = "table" -incremental_strategy = "delete_insert" -replay_on_change = "full" -tags = ["managed"] -``` - -These apply to all models unless overridden by path defaults or the model's own `MODEL()` header. - -### Constants - -Collection constants default to parenthesized SQL value lists. Set a project-wide default when lists and sets should instead compile to first-class adapter-native arrays: - -```toml -[constants] -collection_rendering = "array" -``` - -`collection_rendering` accepts `value_list` (the SQLBuild default) or `array`. A public constant's `render_as` field or a model-local `constant(...)` wrapper overrides the project setting. The complete precedence order is declaration override, project setting, then `value_list`. - -This setting does not make unsupported adapter features portable. In particular, SQL Server rejects native arrays, and BigQuery rejects nested arrays. See [Collections and Rendering](/concepts/constants/collections-and-rendering#project-default) for syntax, adapter output, and value-list usage constraints. - -### Path defaults - -Per-directory model defaults. Useful for applying different config to different parts of your project: - -```toml -[path_defaults."models/staging"] -materialized = "view" -tags = ["staging"] - -[path_defaults.marts] -materialized = "table" -tags = ["marts"] -replay_on_change = "full" -``` - -Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the -`staging` path default. - -#### Config layering order - -Configuration is layered in this order, with later layers overriding earlier ones: - -1. **Project defaults** (`defaults`) -2. **Path defaults** (`path_defaults`) - if the model's path matches -3. **MODEL() header** - the model's own config - -Most keys are overridden by the more specific layer, but three merge instead: - -- `tags` are *unioned* across layers. A model with `tags [marts]` in its header that matches a path default with `tags [managed]` will have both tags. -- `row_diff_exclude_columns` lists are unioned across layers. -- `row_diff_tolerances` mappings are deep-merged across layers, so a header tolerance for one column adds to (rather than replaces) tolerances declared in defaults or path defaults. - -Diff sampling values use ordinary replacement precedence. `row_diff_sample_rows = 0` explicitly -disables an inherited sample for a path or model. CLI `--sample-rows`, `--sample-seed`, and -`--exhaustive` override the compiled configuration for one invocation. - -### Settings - -Global feature toggles: - -```toml -[settings] -sql_analysis = true -query_change_tracking = true -column_contract_mode = "implicit" -concurrency = 1 -microbatch_concurrency = false -auto_load_sources = true -table_promotion_mode = "staged" -default_audit_severity = "warn" -default_audit_run_scope = "final" -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | -| `changes_only` | `false` | In virtual mode, limit `plan` and `build` to [stale-driven execution](/concepts/virtual-environments/building#stale-driven-execution) without passing `--changes-only`. Rejected in direct mode. Can also be set per target under `[targets.]`. The CLI flag takes precedence, then the selected target, local settings, and this project setting. | -| `virtual_environments` | `false` | Enable [virtual environments](/concepts/virtual-environments) (versioned model outputs, promotion, rollback, state management). When `false`, the project runs in direct mode. | -| `query_change_tracking` | `true` | Track query fingerprints for change detection | -| `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/concepts/models/contracts). | -| `concurrency` | `1` | Maximum parallel model execution (currently serial only) | -| `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | -| `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/concepts/python-nodes/loaders). | -| `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | -| `default_audit_severity` | `warn` | Default severity for audits: `warn` or `error` | -| `default_audit_run_scope` | `final` | Default run scope for audits: `final` or `delta_and_final` | - -#### Table promotion mode - -- **`staged`** (default for most adapters): Materializes into a staging table, runs audits, then swaps into the target. If audits fail, the production table is untouched. -- **`immediate`**: Creates the table directly at the target location. Audits run after materialization. Simpler but no pre-promotion safety net. - -### Rules - -Rules configuration belongs in the shared `sqlbuild_project.toml` so local and CI compilation use -the same checks: - -```toml -[rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] -``` - -Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in -codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See -[Compiler-integrated Rules](/concepts/rules) for configuration, authoring, and suppressions. - -### Project variables - -Variables are simple string substitutions available in model SQL via the `@@name` syntax: - -```toml -[vars] -schema_prefix = "analytics" -retention_days = "90" -``` - -Target-specific variables override project-level ones: - -```toml -[vars] -schema_prefix = "analytics" - -[targets.prod.vars] -schema_prefix = "prod_analytics" -``` - -See [Macros](/concepts/macros) for details on variable substitution and how variables interact with macros. - -### Janitor - -Configuration for the `sqb janitor` command, which cleans up stale warehouse relations: - -```toml -[janitor] -enabled = false -retention_days = 30 -delete_tracked_only = true -exclude_patterns = ["audit_*", "tmp_*"] -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `enabled` | `false` | Whether janitor is active | -| `retention_days` | `30` | Only clean relations older than this many days | -| `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | -| `exclude_patterns` | `[]` | Glob patterns for relations to skip | - -### Scenario - -Configuration for scenario snapshot capture safety limits and local type overrides: - -```toml -[scenario.snapshot_limits] -max_rows_per_relation = 10000 -max_total_rows = 50000 -max_bytes_per_relation = 10485760 -max_total_bytes = 52428800 -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `max_rows_per_relation` | none | Maximum rows per captured relation | -| `max_total_rows` | none | Maximum total rows across all relations in one scenario | -| `max_bytes_per_relation` | none | Maximum JSONL bytes per relation file | -| `max_total_bytes` | none | Maximum total JSONL bytes per scenario | - -Local type overrides for DuckDB replay are configured per adapter dialect: - -```toml -[scenario.local_type_overrides.snowflake] -"OBJECT" = "JSON" -"ARRAY" = "JSON" -``` - -See [Scenarios](/concepts/scenarios) for details on local type overrides and capture limits. - -### dbt - -Configuration for running SQLBuild alongside an existing dbt project: - -```toml -[dbt] -project_dir = "../dbt_project" -profiles_dir = "../profiles" -target_path = "../dbt_project/target" -target = "dev" -``` - -| Field | Description | -|-------|-------------| -| `project_dir` | Path to the dbt project root (where `dbt_project.yml` lives) | -| `profiles_dir` | Path to the directory containing `profiles.yml` | -| `target_path` | Path to dbt's `target/` directory (where `manifest.json` is written) | -| `target` | dbt target name override (optional) | - -Paths can be absolute or relative to the SQLBuild project root. See [Using SQLBuild with dbt](/concepts/dbt-compatibility/overview) for setup and usage details. - -### Skills - -Configuration for AI agent skill file installation: - -```toml -[skills] -targets = ["agents", "claude"] -auto_update = false -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `targets` | `["agents", "claude"]` | Agent targets to install. OpenCode consumes `.agents`; use `opencode` only as an explicit override. | -| `auto_update` | `false` | Refresh stale SQLBuild-owned generated files from the installed package during normal commands; custom collisions are never overwritten. | - -See [skills CLI reference](/cli/skills) for usage details. - -### sqlbuild_local.toml - -Local developer overrides. This optional file is loaded automatically and should be -gitignored. Only put values that differ from the shared project configuration here. - -```toml -target = "dev" - -[targets.dev] -schema = "dev_alice" -loader_schema = "raw_alice" - -[connections.local] -database = "my_local.duckdb" - -[settings] -sql_analysis = false -concurrency = 4 - -[vars] -debug_mode = "true" -``` - -| Field | Description | -|-------|-------------| -| `target` | Override which target is active for this developer | -| `adapter` | Override the database adapter (e.g. use DuckDB locally while prod uses Snowflake) | -| `connections` | Override named connection fields; entries merge by connection name | -| `settings` | Override global settings (only explicitly set fields take effect) | -| `vars` | Developer-specific variable overrides (merged on top of project + target vars) | - -Project and local configuration merge named connections by name and merge their explicitly -configured fields. Target blocks merge the same way and may override the connection reference, -`database`, `schema`, `loader_schema`, variables, source deferral, and policy fields. Unspecified -values continue to come from `sqlbuild_project.toml`; a local reference to an unknown merged -connection still fails offline during configuration loading. - -This replaces the common dbt pattern of switching profiles or setting environment variables -to change targets. Each developer sets their target, named connection, and preferences once in -`sqlbuild_local.toml` and it persists across sessions. - -## Resource Identities - -Source: `concepts/resource-identities.mdx` - -Canonical names for SQLBuild resources, selectors, state, and integrations. - -SQLBuild resource identities use lowercase ASCII snake_case. A name must start with a lowercase -letter, end with a lowercase letter or digit, and contain only lowercase letters, digits, and -underscores. Consecutive underscores are valid, so established names such as -`commerce__mart_v_order` remain canonical. - -This contract applies to models, seeds, sources, SQL and Python functions, generic and singular -audits, attached audit definitions and instance names, SQL tests and parameterized cases, -scenarios, SQL and Python hooks, macros, model schemas, enums, constants, materializations, -providers, event exporters, loaders, tasks, assets, and checks. Private scoped declarations use -exactly one leading underscore, such as `_country_codes`; public declarations must not use a -leading underscore. - -Provider classes are the one conventional derivation: when `provider_name` is omitted, -`AnalyticsApiProvider` resolves to `analytics_api_provider`. An explicit `provider_name` is an -authored identity and must already be canonical snake_case; SQLBuild does not normalize it. - -Names derived from files use the filename stem. For example, `models/daily_orders.sql` defines -the model identity `daily_orders`, while `audits/generic/expression_is_true.sql` defines the -generic audit identity `expression_is_true`. Directories organize resources but do not change -their names. - -Physical warehouse identifiers are separate from SQLBuild resource identities. Database, schema, -table, model alias, column, tag, group, and directory names retain their existing adapter-specific -contracts and do not need to follow this profile. - -### Invalid names - -Compilation fails during discovery with `D016` when an authored identity is not canonical: - -```text -error[D016]: Invalid model identity 'DailyOrders' in models/DailyOrders.sql; -use snake_case 'daily_orders' -``` - -SQLBuild suggests a corrected spelling but never silently normalizes an identity. Silent -normalization would make selectors, manifests, persisted execution state, and integration keys -disagree about which resource ran. - -### Migrating existing projects - -1. Rename file-derived resources and explicit `name` values to snake_case. -2. Update `__ref`, `__seed`, `__source`, function, hook, macro, audit, test, and dependency - references. -3. Update selectors and external integrations, including Dagster asset/check keys, that use the - old identity. -4. Compile before building. - -A rename intentionally creates a new resource identity. Existing fingerprints, audit history, -and other persisted state under the old name are not silently reassigned to the new resource. - -## Overview - -Source: `concepts/adapters.mdx` - -Supported database engines and their connection configuration. - -SQLBuild uses adapters to connect to different database engines. - -| Adapter | Status | Install | -|---------|--------|---------| -| [DuckDB](/concepts/adapters/duckdb) | Supported | included by default | -| [MotherDuck](/concepts/adapters/motherduck) | Supported | included by default (uses DuckDB) | -| [Snowflake](/concepts/adapters/snowflake) | Supported | `sqlbuild[snowflake]` | -| [BigQuery](/concepts/adapters/bigquery) | Supported | `sqlbuild[bigquery]` | -| [Databricks](/concepts/adapters/databricks) | Supported | `sqlbuild[databricks]` | -| [PostgreSQL](/concepts/adapters/postgres) | Supported | `sqlbuild[postgres]` | -| [SQL Server](/concepts/adapters/sqlserver) | Supported | `sqlbuild[sqlserver]` | -| ClickHouse | Coming soon | | -| Redshift | Coming soon | | -| Trino | Coming soon | | -| Spark | Coming soon | | -| Athena | Coming soon | | - -Set the adapter in `sqlbuild_project.toml`: - -```toml -name = "my_project" -adapter = "duckdb" -``` - -Or override it per developer in `sqlbuild_local.toml`: - -```toml -adapter = "duckdb" -``` - -### Custom adapters - -You can write your own adapter for any database engine by placing Python files under `adapters/` in your project directory. - -#### Extending a built-in adapter - -The most common case is extending an existing adapter with custom behavior. Subclass the built-in adapter, set a new `adapter_name`, and override what you need: - -```python -# adapters/duckdb_plus.py -from sqlbuild.integrations.duckdb.client import DuckDbAdapter - -class DuckDbPlusAdapter(DuckDbAdapter): - adapter_name = "duckdb_plus" - - def connect(self, config): - connection = super().connect(config) - # custom setup - load extensions, set pragmas, etc. - self.execute(connection, "SET enable_progress_bar = true") - return connection -``` - -```toml -adapter = "duckdb_plus" -``` - -#### Writing an adapter from scratch - -For a database engine with no built-in support, subclass `BaseAdapter`. It provides ANSI SQL defaults for most methods - you only need to implement `connect`, `execute`, `close`, and any methods where your engine differs from standard SQL: - -```python -# adapters/my_database.py -from sqlbuild.adapter.base.base_adapter import BaseAdapter - -class MyDatabaseAdapter(BaseAdapter): - adapter_name = "my_database" - sql_analysis_dialect_name = "postgres" # SQL dialect for validation and lineage - - def connect(self, config): - ... - - def execute(self, connection, sql): - ... - - def close(self, connection): - ... -``` - -Set `sql_analysis_dialect_name` to the SQL dialect name that matches your engine's SQL syntax. This enables compile-time SQL validation, column inference, column lineage, and local scenario replay for your adapter. If omitted, SQLBuild uses generic SQL parsing. SQL analysis is powered by [Polyglot](https://github.com/tobilg/polyglot), a Rust reimplementation of SQLGlot supporting 32+ dialects. - -For full control with no inherited defaults, subclass `StrictAdapter` instead. Every method is abstract and must be implemented explicitly. SQLBuild raises a clear error listing any unimplemented methods. - -#### Discovery rules - -- SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) -- Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) -- Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) - -#### Adapter class hierarchy - -``` -StrictAdapter (fully abstract - all methods must be implemented) - └── BaseAdapter (ANSI SQL defaults - override only what differs) - ├── DuckDbAdapter - ├── SnowflakeAdapter - ├── BigQueryAdapter - ├── DatabricksAdapter - ├── PostgresAdapter - └── SqlServerAdapter -``` - -`StrictAdapter` composes four mixins that define the full adapter contract: - -- **ConnectionMixin** - `connect`, `close`, `begin`, `commit`, `rollback` -- **SchemaMixin** - `relation_exists`, `list_relations`, `describe_relation`, `ensure_schema` -- **MaterializationMixin** - `create_table_as`, `create_view`, `drop`, `rename`, `swap`, `load_seed` -- **DiffMixin** - `diff_schema`, `diff_rows`, `sample_unequal_rows`, `sample_side_only_rows` - -## DuckDB - -Source: `concepts/adapters/duckdb.mdx` - -DuckDB adapter configuration for SQLBuild. - -DuckDB is included as a core dependency. No extra installation needed. - -### Connection config - -```toml -adapter = "duckdb" -default_target = "dev" - -[connections.local] -database = "my_project.duckdb" - -[targets.dev] -connection = "local" -schema = "dev" -``` - -| Field | Description | -|-------|-------------| -| `database` | Path to the DuckDB database file. Use `:memory:` for in-memory databases. | -| `extensions` | List of DuckDB extensions to install and load on connect. | -| `settings` | Key-value pairs passed as `SET` statements on connect. | -| `attach` | List of additional databases to attach. | - -### Extensions and settings - -```toml -[connections.local] -database = "my_project.duckdb" -extensions = ["httpfs", "parquet"] - -[connections.local.settings] -memory_limit = "4GB" -``` - -### Attaching additional databases - -```toml -[connections.local] -database = "my_project.duckdb" - -[[connections.local.attach]] -path = "external_data.duckdb" -alias = "external" -read_only = true -``` - -### Table promotion mode - -DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: - -```toml -[settings] -table_promotion_mode = "staged" -``` - -## MotherDuck - -Source: `concepts/adapters/motherduck.mdx` - -MotherDuck adapter configuration for SQLBuild. - -MotherDuck uses DuckDB's built-in `md:` connection support. No extra installation needed beyond the core DuckDB dependency. - -### Connection config - -```toml -adapter = "motherduck" -default_target = "dev" - -[connections.motherduck] -database = "my_database" -token = "your_motherduck_token" - -[targets.dev] -connection = "motherduck" -database = "my_database" -schema = "dev" -``` - -| Field | Description | -|-------|-------------| -| `database` | MotherDuck database name. Automatically prefixed with `md:` if not already present. Defaults to `md:` (your default MotherDuck database). | -| `token` | MotherDuck access token. Can also be set via environment variable. | - -### Authentication - -MotherDuck requires an access token. Generate one from the MotherDuck UI and pass it via the connection config or an environment variable: - -```toml -[connections.motherduck] -database = "my_database" -token = "${ENV:MOTHERDUCK_TOKEN}" -``` - -### Shared connections across targets - -Use targets to separate production and development databases on MotherDuck: - -```toml -adapter = "motherduck" - -[connections.motherduck] -token = "${ENV:MOTHERDUCK_TOKEN}" -database = "my_database" - -[targets.prod] -connection = "motherduck" -database = "prod_db" -schema = "prod" - -[targets.dev] -connection = "motherduck" -database = "dev_db" -schema = "dev" -``` - -### Local development with DuckDB - -Use `sqlbuild_local.toml` to override the adapter for local development against a plain DuckDB file: - -```toml -adapter = "duckdb" - -[connections.local] -database = "local_dev.duckdb" - -[targets.dev] -connection = "local" -schema = "dev" -``` - -This lets you develop and test locally with zero MotherDuck compute cost, then deploy to MotherDuck in production. SQLBuild's [scenario replay](/concepts/scenarios) also runs locally in DuckDB regardless of the production adapter. - -## Snowflake - -Source: `concepts/adapters/snowflake.mdx` - -Snowflake adapter configuration for SQLBuild. - -Snowflake requires the optional `snowflake-connector-python` dependency: - -```bash -pip install 'sqlbuild[snowflake]' -# or -uv pip install 'sqlbuild[snowflake]' -``` - -### Connection config - -```toml -adapter = "snowflake" -default_target = "dev" - -[connections.warehouse] -account = "my_org-my_account" -user = "my_user" -password = "my_password" -role = "TRANSFORM_ROLE" -warehouse = "TRANSFORM_WH" - -[targets.dev] -connection = "warehouse" -database = "ANALYTICS" -schema = "RAW" -``` - -Connection fields are passed directly to `snowflake.connector.connect()`. See the [Snowflake Connector documentation](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect) for all available options, including key-pair authentication, OAuth, and SSO. Put the authoritative database and schema on the target. - -### Session initialization - -On connect, SQLBuild applies the connection's role and warehouse, then applies the active -target's authoritative database and schema. These ensure the session context is set correctly -regardless of the user's default settings. - -### Shared connections across targets - -Use multiple targets to reuse one Snowflake connection while selecting different authoritative -databases and schemas: - -```toml -adapter = "snowflake" - -[connections.warehouse] -account = "my_org-my_account" -user = "my_user" -password = "my_password" -role = "TRANSFORM_ROLE" -warehouse = "TRANSFORM_WH" - -[targets.prod] -connection = "warehouse" -database = "PROD_DB" -schema = "prod" - -[targets.dev] -connection = "warehouse" -database = "DEV_DB" -schema = "dev" -``` - -## BigQuery - -Source: `concepts/adapters/bigquery.mdx` - -BigQuery adapter configuration for SQLBuild. - -BigQuery requires the optional `google-cloud-bigquery` dependency: - -```bash -pip install 'sqlbuild[bigquery]' -# or -uv pip install 'sqlbuild[bigquery]' -``` - -### Connection config - -```toml -adapter = "bigquery" -default_target = "dev" - -[connections.gcp] -project = "my-gcp-project" -location = "europe-west2" - -[targets.dev] -connection = "gcp" -database = "my-gcp-project" -schema = "analytics_dev" -``` - -| Field | Description | -|-------|-------------| -| `project` | GCP project ID (required) | -| `location` | BigQuery dataset location - a region (`europe-west2`, `us-east1`) or multi-region (`US`, `EU`) | -| `credentials_path` | Path to a service account JSON key file. If omitted, uses Application Default Credentials (ADC). | - -### Authentication - -BigQuery supports two authentication methods: - -**Application Default Credentials (default):** Uses the credentials from `gcloud auth application-default login` or the environment's service account. No config needed beyond `project`. - -**Service account file:** - -```toml -[connections.gcp] -project = "my-gcp-project" -credentials_path = "/path/to/service-account.json" -``` - -### Typed constants - -BigQuery renders typed constant arrays with bracket syntax, objects as native JSON expressions, and exact decimals as `NUMERIC` expressions. For example: - -```sql -['GB', 'FR', 'HK'] -JSON '{"GB":"Great Britain"}' -NUMERIC '2.4700' -``` - -BigQuery does not support arrays of arrays. A nested list or set requested with `render_as array` fails at compile time rather than producing invalid BigQuery SQL. Parenthesized `value_list` rendering remains available for ordinary scalar collections used with `IN`. - -See [Collections and Rendering](/concepts/constants/collections-and-rendering#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. - -## Databricks - -Source: `concepts/adapters/databricks.mdx` - -Databricks adapter configuration for SQLBuild. - -Databricks requires the optional `databricks-sql-connector` dependency: - -```bash -pip install 'sqlbuild[databricks]' -# or -uv pip install 'sqlbuild[databricks]' -``` - -### Connection config - -```toml -adapter = "databricks" -default_target = "dev" - -[connections.workspace] -server_hostname = "my-workspace.cloud.databricks.com" -http_path = "/sql/1.0/warehouses/abc123" -token = "dapi_my_access_token" - -[targets.dev] -connection = "workspace" -database = "my_catalog" -schema = "my_schema" -``` - -| Field | Description | -|-------|-------------| -| `server_hostname` | Databricks workspace hostname (required) | -| `http_path` | SQL warehouse or cluster HTTP path (required) | -| `token` | Personal access token (required) | - -The target's `database` selects the Unity Catalog catalog, and its `schema` selects the schema. - -### Session initialization - -On connect, SQLBuild runs `USE CATALOG` and `USE SCHEMA` statements from the active target to set -the authoritative session namespace. - -## PostgreSQL - -Source: `concepts/adapters/postgres.mdx` - -PostgreSQL adapter configuration for SQLBuild. - -PostgreSQL requires the optional `psycopg` dependency: - -```bash -pip install 'sqlbuild[postgres]' -# or -uv pip install 'sqlbuild[postgres]' -``` - -### Connection config - -```toml -adapter = "postgres" -default_target = "dev" - -[connections.postgres] -host = "localhost" -port = 5432 -user = "my_user" -password = "my_password" -dbname = "my_database" - -[targets.dev] -connection = "postgres" -database = "my_database" -schema = "analytics_dev" -``` - -| Field | Description | -|-------|-------------| -| `host` | PostgreSQL server hostname (default: `localhost`) | -| `port` | PostgreSQL server port (default: `5432`) | -| `user` | Database user | -| `password` | Database password | -| `dbname` | Database name | - -Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. - -### Shared connections across targets - -```toml -adapter = "postgres" - -[connections.postgres] -host = "localhost" -user = "my_user" -password = "${ENV:PG_PASSWORD}" -dbname = "analytics" - -[targets.prod] -connection = "postgres" -database = "analytics" -schema = "prod" - -[targets.dev] -connection = "postgres" -database = "analytics" -schema = "dev" -``` - -## SQL Server - -Source: `concepts/adapters/sqlserver.mdx` - -Microsoft SQL Server adapter configuration for SQLBuild. - -SQL Server requires the optional `pymssql` dependency: - -```bash -pip install 'sqlbuild[sqlserver]' -# or -uv pip install 'sqlbuild[sqlserver]' -``` - -### Connection config - -```toml -adapter = "sqlserver" -default_target = "dev" - -[connections.sqlserver] -host = "localhost" -port = 1433 -user = "sa" -password = "my_password" -database = "my_database" - -[targets.dev] -connection = "sqlserver" -database = "my_database" -schema = "analytics_dev" -``` - -| Field | Description | -|-------|-------------| -| `host` | SQL Server hostname (default: `localhost`). Also accepts `server` as an alias. | -| `port` | SQL Server port (default: `1433`) | -| `user` | Database user (default: `sa`). Also accepts `username` as an alias. | -| `password` | Database password | -| `database` | Database name (default: `master`). Also accepts `dbname` as an alias. | - -Connection fields are passed to `pymssql.connect()`. See the [pymssql documentation](https://pymssql.readthedocs.io/en/stable/ref/pymssql.html) for all available options. The target remains authoritative for database and schema qualification. - -SQL Server supports schema-only, full-row, and bounded `sqb diff` comparisons. - -### Typed constants - -SQL Server supports scalar and parenthesized `value_list` constants. Strings use Unicode literals where required, and objects render through a JSON expression such as: - -```sql -JSON_QUERY(N'{"GB":"Great Britain"}') -``` - -SQL Server has no supported first-class native array expression. Any list or set that resolves to `render_as array`, whether through its declaration or `[constants].collection_rendering`, fails at compile time. SQLBuild does not silently substitute a value list or quoted JSON string. - -See [Collections and Rendering](/concepts/constants/collections-and-rendering#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. - -### Shared connections across targets - -```toml -adapter = "sqlserver" - -[connections.sqlserver] -host = "localhost" -user = "sa" -password = "${ENV:MSSQL_PASSWORD}" -database = "analytics" - -[targets.prod] -connection = "sqlserver" -database = "analytics" -schema = "prod" - -[targets.dev] -connection = "sqlserver" -database = "analytics" -schema = "dev" -``` - -## Sources - -Source: `concepts/sources.mdx` - -Declare external data inputs for your pipeline. - -Sources declare external data that your models depend on. They are defined in YAML files under `sources/` in your project directory. - -### Table sources - -Point at an existing table or view in your warehouse: - -```yaml -sources: - - name: raw_events - database: analytics - schema: raw - table: events -``` - -Reference it in models with `__source("raw_events")`. - -### Expression sources - -Define source data inline as a SQL expression. No external tables or setup scripts needed: - -```yaml -sources: - - name: raw__customers - expression: | - SELECT * FROM (VALUES - (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), - (2, 'Ron', 'Swanson', 'ron@pawnee.gov', TIMESTAMP '2026-02-01 08:00:00') - ) AS raw__customers(id, first_name, last_name, email, created_at) -``` - -Expression sources are resolved at compile time. They're the escape hatch for anything the framework doesn't natively model: external tables, warehouse-specific syntax, function calls, or any other relation type that doesn't fit a standard table reference. - -An inline source expression uses macros, constants, and enums available from its source definition -under `sources/`. Declarations limited to one source folder are not available in sibling folders. -See [How Visibility Works](/concepts/declaration-scopes/visibility). - -### Source audits - -Sources support the same audit system as models. Audits attached to sources run before any dependent model is built: - -```yaml -sources: - - name: raw_orders - columns: - - name: id - audits: - - not_null - - unique - audits: - - expression_is_true: - name: no future orders - expression: "ordered_at <= CURRENT_TIMESTAMP" -``` - -If a source audit with `error` severity fails, all downstream models that depend on that source are blocked. - -### Type enforcement - -Type enforcement is implicit for sources. If any column declares a `type`, SQLBuild activates source type enforcement, casts that column during source resolution, and uses declared types for schema-change detection: - -```yaml -sources: - - name: raw__customers - expression: | - SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name - columns: - - name: id - type: INTEGER - - name: first_name - - name: last_name -``` - -In this example, only `id` has a type declared, so only `id` is cast. Columns without a `type` are passed through unchanged. - -For expression sources, SQLBuild probes the expression's output columns and builds a projection that casts typed columns while preserving the rest. For table sources, it uses warehouse metadata to validate that declared column names exist and applies casts accordingly. - -You can explicitly set `type_enforcement: false` on a source to disable casting even when column types are declared. - -### Managed sources (loaders) - -Sources can be loaded by Python functions instead of pointing at existing tables or inline expressions. Set `managed: true` to bind a source to the `@loader` function **of the same name**, and SQLBuild will call it to populate the source table: - -```yaml -sources: - - name: raw_customers - managed: true - write_strategy: table - columns: - - name: id - type: INTEGER - - name: name - type: VARCHAR -``` - -Managed sources support incremental write strategies (`table`, `append`, `delete_insert`, `merge`), cursor-based loading, and concurrent execution. - -See [Loaders](/concepts/python-nodes/loaders) for the full guide on writing loader functions, write strategies, the loader context API, and auto-load behavior during builds. - -#### Declarative integrations (no Python) - -For common ingestion you can declare the source entirely in YAML, with no `@loader` function. SQLBuild generates the loader for you and runs it during the build: - -- [dlt](/integrations/dlt) - declare `dlt_sources` for `rest_api`, `sql_database`, and `filesystem` sources. -- [ingestr](/integrations/ingestr) - add an `ingestr` block to a source to pull from 50+ sources. - -### Source freshness - -Source freshness lets SQLBuild observe whether a source's data changed between runs. This feeds into -[planning and change detection](/concepts/planning): changed observations propagate to downstream -models and can alter their planned action. Virtual environments can also use the signal for -[stale-driven execution](/concepts/virtual-environments/building#stale-driven-execution). - -Configure freshness per source with a `freshness:` block: - -```yaml -sources: - - name: raw_events - schema: raw - table: events - freshness: - strategy: column - column: updated_at - type: timestamp - lag_tolerance: 15m -``` - -#### Strategies - -| Strategy | Description | Required fields | -|----------|-------------|-----------------| -| `adapter` | Uses warehouse metadata (e.g. Snowflake `LAST_ALTERED`). No query against the source table. | None | -| `column` | Reads `MAX(column)` from the source table. | `column`, `type` | -| `sql` | Runs a custom query that returns a single scalar value. | `query`, `type` | - -##### adapter - -```yaml -freshness: - strategy: adapter -``` - -Uses adapter-level table metadata. Supported on Snowflake, BigQuery, Databricks, PostgreSQL, DuckDB, and SQL Server. Does not support `type`, `column`, or `query`. - -##### column - -```yaml -freshness: - strategy: column - column: updated_at - type: timestamp -``` - -Queries `MAX(column)` from the source table. The column must be a plain column name (no expressions; use `sql` strategy for those). Requires `type`. - -##### sql - -```yaml -freshness: - strategy: sql - query: "SELECT MAX(version_id) FROM raw.events" - type: integer -``` - -Runs an arbitrary SQL query that returns a single scalar. Requires `type`. Does not support `column`. - -#### Type - -The `type` field declares the value kind for comparison: - -| Type | Description | -|------|-------------| -| `timestamp` | Datetime value. Supports `lag_tolerance`. | -| `integer` | Integer value. Change detected by exact comparison. | -| `string` | String value. Change detected by exact comparison. | - -#### Lag tolerance - -`lag_tolerance` is optional and only valid with `type: timestamp`. It declares how much the observed timestamp can drift from the previous observation before being treated as a change: - -```yaml -freshness: - strategy: column - column: updated_at - type: timestamp - lag_tolerance: 2h -``` - -Accepts positive durations: `15m` (minutes), `2h` (hours), `1d` (days). If the current observation is within the tolerance of the previous one, the source is treated as unchanged. - -#### Auto-observation - -Sources without an explicit `freshness:` block are auto-observed using the `adapter` strategy if: - -- The source has a physical table (not an expression source) -- The source is not managed -- The adapter supports table freshness metadata - -This means most unmanaged table sources get freshness tracking automatically on adapters that support it, with no configuration needed. - -Use [`sqb freshness`](/cli/freshness) to observe source freshness on demand without triggering a build. See [Source freshness](/concepts/planning/source-freshness) for how freshness feeds into change-aware builds. - -#### Freshness config reference - -| Field | Description | -|-------|-------------| -| `strategy` | Observation strategy: `adapter`, `column`, or `sql` | -| `type` | Value kind: `timestamp`, `integer`, or `string` | -| `column` | Column name for `column` strategy | -| `query` | SQL query for `sql` strategy | -| `lag_tolerance` | Duration tolerance for timestamp comparisons (e.g. `15m`, `2h`, `1d`). Only valid with `type: timestamp`. | - -### Config reference - -| Field | Description | -|-------|-------------| -| `name` | Source name, used in `__source("name")` references | -| `database` | Target database (optional) | -| `schema` | Physical source schema (optional). For managed sources, overrides target `loader_schema`; otherwise managed loaders fall back to `loader_schema`, then target `schema`. | -| `table` | Target table name (defaults to `name` if omitted) | -| `expression` | Inline SQL expression (alternative to table reference) | -| `managed` | Set to `true` to bind the source to the `@loader` function of the same name (see [Loaders](/concepts/python-nodes/loaders)) | -| `write_strategy` | How the loader writes data: `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | -| `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | -| `unique_key` | Merge key column(s) (required for `merge`) | -| `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | -| `description` | Human-readable description | -| `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | -| `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | -| `columns` | Column declarations with optional types and audits | -| `audits` | Source-level audits | - -## Seeds - -Source: `concepts/seeds.mdx` - -Load static CSV data into your pipeline as tables. - -Seeds are CSV files under `seeds/` that SQLBuild loads as tables in the warehouse. They're useful for small, static reference data like lookup tables, category mappings, or configuration values. - -### Defining a seed - -Place a CSV file in your `seeds/` directory: - -```csv -waffle_type_id,waffle_name,category,price_cents -1,Classic Belgian,sweet,850 -2,Liege,sweet,950 -3,Brussels,sweet,750 -4,Cheddar Herb,savory,1050 -``` - -Declare the seed's column types in any `.yml` file under `seeds/`. You can use any filename and organize declarations however you like - one file per seed, or group related seeds together: - -```yaml -# seeds/lookups.yml -seeds: - - name: waffle_types - description: Waffle menu items with pricing. - columns: - - name: waffle_type_id - type: INTEGER - - name: waffle_name - type: VARCHAR - - name: category - type: VARCHAR - - name: price_cents - type: INTEGER -``` - -Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). - -### Referencing seeds - -Seeds are referenced in models with `__seed()`: - -```sql -SELECT - o.order_id, - w.waffle_name, - w.price_cents * o.quantity AS line_total_cents -FROM __ref("stg_orders") o -LEFT JOIN __seed("waffle_types") w ON o.waffle_type_id = w.waffle_type_id -``` - -`__seed()` is distinct from `__ref()`. Using `__ref()` with a seed name raises a compile error with a message pointing you to `__seed()`. - -### Loading seeds - -Seeds are loaded automatically during `sqb build`. You can also load them standalone: - -```bash -sqb seed -``` - -Seeds are fully replaced on every run. If the CSV changes, the table is recreated with the new data. - -### Target overrides - -Seeds inherit the project's default database and schema. You can override these per seed: - -```yaml -seeds: - - name: waffle_types - database: analytics - schema: lookups - columns: - - name: waffle_type_id - type: INTEGER - - name: waffle_name - type: VARCHAR -``` - -### CSV settings - -For non-standard CSV formats, configure parsing behavior with `csv_settings`: - -```yaml -seeds: - - name: european_prices - csv_settings: - delimiter: ";" - encoding: utf-8 - quotechar: '"' - columns: - - name: product_id - type: INTEGER - - name: price - type: VARCHAR -``` - -#### Available CSV settings - -| Setting | Type | Description | -|---------|------|-------------| -| `delimiter` | string | Field delimiter (default: `,`) | -| `quotechar` | string | Character used to quote fields (default: `"`) | -| `doublequote` | boolean | Whether the quotechar is doubled to escape itself | -| `escapechar` | string | Character used to escape the delimiter or quotechar | -| `skipinitialspace` | boolean | Skip whitespace after the delimiter | -| `lineterminator` | string | Line terminator character | -| `encoding` | string | File encoding (default: `utf-8`) | -| `na_values` | list or mapping | Values to treat as null - a global list or per-column mapping | -| `keep_default_na` | boolean | Whether to use the default set of NA values in addition to `na_values` | - -#### Per-column NA values - -`na_values` can be a flat list (applied to all columns) or a mapping from column names to their NA values: - -```yaml -csv_settings: - na_values: - price: ["N/A", ""] - category: ["unknown"] -``` - -## Overview - -Source: `concepts/models.mdx` - -SQL model anatomy, references, dependencies, and the model documentation guide. - -A model is a SQL file that defines one transformation step and produces a table or view in the warehouse. - -### Model anatomy - -Every model starts with a `MODEL()` header followed by its query: - -```sql -MODEL ( - materialized table, - tags [marts], - description "Order fact table", - columns ( - order_id (type INTEGER, audits [not_null]), - ), -); - -SELECT - o.order_id, - o.customer_id, - p.amount_cents -FROM __ref("stg_orders") o -JOIN __ref("stg_payments") p USING (order_id) -``` - -The header controls how SQLBuild builds, validates, and documents the model. The query remains ordinary SQL apart from SQLBuild reference and macro calls. - -### References - -| Reference | Syntax | Resolves to | -|-----------|--------|-------------| -| Model | `__ref("name")` | Another model | -| Seed | `__seed("name")` | A seed CSV table | -| Source | `__source("name")` | An external source | -| Scalar UDF | `__udf("name")` | A user-defined function | - -SQLBuild discovers the dependency graph from these calls and orders selected work topologically. Among selected models, upstream models run before downstream dependents. An unselected upstream is read from its existing warehouse relation; use an upstream-expanding selector such as `+fact_orders` when it should also be built. Seeds use `__seed()`, not `__ref()`. - -See [Functions](/concepts/functions) for scalar UDF and table-function references. - -### Model guide - -- [Materializations](/concepts/models/materializations): views, tables, incrementals, snapshots, and custom materializations. -- [Schemas](/concepts/models/schemas): inline columns, reusable schemas, inheritance, audits, and model-local extensions. -- [Type Enforcement](/concepts/models/type-enforcement): static type checks and runtime cast behavior by materialization. -- [Contracts](/concepts/models/contracts): exact and open output validation, runtime guarantees, nullability, and enum-backed columns. -- [Hooks](/concepts/models/hooks): SQL and Python lifecycle hooks. -- [Configuration](/concepts/models/configuration): `MODEL()` field reference and SQL-validation controls. - -For deeper execution behavior, see [Incremental](/concepts/incremental), [Snapshots](/concepts/snapshots), and [Audits](/concepts/audits). - -## Materializations - -Source: `concepts/models/materializations.mdx` - -Choose how SQLBuild persists model output. - -The `materialized` field selects how a model becomes a warehouse relation. - -### View - -Creates or replaces a database view on each build: - -```sql -MODEL (materialized view); - -SELECT id AS order_id, customer_id, status -FROM __source("raw__orders") -``` - -View audits run after the view has been replaced. A failing audit marks the build as failed but cannot preserve the previous view. Views use compile-time contract and type analysis; SQLBuild does not rewrite the view with runtime casts or run the staged runtime-contract step. - -### Table - -By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. - -```sql -MODEL (materialized table); - -SELECT customer_id, COUNT(*) AS total_orders -FROM __ref("stg_orders") -GROUP BY customer_id -``` - -Projects may opt into `settings.table_promotion_mode = "immediate"`. Immediate promotion replaces the destination before audits, so a failed audit does not restore the old table. Immediate promotion rejects models that require declared-type enforcement or `contract enforced`; use staged promotion for those guarantees. - -### Incremental - -On a normal incremental run, applies append, delete/insert, or merge DML from a staged delta. Delete/insert removes matching keys or cursor ranges before inserting replacement rows. Cursor configuration controls replay bounds and may split work into microbatches. - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - unique_key [activity_hour], -); -``` - -For non-microbatch models, the first run, `--full-refresh`, and a full replay-on-change rebuild use the full-table path rather than incremental DML. Microbatch models retain batched execution: a full rebuild drops the existing target, creates its replacement from the first batch, and applies later batches with incremental DML. A failed full-refresh microbatch does not preserve the previous target. - -See [Incremental](/concepts/incremental) for cursor semantics, replay, schema changes, and microbatch execution. - -### Snapshot - -Maintains historical row versions with SCD Type 2 semantics: - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at, -); - -SELECT customer_id, name, plan, status, updated_at -FROM __source("customers") -``` - -See [Snapshots](/concepts/snapshots) for timestamp and check strategies, historical inputs, and full-refresh policies. - -### Custom - -A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. - -```sql -MODEL ( - materialized partition_tracked, - placeholders ( - partition_start "'2026-04-01'", - partition_end "'2026-04-05'", - ), - config ( - tracking_table partition_state, - partition_column order_date, - ), -); - -SELECT * -FROM __ref("stg_orders") -WHERE ordered_at >= @@@partition_start - AND ordered_at < @@@partition_end -``` - -The `config` block is passed to the Python function through `ctx.config`. Runtime-owned `@@@placeholder` values remain unresolved until materialization execution. - -The custom function owns staging, runtime type or contract enforcement, audit timing, promotion, and rollback. Framework final audits run after the function returns unless `MaterializationResult.audit_results` is populated. A custom materialization can call `ctx.run_audits` against a staging relation before applying changes and return those results through `MaterializationResult.audit_results`. - -## Schemas - -Source: `concepts/models/schemas.mdx` - -Declare model columns inline or reuse canonical inherited schemas. - -Schema metadata defines column names, types, nullability, descriptions, and column audits. It can live inline in one `MODEL()` header or in a reusable `SCHEMA()` declaration. - -### Inline columns - -Use inline columns for metadata owned by one model: - -```sql -MODEL ( - materialized view, - description "Cleaned order records", - columns ( - order_id (type INTEGER, nullable false, audits [not_null, unique]), - customer_id (type INTEGER, nullable false, audits [not_null]), - status ( - type VARCHAR, - audits [accepted_values (values ["placed", "completed", "cancelled"])], - ), - ), -); -``` - -See [Audits](/concepts/audits) for built-in audits, custom audits, arguments, severity, and incremental run scope. - -### Reusable schemas - -When multiple models implement the same relation shape, declare it once under `schemas/`. SQLBuild discovers schema files recursively and makes public names available throughout the project. - -```sql --- schemas/orders/order.sql -SCHEMA ( - name order, - description "Canonical staged order shape", - columns ( - order_id (type INTEGER, nullable false, audits [not_null]), - customer_id (type INTEGER, nullable false, audits [not_null]), - status (type VARCHAR), - ), -); -``` - -Bind a model with `model_schema`: - -```sql -MODEL ( - materialized view, - schema staging, - model_schema order, - contract enforced, -); -``` - -`schema staging` selects the warehouse destination schema. `model_schema order` selects reusable column metadata. - -The reusable description becomes the model description when the model does not declare one. A model-owned description takes precedence. - -### Model-local columns - -A bound model may add output columns that are not part of the reusable shape: - -```sql -MODEL ( - model_schema order, - columns ( - ingestion_batch_id (type VARCHAR, nullable false), - ), - contract enforced, -); -``` - -Resolved schema columns retain their order and new model-local columns follow them. Use a named child schema when an extension is reusable; use inline columns for an extension owned by one model. - -### Model-specific column audits - -Audits in a reusable schema apply to every bound model. A model can add stricter audits to an inherited column by naming that column and declaring only `audits`: - -```sql -MODEL ( - model_schema order, - columns ( - order_id (audits [unique]), - ), -); -``` - -The effective `order_id` keeps the reusable type, nullability, description, and `not_null` audit, then adds `unique`. A model cannot remove reusable audits or override inherited metadata. An inherited-column entry containing `type`, `nullable`, or `description` fails compilation. Identical audit instances are deduplicated. - -### Inheritance - -A reusable schema may extend one parent with additional columns: - -```sql -SCHEMA ( - name sourced_order, - extends order, - columns ( - source (type VARCHAR, nullable false), - ), -); -``` - -Inheritance may be transitive. Parent columns resolve before child columns. An inherited column cannot be redeclared or overridden in a child schema. SQLBuild rejects unknown parents, cycles, case-insensitive duplicates, and multiple parents. - -Physical SQL output order is not currently enforced. Static contract analysis matches names and checks declared types and proven non-nullability. Runtime exact-contract validation matches names and types but does not validate nullability. - -### Contracts and planning - -`contract enforced` treats the complete named-plus-local declaration as the exact output shape. An unspecified model follows the project's `column_contract_mode`; `contract none` explicitly keeps the resolved columns as metadata, audit attachment, and type-enforcement inputs without activating shape validation. See [Contracts](/concepts/models/contracts). - -For models bound to a reusable schema, effective column names, types, nullability, and enum members participate in model version identity. Changing those fields on a parent therefore affects models bound through descendants. Descriptions do not change model identity. Audits have their own audit-gate identities, so audit changes invalidate reusable audit results without changing the model version itself. - -### Limitations - -Reusable schemas intentionally support a narrow ownership model: - -- One optional parent, with transitive inheritance. -- Additive child and model-local output columns. -- Audit-only model augmentation of inherited columns. -- No general column overrides, multiple inheritance, composition, mixins, parameters, or generated projections. -- No physical output ordinal enforcement. - -## Type Enforcement - -Source: `concepts/models/type-enforcement.mdx` - -Understand declared model types, static checks, and runtime casting. - -Type enforcement makes authored column types operational. It is a model feature as well as a related source feature, and it is separate from whether a model uses an exact contract. - -### Enabling it - -For a SQL model, declaring a type on any inline or reusable-schema column enables type enforcement automatically: - -```sql -MODEL ( - materialized table, - columns ( - order_id (type INTEGER), - ordered_at (type TIMESTAMP), - source_system (), - ), -); -``` - -There is no `MODEL(type_enforcement ...)` switch. In this example, `order_id` and `ordered_at` are typed; `source_system` remains untyped and passes through unchanged. - -Sources have related behavior but different configuration. A source can explicitly set `type_enforcement: false`; see [Sources](/concepts/sources#type-enforcement). - -### Compile-time checks - -When SQL analysis can infer an output type, SQLBuild compares it with the declared type using adapter-aware normalization. A proven mismatch is a compile error when type enforcement is active. If SQLBuild identifies an output column but cannot prove its expression type, it reports an unproven-type warning rather than guessing. If the entire output shape cannot be inferred, static type checks and warnings cannot run. - -Static analysis does not rewrite the authored query. It determines whether the inferred output is compatible before warehouse execution. - -### Runtime casting - -Runtime enforcement inspects a staged relation. If a typed output column does not already have its declared type, SQLBuild rebuilds the staging or delta relation with an explicit cast: - -```sql -CAST(order_id AS INTEGER) AS order_id -``` - -Only columns with declared types are cast. Untyped output columns are preserved. If a cast fails, the model fails during the type-enforcement phase before the staged table is promoted or the incremental delta is applied. - -Runtime cast support currently depends on materialization: - -| Materialization path | Framework runtime casts | -|----------------------|-------------------------| -| Full table with staged promotion | Yes, on the staging table | -| Non-microbatch incremental | Yes, on the staged delta | -| Microbatch incremental | Yes, on each staged batch | -| View | No; the view query defines the warehouse output type | -| Snapshot | No framework cast reconstruction | -| Custom materialization | Owned by the custom materialization | -| Full table with immediate promotion | Rejected when type enforcement is required | - -Staged promotion is the default table mode. If a project selects immediate table promotion, a typed model fails with guidance to use staged promotion because SQLBuild cannot inspect and reconstruct output before mutating the destination. - -### Type enforcement versus contracts - -These controls answer different questions: - -| Feature | Question | -|---------|----------| -| Type enforcement | Should typed columns be checked and, on supported paths, cast to their declared physical types? | -| `contract none` | Which declared columns can SQLBuild verify statically while still allowing additional output columns? | -| `contract enforced` | Must the output have exactly the complete declared shape? | - -Type enforcement can be active with `contract none`. Conversely, an enforced contract may contain untyped columns: exact shape validation still checks their names, but there is no declared physical type to cast or compare. - -### Schema changes - -Declared types are used in schema-change detection. With type enforcement active, the declared type is authoritative for typed columns when SQLBuild compares a planned model with the warehouse relation. - -## Contracts - -Source: `concepts/models/contracts.mdx` - -Validate required or exact model output schemas. - -A model's declared columns provide output metadata, type enforcement, and column audit attachment. The project `column_contract_mode` and model `contract` policy determine whether those declarations also define a statically validated output shape. - -| Model declaration | Behavior | -|-------------------|----------| -| Unspecified | Follow `settings.column_contract_mode`: `implicit` validates declared columns as an open shape; `explicit` treats them as metadata and audit attachment only. | -| `none` | Do not activate declared-shape or nullability contract validation for this model. Explicit type enforcement remains active. | -| `enforced` | Treat declared columns as the complete authoritative output shape and enable runtime exact-schema checks on supported materializations. | - -The project default is `column_contract_mode = "implicit"`, which preserves SQLBuild's original behavior. Projects that use column declarations primarily for metadata and audits, including migrations from dbt, can opt into explicit contracts: - -```toml -[settings] -column_contract_mode = "explicit" -``` - -Under explicit mode, a model must declare `contract enforced` to activate shape validation. A model-level `contract enforced` or `contract none` always overrides the project mode. - -```sql -MODEL ( - materialized table, - contract enforced, - columns ( - order_id (type INTEGER, nullable false), - customer_id (type INTEGER, nullable false), - amount_cents (type INTEGER), - status (type VARCHAR), - ), -); -``` - -### Validation - -#### Compile time - -When declared-shape validation is active and SQL analysis can infer the model's output: - -- A missing declared column fails. -- A proven type mismatch fails when type enforcement or an exact contract is active. -- A declared non-null column fails when its expression is proven nullable. -- Additional inferred columns fail only for `contract enforced`. - -If SQLBuild cannot infer the output shape, these static shape checks cannot run. An enforced contract additionally requires a non-empty column declaration and adds runtime requirements on supported materializations. - -Type enforcement remains independent. Typed columns continue to receive static type checks and supported runtime casts under explicit mode and `contract none`. - -Configuration fields that reference output columns, including `unique_key`, `cursor`, `updated_at`, and `check_columns`, are checked against an enforced declaration. - -#### Runtime - -For supported materialization paths, `contract enforced` inspects the staged relation and rejects: - -- Missing declared columns. -- Additional undeclared columns. -- Warehouse types that differ from declared types. - -Runtime contract validation does not scan data for nulls and does not inspect warehouse nullability metadata. `nullable false` participates in static nullability analysis; add a `not_null` audit when null values must be checked at runtime. - -Runtime exact-schema validation currently runs for: - -- Staged full-table builds. -- Non-microbatch incremental deltas. -- Snapshot deltas. - -Views and custom materializations rely on compile-time contract analysis. Microbatch incrementals currently apply type enforcement and audits but do not perform the framework runtime exact-schema validation step. - -Staged table validation happens before promotion, so a failure leaves the existing destination untouched. Immediate table promotion is incompatible with `contract enforced` because SQLBuild cannot validate output before replacing the destination. - -### Type enforcement - -Declaring a model column type enables type enforcement automatically, independently of the contract policy. Type enforcement controls static type compatibility and runtime casts on supported table and incremental paths; contracts control output shape. See [Type Enforcement](/concepts/models/type-enforcement) for the materialization matrix. - -### Reusable schemas - -Contracts apply to the effective declaration, not only the reusable base: - -```sql -MODEL ( - model_schema order, - columns ( - ingestion_batch_id (type VARCHAR, nullable false), - ), - contract enforced, -); -``` - -With `contract enforced`, this requires exactly the resolved `order` columns plus `ingestion_batch_id`. With `contract none`, the resolved columns remain metadata, audit attachment points, and inputs to explicit type enforcement without activating shape validation. See [Schemas](/concepts/models/schemas). - -### Enum columns - -A column may use a declared enum as a portable logical domain type: - -```sql -MODEL ( - contract enforced, - columns ( - fulfillment_method (type fulfillment_method), - ), -); -``` - -This does not require, create, or reference a warehouse-native enum type. SQLBuild resolves string-valued enums to `VARCHAR` and integer-valued enums to `INTEGER`. Under `contract enforced`, it also generates an `accepted_values` audit for the declared members. Audit severity and timing follow normal audit configuration and materialization behavior. With the default error severity, it gates staged-table promotion and pre-DML delta paths; views and custom materializations may already have changed their relation when the audit runs. - -Enum member references such as `@enum("fulfillment_method").DELIVERY` are a separate feature that render one validated SQL literal. See [Enum Model Contracts](/concepts/enums/model-contracts) for the complete distinction and lowering behavior. - -### Adopting an existing schema - -Ordinary `sqb compile` validates authored contracts offline. To compare declarations with existing -warehouse relations, use the explicitly online, read-only contract commands: - -```bash -sqb contract diff --from prod --select tag:commerce -sqb contract generate --from prod --select tag:commerce -``` - -The target named by `--from` supplies the database and schema namespace to inspect. SQLBuild uses -the active project connection and never resolves a second set of origin credentials. - -#### Preview and write - -`contract diff` returns exit code `1` when declarations and physical relations disagree. -`contract generate` prints proposed declaration changes without changing files: - -```bash -# Fill missing types and append physical columns missing from code -sqb contract generate --from prod --select tag:commerce --write - -# Replace conflicting types/names and remove declarations absent physically -sqb contract generate --from prod --select tag:commerce --write --overwrite -``` - -Contract commands never mutate warehouse relations, lifecycle state, or fingerprints. - -#### Safety rules - -- Additive generation preserves existing types on conflict, repository-only columns, comments, - descriptions, audits, nullability, enums, tags, freshness, and loader configuration. -- `--overwrite` is an explicit repository replacement policy, not a warehouse write. -- Generation does not add `contract enforced`; activation remains an authored decision. -- A shared `SCHEMA()` is not changed from one model's evidence. Ownership conflicts are reported. -- Writes are atomic and SQLBuild recompiles the project, restoring prior contents if validation - fails. -- Sources are first-class selectable resources, for example `--select source:raw_orders`. - -Enforced upstream contracts are authoritative compile-time interfaces. When input evidence is -complete, missing or ambiguous columns fail in projections, joins, filters, grouping, windows, and -ordering. Partial or opaque schemas remain open; SQLBuild does not turn missing metadata into a -false error. Use `sqb lineage --include-uses --format json` to inspect direct -non-projection uses such as `join_on`, `where`, `group_by`, and `window_order_by`. - -### Related policies - -The project default is implicit open-shape validation for models that omit `contract`. A repository can select explicit opt-in contracts with `settings.column_contract_mode = "explicit"` or configure a custom [Rule](/concepts/rules/custom-rules/overview) that requires enforced contracts for selected model families. - -Contracts also constrain schema-change behavior. For example, `snapshot_schema_change append_new_columns` is incompatible with `contract enforced` because an unannounced appended column would violate the exact declaration. - -## Hooks - -Source: `concepts/models/hooks.mdx` - -Run SQL or Python lifecycle hooks around model materialization. - -Pre-hooks and post-hooks run before and after model materialization. Use SQL hooks for warehouse statements and Python hooks for runtime control flow, providers, queries, and skip decisions. - -### Project layout - -SQL and Python hooks live in language-specific directories: - -```text -my_project/ - models/ - marts/ - orders.sql - hooks/ - sql/ - permissions/ - grant_access.sql - record_access.sql - python/ - notifications.py -``` - -Directories below `hooks/sql/` and `hooks/python/` are organizational. They do not namespace resource names: `hooks/sql/permissions/grant_access.sql` is still invoked as `sql("grant_access")`. - -### Choose a hook type - - - - - - - - - - -Hook lists use three explicit entry types: - -| Entry | Definition | Use | -|-------|------------|-----| -| [`inline_sql("...")`](/concepts/models/hooks/sql#inline-sql-hooks) | SQL written directly in the model header | Short, model-specific SQL | -| [`sql("name", args...)`](/concepts/models/hooks/sql) | A reusable resource under `hooks/sql/` | Shared, parameterized SQL | -| [`python("name", args...)`](/concepts/models/hooks/python) | A decorated function under `hooks/python/` | Control flow, providers, queries, or skips | - -Bare SQL strings are not hook entries. The singular `pre_hook` and `post_hook` fields are also invalid; use `pre_hooks` and `post_hooks` with one of the three typed forms above. `sql("...")` always means a named SQL hook and never falls back to inline SQL. - -### Configure model hooks - -Models may mix SQL and Python entries in one ordered hook list: - -```sql -MODEL ( - materialized table, - pre_hooks [ - inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), - ], - post_hooks [ - sql( - "grant_access", - relation: "@@CTX:destination.qualified", - role: "analyst_role", - ), - python( - "notify_complete", - channel: "#data-builds", - ), - ], -); - -SELECT 1 AS id -``` - -Entries execute in the order authored, including when SQL and Python entries are mixed. - - - -### Lifecycle and failure timing - -Pre-hooks run before materialization. A failed pre-hook prevents materialization, and a Python pre-hook that returns `ctx.skip(...)` stops the remaining pre-hooks and prevents the model from running. - -Post-hooks run after warehouse mutation and audits. They are not promotion gates: a failed post-hook marks the model run as failed, but the relation has already been created, promoted, or incrementally updated. Put logic that must prevent materialization in a pre-hook, contract, pre-promotion audit, or the materialization itself. - -A Python post-hook that returns `ctx.skip(...)` stops the remaining post-hooks and changes the reported model result. `mode="hard"` blocks downstream nodes. A soft skip does not automatically block every downstream node; scheduler propagation also depends on the other upstream results. - -### Names and identity - -SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. - -The ordered hook list participates in each model's version identity: - -- Inline and named SQL hooks contribute their fully rendered statements. Named hooks also retain their resource name, definition, arguments, description, and source path in identity metadata. -- Python hooks contribute the invocation name, configured arguments, and a version hash derived from the decorated function, its transitive first-party dependencies, and decorator configuration. -- Changing a hook body, arguments, or order changes the consuming model's identity and makes it stale for change-aware planning. -- Executed Python hooks record their own hook fingerprints after successful completion or an explicit skip. - -Named SQL hooks compile into their consuming models rather than becoming independently scheduled nodes. Python hooks also run as part of their model's lifecycle phase rather than as independently selected Python nodes. - -### Diagnostics - -Discovery and compilation fail early for malformed definitions and invocations. Diagnostics include the resource path and, for model entries, the model name and indexed label such as `post_hooks[1] sql("grant_access")`. - -See the language-specific pages for definition, argument, validation, and runtime diagnostics: - -- [SQL hooks](/concepts/models/hooks/sql) -- [Python hooks](/concepts/models/hooks/python) - -## SQL Hooks - -Source: `concepts/models/hooks/sql.mdx` - -Define, parameterize, compile, and invoke reusable or inline SQL lifecycle hooks. - -SQL hooks submit one rendered SQL payload to the adapter before or after model materialization. Use a named SQL hook for reusable behavior and `inline_sql(...)` for short model-specific payloads. - -For shared lifecycle ordering, failure timing, naming, and identity rules, see the [Hooks overview](/concepts/models/hooks). - -### Reusable SQL hooks - -SQLBuild discovers `.sql` files recursively under `hooks/sql/`. Each file defines exactly one hook and must start with a `HOOK(...)` header as its first non-whitespace content. - -**`hooks/sql/permissions/grant_access.sql`** - -```sql -HOOK ( - description "Grant a warehouse role access to the model relation" -); - -GRANT SELECT ON @relation TO @role -``` - -The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. - -`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. - -Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. - -### Invoke a named hook - -Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` list: - -**`models/marts/orders.sql`** - -```sql -MODEL ( - materialized table, - post_hooks [ - sql( - "grant_access", - relation: "@@CTX:destination.qualified", - role: "analyst_role", - ), - ], -); - -SELECT 1 AS id -``` - -SQLBuild resource-header fields use native `key value` syntax, as in `TEST (mode macro)`, `SCENARIO (tags ["revenue"])`, `AUDIT (severity error)`, and `HOOK (description "...")`. The `relation: ...` and `role: ...` entries above intentionally retain `key: value` syntax because they are nested named arguments to the `sql(...)` hook call, not resource-header fields. The same distinction applies to named arguments passed to `python(...)`. - -### SQL hook arguments - -Named SQL hooks declare parameters by using them in the SQL body. Arguments are supplied as named values in `sql("name", args...)`: - -| Syntax | Behavior | -|--------|----------| -| `@name` | Raw substitution. Strings are inserted verbatim for relations, identifiers, keywords, or SQL fragments. | -| `@'name'` | SQL-literal substitution. Strings are single-quoted and embedded quotes are escaped. | - -**`hooks/sql/record_access.sql`** - -```sql -HOOK ( - description "Record access configuration" -); - -INSERT INTO audit.access_log (relation_name, role_name) -VALUES (@'relation', @'role') -``` - -```sql -MODEL ( - post_hooks [ - sql( - "record_access", - relation: "@@CTX:destination.qualified", - role: "O'Brien", - ), - ], -); - -SELECT 1 AS id -``` - -For both forms, booleans render as `TRUE` or `FALSE`, numbers render directly, and `null` renders as `NULL`. Lists render as comma-separated values, applying raw or quoted behavior to each item. For example, `@'roles'` with `roles: ["reader", "writer"]` renders as `'reader', 'writer'`. - -Every referenced argument is required and every supplied argument must be used. Missing arguments, unused arguments, and unsupported values such as maps fail compilation. Raw string arguments are not escaped; use `@'name'` for data values and reserve `@name` for trusted SQL structure. - -### Inline SQL hooks - -Use `inline_sql("...")` for SQL that is specific to one model: - -```sql -MODEL ( - materialized table, - post_hooks [ - inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), - ], -); - -SELECT 1 AS id -``` - -An inline hook accepts exactly one quoted SQL string and no additional arguments. That string becomes one adapter execution payload. - -### Compile-time context - -Both named and inline SQL hooks receive the invoking model's runtime context. They support: - -- Project variables such as `@@audit_schema` -- Environment variables such as `@@ENV:DEPLOY_ROLE` -- Hook context variables such as `@@CTX:destination.qualified` -- Enums and constants such as `@enum("role").ANALYST` and `@const("retention_days")` -- Python macros such as `@grant_target("@@CTX:destination.qualified")` - -For named hooks, SQLBuild first substitutes `@name` and `@'name'` arguments into the hook body. A supplied argument such as `relation: "@@CTX:destination.qualified"` therefore resolves to the invoking model's final target-overridden destination. - -An inline hook uses macros, constants, and enums available to its model file. A named hook uses those -available to its own file under `hooks/sql/`. See -[How Visibility Works](/concepts/declaration-scopes/visibility#which-file-controls-visibility). - -`${...}` config-template syntax is not valid in SQL hooks. - -| Variable | Value | -|----------|-------| -| `@@CTX:destination.qualified` | Fully qualified destination relation | -| `@@CTX:destination.schema` | Destination schema | -| `@@CTX:destination.database` | Destination database | -| `@@CTX:destination.table` | Destination relation name | -| `@@CTX:model.name` | Model name | -| `@@CTX:model.database` | Model database | -| `@@CTX:model.schema` | Model schema | -| `@@CTX:model.alias` | Model alias | -| `@@CTX:run.target` | Active target name | -| `@@CTX:run.id` | Current run ID | - -Context components can be combined into qualified identifiers without whitespace or a Python -wrapper: - -```sql -HOOK (); - -CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() -RETURNS INTEGER -LANGUAGE SQL -AS 'SELECT 1' -``` - -### Validation and diagnostics - -SQLBuild strictly validates hook resource syntax, arguments, interpolation, macros, metadata, and non-empty payloads. It does not classify executable statement kinds or implement vendor SQL grammar. Each rendered hook payload is passed to the adapter in one `execute` call; whether a driver accepts multiple statements or client-side batch separators is adapter and warehouse behavior. Use separate hook entries when portable ordering between statements matters. - -After expansion, Polyglot validates the complete payload using the active adapter's analysis dialect when the model's effective `sql_analysis` setting is enabled, which it is by default. The `--no-sql-analysis` flag disables this analysis for the invocation. Polyglot does not understand every administrative or procedural command supported by every warehouse. If it cannot parse valid vendor-specific hook SQL, disable SQL analysis for that model or invocation and let the adapter and warehouse provide the authoritative result. SQLBuild does not compensate with keyword allowlists or handwritten parser fallbacks. - -Common errors include: - -- Missing or malformed leading `HOOK(...)` headers -- Unsupported header keys, empty descriptions, and missing SQL bodies -- Missing or unused arguments and unsupported argument values -- Unknown named hooks, unquoted hook names, and bare hook strings -- SQL rejected by Polyglot while optional SQL validation is enabled - -Definition errors point to the hook file. Invocation and argument errors also identify the consuming model entry, such as `post_hooks[1] sql("grant_access")`. Runtime output preserves the authored hook index and identifies named and inline SQL entries. - -## Python Hooks - -Source: `concepts/models/hooks/python.mdx` - -Define Python lifecycle hooks with runtime context, providers, SQL access, and skips. - -Python hooks run model lifecycle logic that needs Python control flow, providers, warehouse queries, logging, or explicit skip decisions. - -For shared lifecycle ordering, failure timing, naming, and identity rules, see the [Hooks overview](/concepts/models/hooks). - -### Define a Python hook - -SQLBuild discovers decorated functions recursively from `.py` files under `hooks/python/`: - -**`hooks/python/notifications.py`** - -```python -from sqlbuild.hooks import hook - -@hook -def notify_complete(ctx, channel="#data-builds"): - ctx.log( - f"Notify {channel}: " - f"{ctx.model_name} completed during {ctx.phase}" - ) -``` - -By default, the hook name is the function name. The decorator accepts optional `name` and `description` arguments. A Python file may define multiple decorated hooks. - -Files named `__init__.py` or beginning with `_` are skipped. Imported decorated functions are not registered again from the importing module. - -### Invoke a Python hook - -Reference the hook by name and optionally pass keyword arguments: - -```sql -MODEL ( - materialized table, - post_hooks [ - python( - "notify_complete", - channel: "#model-alerts", - ), - ], -); - -SELECT 1 AS id -``` - -Python hook arguments are ordinary configuration values and are not SQL-expanded. An argument containing `@@CTX:...` or `@macro()` reaches the function unchanged. - -### Signature validation - -Unknown hooks, unknown keyword arguments, missing required arguments, required positional-only arguments, and arguments that conflict with context or provider injection fail compilation. A function with `**kwargs` can accept additional configured arguments. - -Python hooks must return `None` or `ctx.skip(...)`. Any other return value fails execution. - -### Hook context - -Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, or `hook_context`. It need not be the first parameter when providers or configured arguments are also present: - -| Field | Description | -|-------|-------------| -| `ctx.model_name` | Model being built | -| `ctx.phase` | `pre_hooks` or `post_hooks` | -| `ctx.hook_name` | Invoked hook name | -| `ctx.hook_index` | Zero-based position in the authored hook list | -| `ctx.run_id` | Current run ID | -| `ctx.target` | Active target | -| `ctx.vars` | Effective project variables | -| `ctx.destination` | Destination relation metadata | -| `ctx.adapter_name` | Active adapter name | -| `ctx.adapter` | Adapter instance | -| `ctx.connection` | Live connection | -| `ctx.execute_sql(sql)` | Execute SQL | -| `ctx.query(sql)` | Execute SQL and return rows | -| `ctx.log(message)` | Write run output | -| `ctx.skip(reason="...", mode="soft")` | Return a soft or hard skip result; arguments are keyword-only | -| `ctx.providers` | Access discovered [providers](/concepts/python-nodes/providers) | - -### Providers - -Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, tasks, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. - -```python -from sqlbuild.hooks import hook - -@hook -def notify_complete(ctx, slack_notifier): - slack_notifier.send( - f"Model {ctx.model_name} built successfully" - ) -``` - -### Skips and failures - -Returning `ctx.skip(...)` stops the remaining hooks in the current phase. Runtime exceptions fail that lifecycle phase. See [Lifecycle and failure timing](/concepts/models/hooks#lifecycle-and-failure-timing) for the effects of pre-hook and post-hook skips, soft and hard modes, and failures after warehouse mutation. - -### Identity and diagnostics - -Python hook invocations and version hashes participate in model identity as described in [Names and identity](/concepts/models/hooks#names-and-identity). Executed hooks also record their own fingerprints after successful completion or an explicit skip. - -Compilation reports unknown hooks and invalid signatures with the model name and indexed invocation label, such as `post_hooks[1] python("notify_complete")`. Runtime output preserves the authored index and hook name. Exceptions and unsupported return values fail with the Python hook identity attached. - -## Configuration - -Source: `concepts/models/configuration.mdx` - -MODEL() header fields and SQL-analysis controls. - -### Common fields - -| Field | Description | -|-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | -| `tags` | Tags used by selectors | -| `description` | Human-readable model description | -| `columns` | Model-local column declarations or inherited-column audit augmentation | -| `model_schema` | Reusable column schema name | -| `audits` | Model-level audit instances | -| `enums` | Model-local enum declarations; names must begin with `_` | -| `constants` | Model-local scalar, list, set, or object constants; names must begin with `_`. Use `constant(...)` for an explicit decimal type or collection rendering override. | -| `schema` | Destination warehouse schema override | -| `database` | Destination database override | -| `alias` | Destination relation-name override | -| `pre_hooks` | Ordered `inline_sql(...)`, `sql("name", ...)`, or `python("name", ...)` hooks before materialization | -| `post_hooks` | Ordered `inline_sql(...)`, `sql("name", ...)`, or `python("name", ...)` hooks after materialization | -| `enabled` | Set to `false` to disable the model | -| `contract` | `none` for an open statically checked declaration, or `enforced` for an exact declaration | -| `sql_analysis` | Per-model SQL-analysis override | - -### SQL analysis - -SQL analysis has project, invocation, and model gates: - -1. `settings.sql_analysis` must be enabled. -2. `--no-sql-analysis` must not be present. -3. `MODEL (sql_analysis false)` can disable analysis for that model. If omitted, the project setting - is used. - -A model cannot re-enable analysis when either broader gate disables it. The older `sql_validation` -and `--no-sql-validation` spellings remain compatibility aliases; use `sql_analysis` for new -configuration. - -### Table fields - -| Field | Description | -|-------|-------------| -| `run_despite_unchanged` | Table-only change-aware policy. `always` rebuilds whenever selected. A duration such as `30d`, `12h`, or `90m` rebuilds while the newest timestamp-based upstream source observation is within that age; it is not a periodic schedule. | - -Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](/concepts/models/materializations#table). - -### Incremental fields - -| Field | Description | -|-------|-------------| -| `incremental_strategy` | `append`, `delete_insert`, or `merge` | -| `cursor` | Output column used to track incremental position | -| `cursor_type` | `timestamp` or `integer` | -| `cursor_grain` | Timestamp grain such as `second`, `hour`, or `day` | -| `cursor_start` | Lower cursor bound | -| `cursor_inputs` | Upstream names mapped to cursor columns | -| `unique_key` | Merge or delete/insert matching columns | -| `incremental_mode` | Set to `microbatch` for batched execution | -| `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | -| `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | -| `batch_size` | Timestamp duration string such as `1d` or `1h`; use a numeric string such as `"1000"` for an integer cursor | -| `batch_concurrency` | Concurrent batch workers; values above `1` require `delete_insert` and the project concurrency gate | -| `microbatch_limit` | Nested `max_batches` and `action` policy for watermark execution | -| `lookback` | Backward replay extension | -| `append_cursor_inclusive` | Include (`true`, default) or exclude (`false`) the current append-cursor boundary | -| `merge_exclude_columns` | Columns left unchanged by matched-row merge updates | -| `full_refresh` | Optional model execution override: `false` always runs incrementally, `true` always full-refreshes, and omission follows the command | -| `on_schema_change` | `append_new_columns`, `sync_all_columns`, `ignore`, or `fail` | -| `replay_on_change` | `forward`, `full`, or `bounded-` | - -See [Incremental](/concepts/incremental) for full semantics. - -Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. - -### Snapshot fields - -| Field | Description | -|-------|-------------| -| `unique_key` | Columns identifying one logical record | -| `snapshot_strategy` | `timestamp` or `check` | -| `updated_at` | Source update timestamp for the timestamp strategy | -| `check_columns` | Columns compared by the check strategy, or `[*]` | -| `observed_at` | Observation timestamp for historical inputs | -| `historical_input` | `snapshot` or `changes` | -| `initial_valid_from` | Initial validity policy for first-seen rows | -| `invalidate_hard_deletes` | Close records that disappear from current-state input | -| `valid_from_column` | Override the generated valid-from column name | -| `valid_to_column` | Override the generated valid-to column name | -| `snapshot_full_refresh` | Snapshot full-refresh safety policy | -| `snapshot_schema_change` | Snapshot schema-change policy | - -See [Snapshots](/concepts/snapshots) for strategy combinations, defaults, and safety behavior. - -### Custom materialization fields - -| Field | Description | -|-------|-------------| -| `config` | Arbitrary values passed to `ctx.config` | -| `placeholders` | Defaults for runtime `@@@placeholder` tokens | - -### Diff fields - -| Field | Description | -|-------|-------------| -| `row_diff_exclude_columns` | Columns excluded from row-level comparison | -| `row_diff_tolerances` | Numeric comparison tolerances | -| `row_diff_sample_rows` | Deterministic unique-key sample size; `0` disables inherited sampling | -| `row_diff_sample_seed` | Integer seed used for deterministic key hashing | - -## Enums - -Source: `concepts/enums.mdx` - -Define a fixed set of named string or integer values and use them safely in SQL. - -Enums give a name to a fixed set of allowed values. SQLBuild checks the enum and every member -reference during compilation, then renders the selected value as a safe SQL literal. - -### Create an enum - -Put project-wide enums under the top-level `enums/` directory. Files are discovered recursively, so -subdirectories can organize a large enum library without changing where the enums are available. - -```text -my_project/ -├── enums/ -│ ├── fulfillment/ -│ │ └── fulfillment_method.sql -│ └── order_status.sql -├── models/ -└── sqlbuild_project.toml -``` - -```sql --- enums/fulfillment/fulfillment_method.sql -ENUM ( - name fulfillment_method, - members [DELIVERY, PICKUP, SHIPPING], -); -``` - -The shorthand above uses each member name as its string value. Use explicit values when the name -used in SQLBuild should differ from the stored value: - -```sql -ENUM ( - name source, - members ( - WEB "web", - PARTNER "partner", - ), -); -``` - -Integer enums always use explicit values: - -```sql -ENUM ( - name priority, - members (LOW 1, HIGH 3), -); -``` - -### Use an enum member - -Reference one member with `@enum("name").MEMBER`: - -```sql -SELECT * -FROM orders -WHERE fulfillment_method = @enum("fulfillment_method").DELIVERY - AND source = @enum("source").WEB -``` - -SQLBuild validates the enum name and member name before the query runs. The active adapter safely -renders the underlying string or integer value. - -Enum references work in model queries, SQL hooks, SQL functions, audits, unit tests, scenarios, and -inline source expressions. - -### Validation rules - -- An enum must contain at least one member. -- Every member must use the same value type: all strings or all integers. -- Enum names and member names must be SQL identifiers. -- Member names must be uppercase and lookup is case-sensitive. -- Enum names must be unique across all public enums in the project. -- Project-wide names cannot begin with `_`; that prefix is reserved for model-private values. - -Invalid declarations, unknown enums, and unknown members fail compilation. - -### More enum features - - Use an enum as a portable model-column domain and generate accepted-value validation. - Keep an enum inside one model when no other resource should use it. - -To limit an enum to one folder, or to that folder and its child folders, see -[Declarations and Scopes](/concepts/declaration-scopes). - -## Constants - -Source: `concepts/constants.mdx` - -Define reusable compiler-validated values and reference them safely from SQL. - -Constants give a name to a value used in SQL. SQLBuild validates the value during compilation and -asks the active adapter to render it safely for its SQL dialect. Constant values are data, never raw -SQL snippets. - -### Create a constant - -Put project-wide constants under the top-level `constants/` directory. Files are discovered -recursively, and one file may contain more than one declaration. - -```text -my_project/ -├── constants/ -│ ├── commerce/ -│ │ └── thresholds.sql -│ └── reporting_day.sql -├── models/ -└── sqlbuild_project.toml -``` - -```sql --- constants/commerce/thresholds.sql -CONSTANT (name min_items, value 7); -CONSTANT (name fallback_source, value "web"); -CONSTANT (name enabled, value true); -CONSTANT (name ratio, value 0.75); -CONSTANT (name missing_value, value null); -``` - -Folders below `constants/` are organizational. They do not change where project-wide constants are -available. - -### Use a constant - -Reference a constant with `@const("name")`: - -```sql -SELECT * -FROM order_batches -WHERE item_count >= @const("min_items") - AND source = @const("fallback_source") -``` - -References work in model queries, SQL hooks, SQL functions, audits, unit tests, scenarios, and -inline source expressions. An unknown constant fails compilation. - -### Scalar values - -Constants support strings, signed integers, booleans, finite floating-point numbers, exact -decimals, and `null`. Integers use a portable signed 64-bit range. `NaN` and positive or negative -infinity are rejected. - -Use `type decimal` with a quoted value when decimal precision must be exact: - -```sql -CONSTANT ( - name usd_rate, - type decimal, - value "2.4700", -); -``` - -SQLBuild parses the quoted value directly as a decimal rather than first converting it to a binary -float. An incompatible `type` and `value` fails compilation. - -### Naming rules - -Public constant names must be unique among public constants and cannot begin with `_`. Enum, -constant, and macro names use separate namespaces, so an enum and a constant may share a name. - -### More constant features - - Define lists, sets, and objects, then choose value-list or native-array rendering. - Keep a constant inside one model when no other resource should use it. - -To limit a constant to one folder, or to that folder and its child folders, see -[Declarations and Scopes](/concepts/declaration-scopes). - -## Collections and Rendering - -Source: `concepts/constants/collections-and-rendering.mdx` - -Define list, set, and object constants and control their adapter-specific SQL rendering. - -Constants can hold collections as well as scalar values. Use this page when a constant represents -a reusable value list, native array, set, or structured object. - -### Lists - -Square brackets declare an ordered list. Lists preserve authored order and allow duplicates: - -```sql -CONSTANT ( - name supported_countries, - value ["GB", "FR", "HK"], -); -``` - -### Sets - -Curly braces declare a set. Sets reject duplicate typed values and use a stable order when SQLBuild -renders or fingerprints them: - -```sql -CONSTANT ( - name unique_countries, - value {"GB", "FR", "HK"}, -); -``` - -`{true, 1}` is valid because a boolean and an integer are different logical types. -`{"GB", "FR", "GB"}` fails compilation instead of silently discarding the duplicate. - -### Objects - -Parenthesized key-value entries declare a string-keyed object. Values may be scalars, lists, sets, -or other objects: - -```sql -CONSTANT ( - name country_rules, - value ( - GB ( - label "Great Britain", - threshold 2.47, - enabled true, - regions ["ENG", "SCT", "WLS"], - ), - FR ( - label "France", - threshold 2.5, - enabled true, - regions ["IDF", "NAQ"], - ), - ), -); -``` - -Object keys must be unique. Objects are logical JSON values rather than portable homogeneous SQL -maps or structs. - -### Collection rules - -Lists and sets must be non-empty and have one compatible element type. Nullable elements do not -determine the type, so `[1, null, 2]` is valid, while these declarations fail: - -```sql -CONSTANT (name empty_values, value []); -- no element type -CONSTANT (name unknown_values, value [null]); -- no non-null element type -CONSTANT (name mixed_values, value [1, "two"]); -- incompatible types -``` - -Objects may contain different value types because each key is checked independently. SQLBuild also -applies nesting-depth, element-count, and rendered-size safety limits. - -### Value-list rendering - -Lists and sets render as a parenthesized value list by default. This is designed for `IN`: - -```sql -WHERE country_code IN @const("supported_countries") -``` - -```sql -WHERE country_code IN ('GB', 'FR', 'HK') -``` - -Every element is escaped by the active adapter. - - A value-list constant is intended for a value-list position such as `IN (...)`. It is not a - portable standalone projection. Use native-array rendering when the constant must be an array - expression. - -### Native-array rendering - -Set `render_as array` to request an adapter-native array: - -```sql -CONSTANT ( - name supported_countries_array, - value ["GB", "FR", "HK"], - render_as array, -); -``` - -SQLBuild does not rewrite array membership operations. Use the operators and functions provided by -your adapter. - -Sets support the same `value_list` and `array` modes as lists. Scalar and object constants reject -`render_as` because those rendering modes do not apply to them. - -| Adapter | Native array expression | Object/JSON expression | -|---------|-------------------------|------------------------| -| DuckDB | `['GB', 'FR', 'HK']` | `json('{"GB":"Great Britain"}')` | -| MotherDuck | `['GB', 'FR', 'HK']` | `json('{"GB":"Great Britain"}')` | -| Snowflake | `ARRAY_CONSTRUCT('GB', 'FR', 'HK')` | `PARSE_JSON('{"GB":"Great Britain"}')` | -| BigQuery | `['GB', 'FR', 'HK']` | `JSON '{"GB":"Great Britain"}'` | -| Databricks | `array('GB', 'FR', 'HK')` | `parse_json('{"GB":"Great Britain"}')` | -| PostgreSQL | `ARRAY['GB', 'FR', 'HK']` | `'{"GB":"Great Britain"}'::JSONB` | -| SQL Server | Unsupported | `JSON_QUERY(N'{"GB":"Great Britain"}')` | - -BigQuery does not support arrays containing arrays. SQL Server has no native array representation. -Unsupported requests fail compilation rather than silently changing representation. - -### Project default - -Set the default for list and set constants in `sqlbuild_project.toml`: - -```toml -[constants] -collection_rendering = "array" -``` - -SQLBuild chooses the rendering mode in this order: - -1. The declaration's `render_as` field -2. Project `[constants].collection_rendering` -3. The `value_list` default - -One constant has one representation throughout a compilation. Changing its value or rendering mode -changes the identity of SQL that uses it. - -### Stable values - -- List order and duplicates are preserved. -- Set order is ignored; membership is stored in a stable order. -- Object key order is ignored; keys are stored in a stable order. -- Changing set membership or object values changes dependent query identity. - -## Enum Model Contracts - -Source: `concepts/enums/model-contracts.mdx` - -Use an enum as a portable model-column domain with generated accepted-value validation. - -An enum can describe the allowed domain of a model column as well as provide individual SQL -literals. - -These are separate uses: - -- `@enum("fulfillment_method").DELIVERY` inserts one validated value into SQL. -- `type fulfillment_method` declares that a model column uses the complete enum domain. - -### Declare an enum-typed column - -Use the enum name as the column type: - -```sql -MODEL ( - contract enforced, - columns ( - fulfillment_method (type fulfillment_method), - ), -); -``` - -SQLBuild does not send `fulfillment_method` to the warehouse as a physical type and does not create a -warehouse-native enum. It translates the enum into portable column metadata: - -| Enum values | Physical column type | Generated validation | -|-------------|----------------------|----------------------| -| Strings such as `DELIVERY` and `PICKUP` | `VARCHAR` | `accepted_values` for the enum strings | -| Integers such as `1` and `3` | `INTEGER` | `accepted_values` for the enum integers | - -### Contract behavior - -With `contract enforced`, SQLBuild runs the generated `accepted_values` audit with the model's other -audits. Its severity and timing follow the model's ordinary audit and materialization settings. - -With `contract none`, SQLBuild still resolves the enum to its portable scalar column type but does -not generate the domain audit. - -This behavior is the same on adapters with and without native enum support. The physical warehouse -column remains an ordinary string or integer column. - -Changing the members of an enum-typed contract changes the model's contract identity and generated -validation. - -## Model-Private Values - -Source: `concepts/model-private-values.mdx` - -Keep enums and constants inside one model when no other resource should use them. - -Put an enum or constant directly in `MODEL()` when it belongs to one model and should not enter the -project-wide namespace. - -```sql -MODEL ( - enums ( - _state [OPEN, CLOSED], - ), - constants ( - _min_items 2, - _supported_countries ["GB", "FR", "DE"], - ), -); - -SELECT * -FROM order_batches -WHERE state = @enum("_state").OPEN - AND item_count > @const("_min_items") - AND country_code IN @const("_supported_countries") -``` - -### Where private values work - -A model-private value is available in: - -- The owning model's query -- Inline SQL hooks written in that model - -It is not available in: - -- Another model -- A child or sibling model directory -- A unit test or scenario -- A named SQL hook stored under `hooks/sql/` - -Private names begin with exactly one `_`. Names beginning with `__` are reserved for SQLBuild. -Because the model owns the name, different models may each define `_state` without creating a -collision. - -### Explicit types and rendering - -Use `constant(...)` when a private constant needs an exact type or rendering choice: - -```sql -MODEL ( - constants ( - _usd_rate constant( - type decimal, - value "2.4700", - ), - _supported_countries constant( - value ["GB", "FR", "HK"], - render_as array, - ), - ), -); -``` - -### When a value becomes shared - -Move the declaration out of `MODEL()` when another resource needs it. Remove the `_` prefix and put -it in the narrowest suitable enum or constant directory. See -[Declarations and Scopes](/concepts/declaration-scopes) for those advanced placement options. - -## Writing Macros - -Source: `concepts/macros.mdx` - -Write Python functions that generate reusable SQL fragments at compile time. - -Macros are Python functions that generate SQL during compilation. They provide reusable, -parameterized SQL without introducing a separate template language. - -### Create a macro - -Put project-wide macros in Python files under the top-level `macros/` directory: - -```text -my_project/ -├── macros/ -│ ├── currency.py -│ └── test_helpers.py -├── models/ -└── sqlbuild_project.toml -``` - -Every public function defined by a macro file becomes callable from SQL: - -```python -# macros/currency.py -def cents_to_dollars(expression: str) -> str: - """Convert a cents expression to dollars.""" - return f"ROUND(CAST({expression} AS DOUBLE) / 100, 2)" -``` - -### Call a macro from SQL - -Use `@macro_name(...)` in a model query: - -```sql -MODEL ( - materialized table, - tags [marts], -); - -SELECT - order_id, - @cents_to_dollars("amount_cents") AS amount_dollars -FROM __ref("stg_orders") -``` - -During compilation, SQLBuild replaces the call with the function's returned SQL: - -```sql -ROUND(CAST(amount_cents AS DOUBLE) / 100, 2) -``` - -A macro used directly in SQL must return a string. - -### Arguments - -Macro calls accept Python literal values: - -- Strings: `"value"` or `'value'` -- Numbers: `42`, `3.14`, `-1` -- Booleans: `True`, `False` -- Lists: `[1, 2, 3]` -- Dictionaries: `{"key": "value"}` -- `None` -- The result of another macro call - -Positional and keyword arguments are supported: - -```sql -@mock_orders(count=5, status="completed") -``` - -Use quoted strings when passing SQL expressions such as column names. The macro decides how to -place that text into its returned SQL. - -### Keep implementation details private - -Only public functions owned by the file are exported as macros. Prefix helpers, constants, classes, -and type aliases with `_`: - -```python -# macros/orders.py -_DEFAULT_STATUS = "completed" - -def _status_filter(status: str) -> str: - return f"status = '{status}'" - -def completed_orders() -> str: - return _status_filter(_DEFAULT_STATUS) -``` - -Here, SQL may call `@completed_orders()`. `_status_filter` and `_DEFAULT_STATUS` remain ordinary -Python implementation details. - -Imported functions are not re-exported as new macros from the importing file. - -### Use macros in tests - -Tests are SQL, so they can use macros as reusable fixture generators: - -```python -# macros/test_helpers.py -def mock_orders(count: int = 1) -> str: - rows = [ - f"SELECT {i} AS id, {i * 100} AS customer_id, 'completed' AS status" - for i in range(1, count + 1) - ] - return " UNION ALL ".join(rows) -``` - -```sql -TEST(); - -WITH -__source__raw__orders AS ( - @mock_orders(3) -), -__expected__stg_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status - UNION ALL - SELECT 2 AS order_id, 200 AS customer_id, 'completed' AS status - UNION ALL - SELECT 3 AS order_id, 300 AS customer_id, 'completed' AS status -) -SELECT 1 -``` - -### Use macros in hooks - -Macros work inside inline and named SQL hooks: - -```python -# macros/permissions.py -def grant_target(target: str) -> str: - return f"GRANT SELECT ON {target} TO analyst_role" -``` - -```sql -MODEL ( - post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], -); - -SELECT 1 AS id -``` - -See [SQL Hooks](/concepts/models/hooks/sql) for hook lifecycle and context syntax. - -### Where macros work - -Macros are supported in: - -- Model query SQL -- Inline and named SQL hooks -- Unit tests and scenarios -- Standalone audit SQL -- SQL functions and supported inline source expressions - -Macros are not accepted in ordinary `MODEL()` configuration fields. SQL hook entries are the -exception because their contents are SQL. - -### Next steps - - Compose macros through Python imports and use adapter or target context. - Limit a macro to one folder, or to that folder and its child folders. - -## Composition and Context - -Source: `concepts/macros/composition-and-context.mdx` - -Compose macros through Python, use compile context, and understand scoped imports. - -Macros are Python functions, so reusable macros should compose through ordinary Python calls. A -macro returns final SQL; SQLBuild does not treat its output as another layer of macro source. - -### Compose helpers in one file - -Use underscore-prefixed helpers for implementation details that should not be callable from SQL: - -```python -# macros/currency.py -def _divide_by_100(expression: str) -> str: - return f"({expression} / 100.0)" - -def cents_to_dollars(expression: str) -> str: - return f"ROUND({_divide_by_100(expression)}, 2)" -``` - -Only `cents_to_dollars` is exported as a SQLBuild macro. - -Public macros in the same file are also ordinary Python functions and may call one another: - -```python -def add_tax(expression: str) -> str: - return f"({expression} * 1.2)" - -def round_money(expression: str) -> str: - return f"ROUND({expression}, 2)" - -def formatted_total(expression: str) -> str: - return round_money(add_tax(expression)) -``` - -### Compose macros from different files - -Import another project macro when it is visible from the importing macro file: - -```python -# macros/orders.py -from macros.currency import add_tax, round_money - -def formatted_order_total(expression: str) -> str: - return round_money(add_tax(expression)) -``` - -SQLBuild records the imported macro files as dependencies. It rejects an import when the target -macro is outside the importing file's declaration scope or when imports form a cycle. - -Imported functions do not become duplicate exports from the importing file. In the example above, -`add_tax` and `round_money` retain their original identities; only `formatted_order_total` is newly -exported by `orders.py`. - -The same visibility direction applies to scoped macros: - -- A scoped macro may import a project-wide macro. -- A scoped macro may import a macro available from its own or an ancestor directory. -- A project-wide macro cannot import a narrower macro. -- A macro cannot import from a sibling or unrelated scope. - -See [Declarations and Scopes](/concepts/declaration-scopes) when macros are stored under `macros/` -or `_macros/`. - -### Macro output is final SQL - -Do not return SQL containing another `@macro()` call: - -```python -# Invalid: creates another macro-expansion layer. -def formatted_order_total(expression: str) -> str: - return f"@round_money(@add_tax({expression!r}))" -``` - -SQLBuild rejects this output. Import and call the Python functions instead: - -```python -from macros.currency import add_tax, round_money - -def formatted_order_total(expression: str) -> str: - return round_money(add_tax(expression)) -``` - -This keeps macro behavior readable in Python and ensures one expansion produces final SQL. - -### Nested calls written in SQL - -The SQL author may explicitly pass one macro's result to another: - -```sql -SELECT @round_money(@add_tax("subtotal")) AS order_total -``` - -SQLBuild evaluates `add_tax` first and passes its returned string to `round_money`. This is not a -second expansion of generated output: both calls are visible in the SQL source. - -Inner macros may return any Python value accepted by the outer macro. A macro used directly in SQL -must return a string. - -### Macro context - -When the first parameter is named `ctx`, SQLBuild passes a `MacroContext` with adapter, target, and -project-variable information: - -```python -# macros/datetime.py -def timestamp_trunc(ctx, grain: str, expression: str) -> str: - if ctx.adapter_name == "bigquery": - return f"TIMESTAMP_TRUNC({expression}, {grain.upper()})" - return f"DATE_TRUNC('{grain}', {expression})" -``` - -| Field | Description | -|-------|-------------| -| `adapter_name` | Active adapter, such as `duckdb` or `snowflake` | -| `sql_analysis_enabled` | Whether SQL analysis is enabled | -| `target_name` | Active target name, when selected | -| `vars` | Effective project variables after project, target, local, and CLI merging | - -```python -def schema_qualified(ctx, table: str) -> str: - schema = ctx.vars.get("schema_prefix", "public") - return f"{schema}.{table}" -``` - -Use context when generated SQL genuinely differs by adapter or target. Prefer ordinary parameters -for values that the SQL caller should choose explicitly. - -## Interpolation - -Source: `concepts/interpolation.mdx` - -How SQLBuild processes variables, context, and dynamic content in SQL and config. - -SQLBuild uses two syntax layers for dynamic content: - -- **`@` syntax** is for any executable SQL - model queries, SQL hooks, tests, audits, and inline source expressions -- **`${...}` syntax** is for config values - project TOML config, MODEL() header fields (excluding SQL hooks), and source/seed YAML declarations - -The rule is simple: if it's any SQL that will be executed, it uses `@`. If it's a config value, it uses `${...}`. These layers never mix. - -### Syntax reference - -| Syntax | Where | Resolved | -|--------|-------|----------| -| `@enum("name").MEMBER` | Executable SQL | Compile time - expands to the validated enum member value | -| `@const("name")` | Executable SQL | Compile time - expands to the named typed scalar, value list, native array, or object expression | -| `@macro(args)` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - expands to macro return value | -| `@@name` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - project variable substitution | -| `@@ENV:NAME` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - environment variable | -| `@@CTX:name` | SQL hooks only | Compile time - destination relation, target, run ID | -| `@@@name` | Model SQL | Preserved for runtime (custom materializations) | -| `@name` / `@'name'` | Named SQL hook bodies | Raw / SQL-literal invocation argument, resolved at compile time | -| `@name` / `@'name'` | Generic audit SQL | Raw / SQL-literal audit argument | -| `${CTX:...}` | TOML/YAML config values | Config compilation | -| `${ENV:...}` | TOML/YAML config values | Config compilation | - -`@@CTX:` is intentionally SQL-hook-only. Model SQL describes a relation's data and should not reference its own destination identity. SQL hooks are the operational SQL layer where destination context is useful - grants, logging, post-materialization DDL. Python hooks access the same information through `ctx.destination` on the [`HookContext`](/concepts/models/hooks/python#hook-context) object. - -See [Enums](/concepts/enums) and [Constants](/concepts/constants) for reusable validated values. See -[Collections and Rendering](/concepts/constants/collections-and-rendering) for lists, sets, objects, -and adapter-specific rendering. - -SQLBuild uses the location of a SQL file to determine which macros, enums, and constants it can -use. Named SQL hooks use their own file location rather than the location of the calling model. See -[How Visibility Works](/concepts/declaration-scopes/visibility) for the complete directory rules. - -### Project variables - -Project variables use `@@name` syntax in SQL and are defined in `sqlbuild_project.toml` or per-target: - -```toml -# sqlbuild_project.toml -[vars] -schema_prefix = "analytics" - -[targets.prod.vars] -schema_prefix = "prod_analytics" -``` - -```sql -SELECT * FROM @@schema_prefix.customers -``` - -Variables can also be set in `sqlbuild_local.toml` for developer-specific overrides, or passed via the CLI using `--vars` with a JSON object: - -```bash -sqb build --vars '{"schema_prefix": "staging_analytics"}' -``` - -CLI vars take precedence over local and project config vars. - -#### JSON vars and nested values - -`--vars` accepts full JSON objects. Values can be strings, numbers, booleans, null, arrays, or nested objects: - -```bash -sqb build --vars '{"schema_prefix": "staging", "grants": {"primary_role": "analyst"}, "enabled": true}' -``` - -In SQL interpolation (`@@name`), only top-level scalar values can be used directly. `null` renders as an empty string. If a variable resolves to an array or object, SQLBuild raises a clear error suggesting you use a macro instead. - -For nested or complex values, use `ctx.vars` in a macro: - -```python -def grant_role(ctx): - return ctx.vars["grants"]["primary_role"] -``` - -The macro context receives the full native JSON structure including nested dicts, lists, booleans, numbers, and `None`. - -### Environment variables - -Environment variables use `@@ENV:NAME` syntax to inject values from the shell environment: - -```sql -SELECT * -FROM @@schema_prefix.customers -WHERE source_system = '@@ENV:SOURCE_SYSTEM' -``` - -If the environment variable is not set, SQLBuild raises a compile error. - -### Context variables - -Context variables provide access to the current model's destination relation, active target, and run metadata. - -**In SQL hooks** (`@@CTX:` syntax): - -```sql -post_hooks [inline_sql('GRANT SELECT ON @@CTX:destination.qualified TO analyst_role')], -``` - -Context components can appear directly beside qualification dots. SQLBuild resolves the longest -available context key before treating the dot as SQL punctuation: - -```sql -CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() -RETURNS INTEGER -LANGUAGE SQL -AS 'SELECT 1' -``` - -**In TOML/YAML config values** (`${CTX:...}` syntax): - -```toml -[targets.prod] -schema = "${CTX:destination.schema}" -``` - -Available context variables: - -| Variable | Value | -|----------|-------| -| `destination.qualified` | Fully qualified destination relation name | -| `destination.schema` | Destination schema | -| `destination.database` | Destination database | -| `destination.table` | Destination relation name | -| `model.name` | Model name | -| `model.database` | Model database | -| `model.schema` | Model schema | -| `model.alias` | Model alias | -| `run.target` | Active target name | -| `run.id` | Current run ID | - -**In macros**, the `MacroContext` object is passed as the first argument when a macro function accepts a `ctx` parameter: - -```python -def timestamp_trunc(ctx, grain: str, expr: str) -> str: - if ctx.adapter_name == "bigquery": - return f"TIMESTAMP_TRUNC({expr}, {grain.upper()})" - return f"DATE_TRUNC('{grain}', {expr})" -``` - -The macro context provides `adapter_name`, `sql_analysis_enabled`, `target_name`, and `vars`. - -### Deferred placeholders - -Custom materializations can define runtime placeholders using `@@@name` syntax. These are preserved through compilation and resolved by the materialization at execution time: - -```sql -WHERE CAST(ordered_at AS DATE) >= CAST(@@@partition_start AS DATE) - AND CAST(ordered_at AS DATE) < CAST(@@@partition_end AS DATE) -``` - -### Audit parameters - -Generic audit SQL uses `@name` (single `@`, no parentheses) for raw SQL placeholders and `@'name'` for escaped SQL-literal placeholders. These are resolved when the audit is attached: - -```sql -SELECT @column -FROM @relation -WHERE @column IS NULL -``` - -Use the quoted form for values rather than SQL identifiers or expressions: - -```sql -WHERE status = @'expected_status' -``` - -This is distinct from `@@name` (project variables) and `@macro()` (macro calls), so there is no ambiguity. See [Audits](/concepts/audits) for details on generic audit parameters. - -### Named SQL hook parameters - -Reusable hooks under `hooks/sql/` use `@name` for raw SQL substitution and `@'name'` for escaped SQL-literal substitution. These parameters are supplied by `sql("name", args...)` and resolved before the rest of the hook SQL is compiled. See [SQL hooks](/concepts/models/hooks/sql#sql-hook-arguments) for value rendering and validation rules. - -### Compilation order - -SQLBuild processes authored SQL in this order: - -1. **Config templates** (`${CTX:...}`, `${ENV:...}`) in TOML/YAML config values are resolved during config compilation -2. **Named SQL hook arguments** (`@name` and `@'name'`) are substituted into reusable hook bodies -3. **Project variables** (`@@name`), **environment variables** (`@@ENV:NAME`), and **context variables** (`@@CTX:name` in SQL hooks) are substituted -4. **Enum and constant references** are validated, normalized, and expanded through the active adapter -5. **Macro calls** (`@name(args)`) are expanded -6. **SQL analysis validation** runs against the fully expanded SQL - -This means: -- Config templates resolve first, before any SQL processing -- Named SQL hook arguments can contain `@@CTX:...`, `@@ENV:...`, project-variable, declaration, or macro text; model context values describe the invoking model, while declarations and macros in the resulting named hook body resolve from the hook definition path -- Macros see already-substituted variable values in the SQL -- `@@CTX:destination.qualified` in SQL hooks sees the final target-overridden destination name because hooks are expanded after destination naming is fully resolved -- SQL analysis validates the final expanded SQL, catching syntax errors from both vars and macros -- Python hook arguments are not SQL and remain unexpanded - -## Functions - -Source: `concepts/functions.mdx` - -User-defined functions and table functions managed as part of your project. - -Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG - if a function definition changes, every model that uses it is rebuilt. - -### Scalar UDFs - -Scalar UDFs return a single value per row. They can be written in SQL or Python. - -#### SQL UDFs - -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body: - -```sql --- functions/sql/udf__is_completed_order.sql -FUNCTION ( - arguments (order_status STRING), - returns BOOLEAN, -); - -order_status = 'completed' -``` - -#### Python UDFs - -Place Python function files under `functions/python/`. Each file has exactly one function decorated with `@udf(...)`: - -```python -# functions/python/is_completed_order_py.py -from sqlbuild.functions import udf - -@udf( - arguments={"order_status": "STRING"}, - returns="BOOLEAN", - runtime_version="3.11", -) -def main(order_status: str | None) -> bool: - return order_status == "completed" -``` - -Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. - -#### Using scalar UDFs - -Reference scalar UDFs in models with `__udf("name")`: - -```sql -SELECT - order_id, - __udf("udf__is_completed_order")(status) AS is_completed -FROM __ref("stg_orders") -``` - -The function name is the file stem (filename without extension). `__udf()` resolves to the adapter-native function call at compile time. - -### Table functions - -Table functions return multiple rows and columns. They are written in SQL under `functions/sql/` with a `returns table(...)` declaration: - -```sql --- functions/sql/table_fn__customer_orders.sql -FUNCTION ( - arguments (p_customer_id INTEGER), - returns table ( - order_id INTEGER, - ordered_at TIMESTAMP, - waffle_name VARCHAR, - line_total_cents INTEGER, - order_status VARCHAR, - is_completed_order BOOLEAN - ) -); - -SELECT - order_id, - ordered_at, - waffle_name, - line_total_cents, - order_status, - is_completed_order -FROM __ref("fact_orders") -WHERE customer_id = p_customer_id -``` - -#### Why table functions exist - -Table functions are designed as an alternative to final-layer views for cases where views don't push predicates efficiently. A view over `fact_orders` with a `WHERE customer_id = ?` filter may scan the entire table if the engine doesn't push the predicate down. A table function accepts the filter as an argument and guarantees the predicate is applied at execution time. - -#### Managed table function dependencies - -Models call managed table functions with `__table_fn("name")(arguments...)`: - -```sql -SELECT customer_id, order_id -FROM __table_fn("table_fn__customer_orders")(42) -``` - -The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. - -The reference creates a typed dependency in the project DAG. Selecting a consuming model also selects the required function, function changes propagate to consumers, and cycles through models and functions are rejected. SQLBuild treats the call as an opaque relation for SQL analysis and uses the function's declared `returns table(...)` columns for star expansion and terminal lineage. - -Incremental state still belongs to the consuming model. A table function does not own a cursor interval or retain execution state. - -#### Using table functions - -Applications and analysts can call the deployed warehouse function directly: - -```sql -SELECT * FROM table_fn__customer_orders(42) -``` - -### References inside functions - -SQL functions can reference the same resources as models: - -| Reference | Syntax | -|-----------|--------| -| Model | `__ref("model_name")` | -| Seed | `__seed("seed_name")` | -| Source | `__source("source_name")` | -| Scalar UDF | `__udf("function_name")` | -| Table function | `__table_fn("function_name")(arguments...)` | - -These references are resolved at compile time and create DAG edges. If a referenced model changes, the function is redeployed. - -A SQL function uses macros, constants, and enums available from its file under `functions/sql/`. -See [How Visibility Works](/concepts/declaration-scopes/visibility) to limit declarations to one -function folder or to that folder and its children. - -### Change propagation - -Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it and marks dependent models as changed. - -### Project layout - -``` -functions/ - sql/ - udf__is_completed_order.sql # scalar SQL UDF - table_fn__customer_orders.sql # table function (returns table) - python/ - is_completed_order_py.py # scalar Python UDF -``` - -### Python UDF options - -The `@udf` decorator accepts these keyword arguments: - -| Argument | Required | Description | -|----------|----------|-------------| -| `arguments` | Yes | Dict mapping argument names to SQL types | -| `returns` | Yes | SQL return type string | -| `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | -| `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | -| `packages` | No | List of Python packages required at runtime | - -### Adapter support - -All four supported adapters implement SQL UDFs, Python UDFs, and table functions: - -| Feature | DuckDB | Snowflake | BigQuery | Databricks | -|---------|--------|-----------|----------|------------| -| SQL UDFs | Yes | Yes | Yes | Yes | -| Python UDFs | Yes | Yes | Yes | Yes | -| Table functions | Yes | Yes | Yes | Yes | - -Future adapters may not support all function types. SQLBuild raises a clear error if a function type is unsupported by the configured adapter. - -## Incremental Models - -Source: `concepts/incremental.mdx` - -Cursor-based incremental strategies, microbatch execution, and backfill policies. - -Incremental models process only new or changed data instead of rebuilding the entire table. In the -default sequential path, SQLBuild works out where to resume from the current target and input -relations with no separate checkpoint store. A retry recomputes its interval from current warehouse -state rather than reusing the exact interval of a failed attempt. Opt-in concurrent microbatching -adds immutable coordination facts as described below. - -### Strategies - -#### append - -Inserts new rows without modifying existing data. Optionally uses a cursor (read from the target table's highest value) to avoid reprocessing the full source on every run. - -```sql -MODEL ( - materialized incremental, - incremental_strategy append, - cursor created_at, - cursor_type timestamp, - cursor_grain second, - append_cursor_inclusive true, -); - -SELECT id, customer_id, created_at -FROM __source("raw_events") -``` - -When `append_cursor_inclusive` is `true` (the default), the lower bound uses `>=`, which may duplicate the boundary row but avoids missing late-arriving data with the same cursor value. Set to `false` for an exclusive (`>`) lower bound if your cursor values are guaranteed unique. - -Append without a cursor is also valid. The model simply inserts all rows from the source query on every run. - -#### delete_insert - -Deletes rows in the cursor range, then inserts the new delta. Requires either `cursor` or `unique_key`. - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - unique_key [order_id], - cursor order_id, - cursor_type integer, -); - -SELECT order_id, customer_id, order_status, ordered_at, line_total_cents -FROM __ref("fact_orders") -``` - -With a cursor, `delete_insert` removes rows where the cursor column falls within the replay window, then inserts the new delta. With a `unique_key` only, it deletes matching rows by key before inserting. - -#### merge - -Upserts rows using a unique key. Matched rows are updated; unmatched rows are inserted. - -```sql -MODEL ( - materialized incremental, - incremental_strategy merge, - unique_key [customer_id], - cursor last_ordered_at, - cursor_type timestamp, - cursor_grain second, - cursor_inputs ( - fact_orders ordered_at, - ), -); - -SELECT - customer_id, - MAX(ordered_at) AS last_ordered_at, - COUNT(*) AS total_orders, - SUM(line_total_cents) AS total_revenue_cents -FROM __ref("fact_orders") -GROUP BY customer_id -``` - -`merge` always requires `unique_key`. The cursor controls which upstream rows are scanned; the unique key determines how they're matched against the target. - -### Full-refresh overrides - -Incremental models can override command-level full-refresh behavior with `full_refresh`: - -| Model setting | Normal command | Command with `--full-refresh` | -|---------------|----------------|-------------------------------| -| omitted | incremental | full refresh | -| `full_refresh false` | incremental | incremental | -| `full_refresh true` | full refresh | full refresh | - -Use `full_refresh false` for models that must retain their normal cursor or microbatch execution even when a broader job requests full refresh: - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - full_refresh false, - cursor event_date, - cursor_type timestamp, - cursor_grain day, -); -``` - -The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. It controls execution mode rather than acting as a safety rejection: an opted-out model does not abort the rest of the build. - -This does not skip initial loading. If the destination relation does not exist, an incremental or microbatch model still builds the history required by its cursor policy. Cloning an existing destination before the build can provide a current watermark and avoid a first-run historical replay. - -### Cursors - -Cursors define the incremental replay boundary. SQLBuild queries `MAX(cursor)` from the target table and `MIN/MAX` from upstream inputs to compute the replay window automatically. Observed maxima are inclusive warehouse values; SQLBuild advances them once to produce an effective exclusive end bound. - -| Field | Description | -|-------|-------------| -| `cursor` | Output column used to track incremental position | -| `cursor_type` | `timestamp` or `integer` | -| `cursor_grain` | Time grain for timestamp cursors: `second`, `minute`, `hour`, `day`, `month`, `year` | -| `cursor_start` | Lower bound floor; the cursor will never replay before this value | -| `cursor_inputs` | Map of upstream ref/source names to their cursor columns | - -#### cursor_inputs - -When a model references multiple upstream inputs, `cursor_inputs` is required to tell SQLBuild which column on each input carries the cursor: - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - cursor_inputs ( - fact_orders ordered_at, - ), -); -``` - -SQLBuild uses these to determine the replay window. With multiple listed inputs, the end is the conservative common watermark: the minimum of their maxima. This prevents a faster input from advancing the model beyond data available from a slower input. - -On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. If it cannot establish a valid interval, the build fails before mutating the destination. - -#### Cursor bounds in model SQL - -Cursor-based incremental models can read their effective interval with zero-argument intrinsics: - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - cursor_inputs ( - fact_orders ordered_at, - ), -); - -SELECT customer_id, ordered_at AS activity_hour -FROM __ref("fact_orders") -WHERE ordered_at >= __cursor_start() - AND ordered_at < __cursor_end() -``` - -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. - -In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. - -##### Listed inputs bound the window; unlisted inputs do not - -Only the inputs you list in `cursor_inputs` bound the replay window. This is an explicit choice, and it has two consequences worth understanding: - -- **Listed inputs** drive the window. Their new data advances the `MAX`, which is what tells SQLBuild how far to reprocess and which rows of the target to rewrite. -- **Unlisted inputs are read in full.** SQLBuild does not add a cursor filter to them, and they do not bound the window. This is correct for lookup or dimension tables that have no meaningful cursor column: you do not list them, and SQLBuild reads them whole rather than trying to filter on a column that may not exist. - -The implication for `delete_insert` and `merge`: the target rows that get rewritten are the ones whose cursor falls inside the window derived from the listed inputs. If an unlisted input changes in a way that should affect target rows outside that window, those rows are not rewritten on a normal incremental run. List every input whose new data should drive reprocessing; leave unlisted only the inputs you intend to read in full. - -To capture changes that fall outside the normal forward window, see [Lookback](#lookback) for late-arriving data and [Replay on change](#replay-on-change) for model changes. - -#### Lookback - -Lookback extends the start of the replay window backwards to re-process recent data. The cursor is forward-moving, so use lookback to capture late-arriving or backfilled records that land just behind the current position: - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor event_date, - cursor_type timestamp, - cursor_grain day, - lookback 3d, -); -``` - -With `lookback 3d`, the replay window starts 3 days before the normal cursor position, ensuring that any late-arriving data within that window is picked up. - -### Microbatch execution - -For large incremental ranges, microbatch mode splits the replay window into configurable batches. -Choose an explicit strategy: - -| Strategy | Window source | -|----------|---------------| -| `watermark` | Declared input availability; supports timestamp and integer cursors | -| `rolling_window` | A timestamp window relative to the current run rather than input maxima | - -Each batch has its own audit cycle: create delta, run delta audits, apply DML, and clean up. - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - incremental_mode microbatch, - microbatch_strategy watermark, - cursor_watermark_mode all, - cursor_inputs ( - fact_orders (column ordered_at, roles [filter, watermark]), - ), - batch_size 1d, -); -``` - -Without microbatch mode, the entire replay range is processed in one pass. - -#### Batch size - -`batch_size` controls the window size for each batch. For timestamp cursors, use duration strings like `1d`, `6h`, `1mo`. For integer cursors, use an integer value. - -For watermark models, each `cursor_inputs` entry declares whether the input filters model SQL, -contributes an availability watermark, or does both. `cursor_watermark_mode all` uses the -conservative common watermark; `any` permits progress from any watermark input. A filter-only input -does not claim that its intervals are available. - -#### Concurrent batches - -Sequential execution is the default and does not read or write `_sqlbuild_microbatches`. To opt into -concurrent `delete_insert` batches, enable the project gate and choose a model limit: - -```toml -[settings] -microbatch_concurrency = true -``` - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - incremental_mode microbatch, - microbatch_strategy watermark, - batch_concurrency 4, - -- cursor configuration omitted -); -``` - -Concurrent execution uses immutable requirement and completion facts to coordinate dependencies and -reconcile retries. It does not update one mutable status row through planned/running/complete -states. Increase concurrency deliberately because every active batch can consume warehouse work. - -#### Watermark batch limits - -Watermark microbatch models can declare what to do when their resolved range contains more batches than an ordinary run should process: - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - incremental_mode microbatch, - microbatch_strategy watermark, - cursor_watermark_mode all, - cursor event_date, - cursor_type timestamp, - cursor_grain day, - cursor_inputs ( - raw_events (column event_date, roles [filter, watermark]), - ), - batch_size 1d, - microbatch_limit ( - max_batches 7, - action cap_from_end, - ), -); -``` - -| Action | Behavior when the range exceeds `max_batches` | -|--------|------------------------------------------------| -| `error` | Fail before hooks or model mutation | -| `warn` | Emit a prominent warning and execute the full range | -| `cap_from_start` | Execute the earliest N aligned batches and defer later work | -| `cap_from_end` | Execute the latest N aligned batches ending at the resolved watermark and defer older work | - -A cap changes only the work selected for that invocation. Deferred batches are not recorded as complete. `cap_from_end` is useful for feeds where keeping the latest projection current is more important than catching up oldest-first; `cap_from_start` is the oldest-first catch-up policy. - -Downstream watermark models consume durable completion facts from capped upstream models. A target -table's physical minimum/maximum does not prove that deferred or intervening intervals were built; -if completion evidence is unavailable, SQLBuild fails closed rather than inventing availability. - -The project can also set an outer safety policy: - -```toml -[microbatches.limits] -max_batches = 100 -action = "error" # or "warn" -``` - -Project limits support only `error` and `warn`; they never silently cap work. When a model has a nested `microbatch_limit`, the project limit checks the full resolved range first and the model policy then applies. - -`--max-microbatches N` is an invocation-wide, hard `error` ceiling and an explicit one-run authorization. It takes precedence over project and model limits, applies to models without a declared limit, and never inherits `cap_from_start` or `cap_from_end`. For example, passing a value large enough for an intentional backfill authorizes the full range instead of retaining the model's ordinary-run cap. - -The legacy scalar `max_microbatches` model field remains supported as a fail/warn guard. New models should use the nested form when the action is part of the model's execution policy. - -#### Mixed-grain chains - -When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` cascade behavior described below. - -Alignment does two things: it floors the replay window edges to the coarsest grain, and it coarsens the batch size to that grain. For example, an hourly model downstream of a daily model processes in day-sized batches, so each batch lines up with a unit of upstream data that actually advances instead of producing empty or boundary-straddling windows: - -```sql --- Upstream: daily grain, 2d batches -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_day, - cursor_type timestamp, - cursor_grain day, - cursor_inputs ( - hourly_order_activity activity_hour, - ), - incremental_mode microbatch, - batch_size 2d, -); - --- Downstream: hourly grain, but aligns to day automatically -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - cursor_inputs ( - daily_activity_rollup activity_day, - ), - incremental_mode microbatch, - batch_size 6h, -); -``` - -### Replay on change - -When a model's version identity changes (query, config, upstream cascade, or any other change reason), `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large downstream tables. You choose the cost per model: - -| Value | Effect | -|-------|--------| -| `forward` | Run the normal incremental delta from the cursor (default) | -| `full` | Full table rebuild | -| `bounded-14d` | Replay the last 14 days of data | - -The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (seconds). For example: `bounded-7d`, `bounded-24h`, `bounded-30m`. - -```sql -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor activity_hour, - cursor_type timestamp, - cursor_grain hour, - replay_on_change full, -); -``` - -See [Cascade propagation](/concepts/planning/cascade-propagation) for how replay policies propagate through the DAG and how downstream models can override inherited replay behavior. - -#### on_schema_change - -Controls how schema differences are handled at execution time when the incremental delta has different columns than the target table: - -| Value | Effect | -|-------|--------| -| `append_new_columns` | Add new columns to the target table (default) | -| `sync_all_columns` | Add, drop, and alter columns to match the delta | -| `ignore` | Log and continue without schema changes | -| `fail` | Reject the build with an error | - -## Planning and change detection - -Source: `concepts/planning.mdx` - -How SQLBuild explains build work, detects changes, and chooses safe model actions. - -`sqb plan` previews the work for a build without executing it. `sqb build` uses the same planner -before running the selected resources. - -Planning and selection are separate concerns: - -- **Selection** determines which resources are in scope. By default, that is the whole project; use - `--select` and `--exclude` to choose a smaller scope. -- **Change detection** compares compiled resources with recorded warehouse state. It explains why a - resource needs work, chooses the correct action for incremental models, propagates upstream - changes, and warns when a partial selection would be incoherent. - -In the default **direct mode**, a build runs its selected scope. Change detection still matters: the -plan can show query and configuration differences, calculate replay or full-refresh actions, and -identify stale upstreams. It does not filter a direct build to stale models. - -[Virtual environments](/concepts/virtual-environments) additionally bind each model to a versioned -physical relation. Virtual builds can use those bindings to reuse existing versions or limit an -invocation to stale work. That virtual execution behavior is documented under -[Building in a virtual environment](/concepts/virtual-environments/building#stale-driven-execution). - -### Reading a plan - -Run the plan command with the same target and selectors you intend to build: - -```bash -sqb plan -sqb --target dev plan --select +fact_orders -``` - -A plan answers four questions: - -1. Which resources are selected? -2. What does SQLBuild know about their current warehouse state? -3. Why does each selected model need work? -4. What materialization or incremental action will the build take? - -Plans may include query diffs, schema changes, backfill ranges, source freshness signals, cascade -reasons, and warnings about stale dependencies. Planning does not modify model relations. - -### What SQLBuild compares - -#### Models and functions - -Each model and function has a version identity derived from the inputs that affect its result: - -- normalized SQL after macro expansion and reference resolution; -- materialization settings, contracts, rendered SQL hooks, Python hook versions, and other - version-relevant configuration; -- referenced function identities; and -- upstream version identities where dependency changes must propagate. - -A function change therefore appears on the function itself and on dependent models. - -#### Seeds - -Seeds use a content hash plus load-affecting configuration. SQLBuild can distinguish an unchanged -seed from one whose file or loading behavior changed. - -#### Python nodes - -Loaders, tasks, assets, checks, and Python hooks have identities derived from project-owned source, -transitive project dependencies, and decorator configuration. Python hook identities also -participate in the identity of models that invoke them. - -For standalone Python nodes, this identity is primarily planning evidence. SQLBuild cannot observe -arbitrary external inputs such as APIs or files, so a node controls its own no-work decision with -`ctx.skip()` rather than relying on SQL identity alone. - -#### Sources - -[Source freshness](/concepts/planning/source-freshness) records an observable data version for a -source. A new observation can mark downstream models as affected even when their SQL is unchanged. - -### Change reasons - -The plan assigns reasons to explain detected work: - -| Reason | Meaning | -|--------|---------| -| First run | No previous identity exists for the resource | -| Query changed / checksum changed | Compiled SQL or seed content differs from the recorded identity | -| Config changed | Version-relevant configuration differs | -| Schema changed | A relevant warehouse or upstream schema changed | -| Function changed | A referenced function has a different identity | -| Upstream changed | A dependency change propagated to this model | -| Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | - -These reasons explain the plan; they are not a promise that direct mode will skip entries labelled -current. Direct builds execute the selected scope. Virtual mode can compare expected identities with -the versions already bound to a VDE and avoid unnecessary version creation. - -### Cascades and incremental actions - -Changes propagate through the DAG in dependency order. The resulting action depends on the -materialization: - -- views are recreated when selected for a direct build; -- tables rebuild when selected; -- incremental models use their cursor state, backfill policy, and `replay_on_change` setting to - determine the affected range; and -- a changed upstream can alter the action of downstream models even when their own SQL is unchanged. - -See [Cascade propagation](/concepts/planning/cascade-propagation) for the detailed materialization -rules. - -### Selection and coherence - -SQLBuild reasons about dependencies outside a scoped selection without silently adding them to the -plan. If building the selected resources would use a stale or missing upstream, the planner warns or -blocks instead of presenting an incoherent partial build as current. - -See [Selection and staleness](/concepts/planning/selection-and-staleness) for closure selectors and -stale-upstream handling. - -### Recorded state - -Direct mode records append-only planning evidence in the warehouse: - -- `_sqlbuild_fingerprints` stores resource identities; -- `_sqlbuild_source_freshness` stores source observations; and -- `_sqlbuild_node_results` stores Python node outcomes. - -The planner reads the latest applicable facts and appends new facts after successful work. Virtual -mode stores version bindings and environment-scoped planning state in its configured state backend; -see [Virtual environment setup](/concepts/virtual-environments/setup). - -### Related topics - -- [Cascade propagation](/concepts/planning/cascade-propagation) -- [Source freshness](/concepts/planning/source-freshness) -- [Selection and staleness](/concepts/planning/selection-and-staleness) -- [Building in a virtual environment](/concepts/virtual-environments/building) - -## Cascade propagation - -Source: `concepts/planning/cascade-propagation.mdx` - -How a change signal propagates downstream through the DAG, and how each materialization type responds. - -When a model or function changes, the change signal propagates downstream through the DAG. Every model downstream of a changed node is marked `Upstream changed` in the plan, even if its own SQL and config are identical. - -The cascade walk is topological: it processes models in dependency order, so each model sees the resolved state of all its upstreams before deciding its own effective action. - -### What cascades - -- A query, config, or schema change on any model cascades to all its downstream dependents. -- A function change cascades to every model that calls it (directly or transitively). -- Source freshness changes propagate downstream the same way. See [Source freshness](/concepts/planning/source-freshness). - -### How materialization types respond - -- **Views** are recreated on every build regardless, so a cascade has no extra cost. -- **Tables** are fully rebuilt, same as if they had changed themselves. -- **Incremental models** receive a replay window from the cascade. A `full` replay always cascades. A `bounded` replay only cascades when the upstream and downstream models share the same `cursor_type` (e.g. both use `timestamp`), so unrelated cursor types don't inherit bounded windows that don't apply. The downstream model's own `replay_on_change` policy takes precedence over any cascaded signal. - -### Resolution when multiple upstreams are stale - -If a model has multiple stale upstreams with different replay actions, the most aggressive action wins. `full` beats `bounded`, and among bounded actions, the longer duration wins. Ties are broken alphabetically by model name for determinism. - -### Overriding cascaded replay - -Downstream incremental models can set their own `replay_on_change` policy to override the cascaded signal. If a downstream model declares its own policy, that policy applies instead of the upstream's. If it has no policy, it inherits the upstream's replay scope. - -### Replay on change - -When a change is detected on an incremental model, `replay_on_change` controls how much data to reprocess: - -- **`forward`** (default) - run the normal incremental delta from the cursor. No reprocessing. -- **`bounded-`** (e.g. `bounded-14d`) - replay the specified window of data. -- **`full`** - drop and rebuild the entire table. - -See [Incremental Models: Replay on change](/concepts/incremental#replay-on-change) for configuration details. - -## Source freshness - -Source: `concepts/planning/source-freshness.mdx` - -Observe external source changes and propagate them through planning. - -Source freshness lets SQLBuild observe whether external source data changed between runs. A changed -observation propagates through the dependency graph, so plans can explain which downstream models -are affected and choose the correct incremental action. In a virtual environment, the same signal -also participates in [stale-driven execution](/concepts/virtual-environments/building#stale-driven-execution). - -### Configuration - -Source freshness is configured per source in `sources/*.yml` with a `freshness:` block. See [Sources: Source freshness](/concepts/sources#source-freshness) for the full configuration reference. - -### How observations work - -During planning, SQLBuild observes the current data version of each source that has freshness configured (or that the adapter can observe automatically): - -1. **Observe** - query the source's current data version using the configured strategy. -2. **Compare** - compare the observed version against the last recorded observation from `_sqlbuild_source_freshness` in the target schema. -3. **Propagate** - walk the DAG downstream from changed or unknown sources to identify which models are affected. - -Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata and the source has a physical table (not an expression source, not a managed source). - -### Lag tolerance - -For timestamp-based freshness, `lag_tolerance` controls how much the observed value can drift before being considered a real change. If the current timestamp is within the tolerance of the previous observation, the source is treated as unchanged. This is useful for sources where the freshness timestamp moves by seconds or minutes on every query but the underlying data hasn't meaningfully changed. - -### State storage - -In direct mode, source freshness observations are appended to `_sqlbuild_source_freshness` in each -target schema. In virtual mode, observations are scoped to the VDE in the configured state backend. -In both modes, failed work does not replace the prior successful observation, so the next plan still -sees the pending source change. - -Direct observations are resolved across all target schemas in the project, so a source referenced by -models in different schemas is tracked consistently. - -Use [`sqb freshness`](/cli/freshness) to observe source freshness on demand without triggering a build. - -## Selection and staleness - -Source: `concepts/planning/selection-and-staleness.mdx` - -How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. - -Change detection is selection-aware. When you scope a run with `--select`, SQLBuild only runs (and only displays) the resources you selected, but it still reasons about the *whole* graph to keep the result honest. The key case it handles: a selected model whose upstream changed but is **not** in the selection. - -### Only selected resources appear in the plan - -For a scoped run like `sqb dbt build --select agg_daily_revenue`, the plan shows only the selected resources, not their passive upstream closure. SQLBuild still walks upstreams internally for change propagation (a selected model flips to run when an upstream changed), but those passive upstreams are not listed as plan entries or counted in the header. - -`--full-refresh` is likewise scoped to the selected models, not the expanded upstream closure. - -### Stale warnings instead of silent partial rebuilds - -If a selected model has an upstream that changed but is **outside the selection**, rebuilding the selected model alone would run it on top of a stale upstream and silently produce a result that looks current but isn't. SQLBuild does not do that. It leaves the selected model current (it is not rebuilt) and emits a stale warning naming the changed upstreams, with the selector you can use to incorporate them. - -Suppose `stg_orders` (upstream) changed, and you select only the downstream `agg_daily_revenue`: - -```bash -sqb dbt plan --select agg_daily_revenue -``` - -``` -Plan ready (1 selected resources) - -dbt (1 selected resources) - planned models: 0 run, 1 current, 0 blocked - - Model plan - Current (1) - model.jaffle_analytics.agg_daily_revenue no change - -Warnings (1) - - selected dbt model 'agg_daily_revenue' is stale: upstream stg_orders changed - but will not be rebuilt or is stale; rebuild with a closure selector - (e.g. +model) to incorporate it -``` - -The model stays current rather than being rebuilt on a stale input, and the warning tells you exactly which upstreams changed and how to pull them in. - -### Incorporating the changed upstreams - -Use a closure selector to include the changed upstreams in the run. With `+agg_daily_revenue`, SQLBuild pulls in the changed `stg_orders`, cascades the change through the intermediate models, and rebuilds: - -```bash -sqb dbt plan --select +agg_daily_revenue -``` - -``` -Plan ready (8 selected resources) - planned models: 5 run, 1 current, 0 blocked - - Model plan - Checksum changed (1) - model.jaffle_analytics.stg_orders checksum changed - Upstream changed (4) - model.jaffle_analytics.agg_daily_revenue upstream changed - model.jaffle_analytics.fct_orders upstream changed - model.jaffle_analytics.int_order_payments upstream changed - model.jaffle_analytics.stg_order_statuses upstream changed -``` - -### Seeds follow the same rule - -Seed changes are tracked the same selection-aware way. A changed seed that is outside the current selection does not trigger a partial rebuild of its dependents; instead the dependents are left current and a stale warning is emitted, exactly as for models. This keeps a scoped run from quietly building on a seed that the run didn't reload. - -### Why this matters - -A run that reports success but silently built a model on stale inputs is a correctness hazard - the data looks fresh but isn't. Surfacing staleness as an explicit warning (and refusing the misleading partial rebuild) keeps scoped runs honest: you either get a result built on current inputs, or a clear warning telling you it would not be, with the fix. - -## Snapshots (SCD Type 2) - -Source: `concepts/snapshots.mdx` - -Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. - -Snapshot models maintain historical row versions with validity windows. They answer questions like: what does this entity look like now, what did it look like before, when did it change, and was it absent during a period. - -### How snapshots work - -SQLBuild adds two generated columns to the target table: - -| Column | Meaning | -|--------|---------| -| `valid_from` | When this version became valid (inclusive) | -| `valid_to` | When this version stopped being valid (exclusive). `NULL` means currently active. | - -A point-in-time query uses the interval `valid_from <= point_in_time < valid_to`. - -### Change detection strategies - -#### Timestamp strategy - -Use when your source has a reliable column recording when the entity changed. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at, -); - -SELECT - customer_id, - name, - plan, - status, - updated_at -FROM __source("customers") -``` - -If the source `updated_at` is newer than the active target row's `updated_at`, SQLBuild closes the old version and inserts the new one. If `updated_at` is unchanged or older, nothing happens. - -#### Check strategy - -Use when the source does not have a reliable update timestamp. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy check, - check_columns [name, plan, status], -); - -SELECT - customer_id, - name, - plan, - status -FROM __source("customers") -``` - -SQLBuild compares `check_columns` between source and active target rows. If any checked value differs, a new version is created. Changes to unchecked columns are ignored. - -`check_columns [*]` checks all output columns except `unique_key` and the generated validity columns. Explicit columns are recommended for important models to avoid noisy history from volatile metadata columns. - -### Historical input - -By default, SQLBuild treats the model query as returning the current state of each entity (one row per `unique_key`). When your source contains historical observations over time, add `observed_at` to switch to historical mode. - -#### Historical check snapshot - -Use for daily full exports or periodic snapshots without a business update timestamp. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy check, - check_columns [plan, status], - observed_at snapshot_date, -); - -SELECT - customer_id, - plan, - status, - snapshot_date -FROM __source("customers_daily_snapshot") -``` - -Each `observed_at` group is treated as a complete picture of the source at that time. Consecutive unchanged observations are collapsed into a single version. - -#### Historical timestamp snapshot - -Use for historical observations that include a business update timestamp. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at, - observed_at extract_date, - historical_input snapshot, -); - -SELECT - customer_id, - plan, - status, - updated_at, - extract_date -FROM __source("customers_historical_extracts") -``` - -Each row means: "at `extract_date`, the source's current state for this key had this `updated_at`." Validity windows use `updated_at`, not `observed_at`. - -#### Historical change records - -Use for CDC tables, audit logs, or historical backfills where rows are individual version records. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at, - observed_at loaded_at, - historical_input changes, -); - -SELECT - customer_id, - plan, - status, - updated_at, - loaded_at -FROM __source("customers_cdc") -``` - -Multiple changes for the same key in one batch are allowed. `updated_at` determines version ordering. `observed_at` is arrival/load time, not validity time. - -#### Historical input rules - -| Strategy | `historical_input` | Source shape | Uniqueness | Hard deletes | -|----------|-------------------|-------------|-----------|-------------| -| `check` | `snapshot` (default) | Complete observations over time | `unique_key + observed_at` | Allowed | -| `timestamp` | `snapshot` | Complete observations with update timestamp | `unique_key + observed_at` | Allowed | -| `timestamp` | `changes` | Individual change/version records | `unique_key + updated_at` | Not allowed | -| `check` | `changes` | Not supported | - | - | - -Timestamp snapshots with `observed_at` require `historical_input` to be set explicitly. - -### Hard deletes - -```sql -invalidate_hard_deletes true, -``` - -When enabled, active target rows whose keys are missing from the source are closed: - -- **Current-state input**: closed at execution time -- **Historical input** (`historical_input snapshot`): closed at the `observed_at` time of the group where the key is missing - -Hard deletes are not allowed with `historical_input changes` because change-record batches are not complete source snapshots - a missing key just means no change, not deletion. - -Reappearing keys create a new active version. - -### Configuration reference - -| Field | Required | Description | -|-------|----------|-------------| -| `materialized snapshot` | Yes | Enables snapshot lifecycle | -| `unique_key` | Yes | Column(s) identifying one entity | -| `snapshot_strategy` | Yes | `timestamp` or `check` | -| `updated_at` | Timestamp only | Source column with business update time | -| `check_columns` | Check only | Columns compared to detect changes. Use `[*]` for all non-key columns. | -| `observed_at` | No | Source column with observation/extract time. Presence enables historical mode. | -| `historical_input` | Conditional | `snapshot` or `changes`. Required for timestamp with `observed_at`. Defaults to `snapshot` for check with `observed_at`. | -| `invalidate_hard_deletes` | No | Close active rows missing from source. Default `false`. | -| `valid_from_column` | No | Override generated column name. Default `valid_from`. | -| `valid_to_column` | No | Override generated column name. Default `valid_to`. | -| `initial_valid_from` | No | First-version start time: `updated_at`, `observed_at`, or `execution_time`. See defaults below. | -| `snapshot_full_refresh` | No | Model-level full-refresh safety: `deny`, `require_confirmation`, or `allow`. | - -#### Initial valid_from defaults - -| Case | Default | -|------|---------| -| Timestamp, no `observed_at` | `updated_at` | -| Timestamp, with `observed_at` | `updated_at` | -| Check, no `observed_at` | `execution_time` | -| Check, with `observed_at` | `observed_at` | - -### Full refresh safety - -Snapshot full refresh can permanently discard history that cannot be reconstructed from the source. SQLBuild guards against this with configurable safety policies. - -#### Project config - -```toml -[snapshots] -current_state_full_refresh = "deny" -historical_full_refresh = "require_confirmation" -``` - -| Policy | Behavior | -|--------|----------| -| `deny` | Full refresh is blocked regardless of CLI flags | -| `require_confirmation` | Requires `--allow-snapshot-full-refresh` flag or interactive confirmation | -| `allow` | No snapshot-specific confirmation required | - -#### Defaults - -| Snapshot type | Default policy | Reason | -|--------------|---------------|--------| -| Current-state (no `observed_at`) | `deny` | Cannot reconstruct older history from current-state source | -| Historical (with `observed_at`) | `require_confirmation` | Can reconstruct if query returns full history, but SQLBuild cannot prove that | - -#### Model override - -The model `snapshot_full_refresh` field can only make the policy **stricter** than the project setting. A model cannot weaken `deny` to `allow`. - -```sql -MODEL ( - materialized snapshot, - ... - snapshot_full_refresh deny, -); -``` - -#### CLI usage - -```bash -# Fails if any selected snapshot has effective policy 'deny' -sqb build --full-refresh - -# Satisfies 'require_confirmation' policy (cannot override 'deny') -sqb build --full-refresh --allow-snapshot-full-refresh -``` - -### Audits - -Snapshot models support the same audit system as other materializations. Audits with `delta_and_final` run scope execute against the snapshot delta relation before target mutation, blocking promotion if an error-severity audit fails. Final audits run after target mutation. - -```sql -MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at, - columns ( - customer_id (audits [not_null (run_scope delta_and_final)]), - ), -); -``` - -### Duplicate handling - -SQLBuild fails with an actionable error if the source query produces duplicate rows at the snapshot identity grain: - -- Current-state: duplicate `unique_key` -- Historical snapshot: duplicate `unique_key + observed_at` -- Historical changes: duplicate `unique_key + updated_at` - -Deduplicate in your model SQL: - -```sql -WITH ranked AS ( - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY customer_id, snapshot_date - ORDER BY loaded_at DESC - ) AS rn - FROM __source("customer_daily") -) - -SELECT customer_id, plan, status, snapshot_date -FROM ranked -WHERE rn = 1 -``` - -### Querying snapshots - -#### Current rows - -```sql -SELECT * FROM customer_snapshot WHERE valid_to IS NULL -``` - -#### Point-in-time - -```sql -SELECT * -FROM customer_snapshot -WHERE customer_id = 1 - AND TIMESTAMP '2026-02-15' >= valid_from - AND (valid_to IS NULL OR TIMESTAMP '2026-02-15' < valid_to) -``` - -#### Fact-to-dimension historical join - -```sql -SELECT - o.order_id, - o.ordered_at, - c.plan -FROM orders o -JOIN customer_snapshot c - ON o.customer_id = c.customer_id - AND o.ordered_at >= c.valid_from - AND (c.valid_to IS NULL OR o.ordered_at < c.valid_to) -``` - -### Examples - -#### Composite key - -```sql -MODEL ( - materialized snapshot, - unique_key [user_id, role_id], - snapshot_strategy check, - check_columns [role_name, role_status], - observed_at snapshot_date, - invalidate_hard_deletes true, -); - -SELECT - user_id, role_id, role_name, role_status, snapshot_date -FROM __source("user_role_daily") -``` - -#### Custom validity column names - -```sql -MODEL ( - materialized snapshot, - unique_key [product_id], - snapshot_strategy timestamp, - updated_at modified_at, - valid_from_column effective_from, - valid_to_column effective_to, -); - -SELECT - product_id, name, price, modified_at -FROM __source("products") -``` - -## Audits - -Source: `concepts/audits.mdx` - -Violation and measurement checks that gate data and record quality outcomes. - -Audits are SQL queries that verify data quality. Violation audits return invalid rows. Measurement -audits return a value that SQLBuild evaluates against authored thresholds and sample policy. -SQLBuild can run audits before table promotion or incremental DML so error-severity failures do not -reach the target. - -### How audits work - -Violation audits pass when their query returns zero rows. Measurement audits produce one value and -optionally a sample count; their outcome is `pass`, `warn`, `fail`, or `insufficient`. - -For `error` severity audits: -- **Full table builds:** SQLBuild materializes into a staging table, runs audits against it, and only promotes to the target if all audits pass. If any fail, the staging table is kept for inspection and the production table is untouched. -- **Incremental models:** Delta-phase audits validate each batch before DML is applied. If an audit fails, the batch is not applied. - -For `warn` severity audits, the build continues and the failure is reported in the output. - -### Measurement audits - -A reusable measurement audit separates the aggregate query from optional bounded evidence: - -```sql --- audits/generic/valid_order_rate.sql -AUDIT ( - evaluation measurement, - value valid_rate, - sample_count total_rows, - sample_unit rows -); - -MEASURE ( - SELECT - COUNT(*) AS total_rows, - 100.0 * AVG(CASE WHEN @condition THEN 1 ELSE 0 END) AS valid_rate - FROM @relation -); - -EVIDENCE ( - SELECT * FROM @relation WHERE NOT (@condition) -); -``` - -Attach threshold and sample policy where the audit is used: - -```sql -MODEL ( - audits [ - valid_order_rate ( - condition "order_id IS NOT NULL", - minimum_samples 100, - evidence_limit 20, - thresholds (warn (below 99.9), error (below 99)) - ) - ] -); -``` - -`minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a -pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and -threshold determine the outcome. - -### Audit factories - -Use a Python audit factory when many related audit instances should be generated from one reviewed -declaration: - -```python -from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory - -@audit_factory -def order_quality(): - return [ - AuditCase( - name="positive_amount", - definition="expression_is_true", - arguments={"expression": "amount > 0"}, - severity=AuditSeverity.ERROR, - ) - ] -``` - -Attach it with `MODEL (audit_factories [order_quality])`. Generated cases compile to the same audit -contract as directly authored instances. - -### Result history - -Native warehouse adapters best-effort append confirmed audit outcomes to -`_sqlbuild_audit_results`. Rows are immutable and use deterministic IDs, so retrying the same result -is idempotent. Projection failure is reported separately and does not change the audit outcome or -command exit code. Lifecycle sinks can also consume the corresponding `audit_completed` fact. - -### Built-in audits - -SQLBuild includes four generic audits out of the box. You do not need to define these in `audits/generic/` - they are available automatically: - -| Audit | Description | Parameters | -|-------|-------------|------------| -| `not_null` | Fails if any row has a NULL value in the column | Column-level only | -| `unique` | Fails if any non-NULL value appears more than once | Column-level only | -| `accepted_values` | Fails if any non-NULL value is not in the allowed list | `values` - list of allowed values | -| `relationships` | Fails if any non-NULL value does not exist in the referenced column | `to` - target relation, `field` - target column | - -#### Using built-in audits - -Attach them in the `MODEL()` header like any generic audit: - -```sql -MODEL ( - materialized view, - tags [staging], - columns ( - order_id (audits [not_null, unique]), - customer_id (audits [not_null]), - status ( - audits [ - accepted_values (values ["placed", "preparing", "ready", "completed", "cancelled"]), - ], - ), - payment_method ( - audits [ - relationships (to "stg_payments", field "method"), - ], - ), - ), -); -``` - -#### Overriding built-in audits - -If you define a generic audit with the same name as a built-in (e.g. `audits/generic/not_null.sql`), your definition takes precedence. SQLBuild emits a warning so you're aware of the override: - -``` -warning[P003]: project audit 'not_null' overrides built-in audit 'not_null' -``` - -### Custom generic audits - -Beyond the built-ins, you can define reusable SQL templates under `audits/generic/`. They use `@parameter` placeholders that are resolved by the audit engine at compile time. - -```sql --- audits/generic/expression_is_true.sql -AUDIT (); - -SELECT * -FROM @relation -WHERE NOT (@expression) -``` - -#### Audit parameters - -Generic audit SQL uses `@name` for parameter placeholders. These are resolved by the audit engine, not the general SQL interpolation system: - -| Parameter | Description | -|-----------|-------------| -| `@column` | The column name (auto-populated for column-level audits) | -| `@relation` | The target relation (auto-populated from the attached model or source) | -| `@'name'` | A quoted parameter passed from the audit declaration (e.g. `@'values'`) | -| `@name` | An unquoted parameter passed from the audit declaration (e.g. `@expression`) | - -Generic and singular audit SQL uses macros, constants, and enums available from the audit file under -`audits/`, not from a model or source that uses the audit. See -[How Visibility Works](/concepts/declaration-scopes/visibility). - -#### Attaching custom generic audits - -```sql -MODEL ( - materialized table, - audits [ - expression_is_true ( - name "revenue_is_non_negative", - expression "total_revenue_cents >= 0", - ), - ], -); -``` - -### Singular audits - -Singular audits are standalone SQL files. Their canonical home is `audits/singular/`, and they -reference models directly. They're useful for one-off checks that don't fit a reusable template. -For backward compatibility, singular audits directly under `audits/` or another non-`generic` -child directory continue to compile. - -```sql --- audits/singular/orders_have_payments.sql -AUDIT ( - name "completed_orders_have_payments", - severity error -); - -SELECT o.order_id -FROM __ref("fact_orders") o -LEFT JOIN __ref("stg_payments") p ON o.order_id = p.order_id -WHERE p.payment_id IS NULL - AND o.order_status = 'completed' -``` - -SQLBuild automatically infers which model a singular audit attaches to based on the `__ref()` calls in the query. If the audit references a single model, it attaches to that model. If it references multiple models, SQLBuild attaches it to the latest (most downstream) model in the DAG. If attachment can't be inferred, the audit runs at the end of the build. - -### Source audits - -Sources support the same audit system as models. Audits attached to sources run *before* any dependent model is built: - -```yaml -sources: - - name: raw__orders - columns: - - name: id - audits: - - not_null - - unique - audits: - - expression_is_true: - name: no_future_orders - expression: "ordered_at <= CURRENT_TIMESTAMP" -``` - -If a source audit with `error` severity fails, all downstream models that depend on that source are blocked. This lets you catch data quality issues at the source before any transformations run. - -### Severity - -| Severity | Behavior | -|----------|----------| -| `error` | Blocks the build. Staging table is not promoted, DML is not applied. | -| `warn` | Reports a warning but allows the build to continue. | - -Set the default severity in `sqlbuild_project.toml`: - -```toml -[settings] -default_audit_severity = "warn" -``` - -Override per audit instance in the `MODEL()` header: - -```sql -columns ( - order_id (audits [not_null (severity error)]), -), -``` - -### Run scope - -Audits on incremental models can run at different lifecycle phases: - -| Scope | Behavior | -|-------|----------| -| `final` | Run once against the staged table before promotion (default). | -| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete. | - -```sql -MODEL ( - materialized incremental, - ... - columns ( - activity_hour (audits [not_null (run_scope delta_and_final)]), - ), - audits [ - expression_is_true ( - name "orders_placed_is_non_negative", - expression "orders_placed >= 0", - run_scope delta_and_final, - ), - ], -); -``` - -Delta-phase audits with `error` severity block DML before the target is updated. This is visible in the build output as `audit (d)` for delta-phase and `audit (f)` for final-phase: - -``` - 10/13 table hourly_order_activity (delete_insert) OK 0.16s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS - audit (f) not_null (activity_hour) PASS -``` - -The `4/4` indicates the audit passed for all 4 microbatch batches. - -If a model is not incremental, `delta_and_final` degrades to `final` automatically. - -### Running audits standalone - -```bash -sqb audit -``` - -This runs all audits without rebuilding any models. - -Standalone audits run serially unless concurrency is configured explicitly, through -`SQLBUILD_CONCURRENCY`, or in project settings. For example, `sqb audit --concurrency 8` runs up -to eight selected audits at once, using one warehouse connection per active worker. Increase this -limit deliberately because parallel queries can increase warehouse load and cost. See -[`sqb audit`](/cli/audit) for precedence, ordering, and cancellation details. - -## Overview - -Source: `concepts/rules.mdx` - -Turn repeated SQL and project review decisions into compiler-enforced requirements. - -Code review is an expensive place to keep enforcing a decision the team has already made. - -Comments such as these are useful once, but repetitive after that: - -- “This query needs deterministic ordering before `LIMIT`." -- “Move this model into the agreed area and layer." -- “Final models cannot depend on another area's private intermediate models." -- “This incremental output needs a declared grain, an enforced contract, and a focused test." -- “External sources must enter through staging." - -Rules turn those decisions into configurable compile-time requirements. They run over the same -project representation SQLBuild uses to produce executable artifacts, so a failed Rule is a failed -compile rather than an optional report somebody must remember to run. - -### What Rules are for - -Start with built-in Rules, then add custom Rules for requirements specific to your project. - -- **Built-in Rules** are opinionated, reusable checks for areas such as SQL safety, contracts, and - dependency boundaries. Select or ignore each Rule individually. -- **Custom Rules** encode local decisions such as directory structure, architectural boundaries, - public interfaces, or the contract and test coverage required for a particular kind of model. - -Both evaluate authored SQL and the compiled project without querying warehouse data. They use -compiler-owned facts about SQL, models, paths, configuration, columns, contracts, tests, audits, -declarations, and dependencies. - -That context can be combined in one check—for example, allowing `SELECT *` in staging while requiring -contracted outputs in final models, or allowing cross-area dependencies only through an `interface` -model. - -### How Rules work - -Use `sqb rules show` to understand a Rule before enabling it: - -```bash -sqb rules show SQBRSQL004 -``` - -```text -SQBRSQL004: Row selection is nondeterministic -Family: SQBRSQL -Slug: unordered-limit -Subject: model -Enabled by default: yes -Remediation: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. -``` - -`Enabled by default` means the Rule is included when a matching family is selected. Configurable -Rules do not run unless `select` contains an exact code or matching family. - -Select Rules in `sqlbuild_project.toml`. This example enables the complete built-in SQL family: - -```toml -[rules] -select = ["SQBRSQL"] -``` - -An exact code such as `SQBRSQL004` selects one Rule. A family code such as `SQBRSQL` selects every -Rule in that family. An empty `select` disables configurable Rules, but mandatory compiler -correctness still applies. - -Now consider a model that returns an arbitrary row because its `LIMIT` has no ordering: - -```sql --- models/orders.sql -SELECT order_id FROM orders LIMIT 1 -``` - -Run normal compilation: - -```bash -sqb compile -``` - -The selected Rule reports the source location, problem, and remediation: - -```text -error[SQBRSQL004]: Row selection is nondeterministic - --> models/orders.sql:2:36 - | - 2 | SELECT order_id FROM orders LIMIT 1 - | ^ - = help: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. -``` - -Compilation fails before SQLBuild completes artifacts or opens a warehouse connection. Add -deterministic ordering and compile again: - -```sql -SELECT order_id FROM orders ORDER BY order_id LIMIT 1 -``` - -Use focused commands when discovering or adopting Rules: - -```bash -sqb rules list -sqb rules run SQBRSQL004 -``` - -A focused run checks only the requested Rule or family. Run `sqb compile` before treating the whole -project as valid. - -### Where SQL linting went - -SQL linting is part of the Rules system rather than a separate command. - -The `SQBRSQL` built-in family covers deterministic SQL diagnostics such as: - -- unsafe `NULL` comparisons; -- implicit cartesian joins; -- unordered `LIMIT` and `OFFSET`; -- unused CTEs; -- duplicate aliases; -- unstable window ordering; -- risky set-operation shapes; -- selected SQL structure and convention requirements. - -These checks run through ordinary compilation. There is intentionally no separate `sqb lint` -lifecycle. If an SQL requirement affects whether the project is acceptable, the compiler enforces -it with the rest of the configured Rules. - -Not every traditional lint concern should become a diagnostic. SQLBuild separates three kinds of -SQL responsibility: - -| Responsibility | Owner | Example | -|---|---|---| -| Correctness required to understand the project | Compiler | Invalid syntax, unresolved references, incompatible output shape | -| Configurable, deterministic requirements | `SQBRSQL` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | -| Canonical presentation | [`sqb format`](/cli/format) (separate from the compiler) | Capitalization, indentation, spacing, comma and clause layout | - -Rules never rewrite source. `sqb format` rewrites source and does not decide whether a project -passes its configured Rules. This keeps diagnostics and source mutation separate. - -### Built-in and custom Rules - -SQLBuild maintains native built-in Rules for reusable requirements. Their codes begin with `SQBR`, -for example `SQBRSQL004` and `SQBRGRAPH101`. - -Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for -example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, -and cache system as built-ins. - -```toml -[rules] -select = [ - "SQBRSQL", - "SQBRGRAPH", - "XSQBRARCH", -] -``` - - Define typed model or project checks over compiler-owned facts with `@rule`, `RuleContext`, and - `Finding`. - -### One definition of project validity - -`sqb compile` is authoritative. SQLBuild evaluates a project in this order: - -```text -mandatory compiler correctness -→ selected native built-in Rules -→ selected custom Python Rules -→ artifact completion -``` - -Build and planning commands use the same compiler path. Configured findings stop execution before a -warehouse connection is opened. Compile artifacts are not completed from a project that failed its -Rules. - -### Rules compared with tests, audits, and contracts - -Rules do not replace every form of validation: - -| Capability | Question it answers | -|---|---| -| Mandatory compiler correctness | Can SQLBuild construct a trustworthy project and executable representation? | -| Rules | Does the authored and compiled project satisfy selected static requirements? | -| Contracts | Does a model declare and preserve the required output shape? | -| SQL tests | Does SQL logic produce an expected result for controlled inputs? | -| Audits | Does warehouse data satisfy a runtime quality requirement? | -| Runtime checks | Are external resources and execution state valid now? | - -Use a Rule when the answer can be determined from source and compiler facts. Use a test when you -need to execute SQL against controlled data. Use an audit when the answer depends on warehouse data. - -### Findings and intentional exceptions - -A finding contains a stable code, project-relative path, line, column, explanation, and remediation. -This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. - -Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores -require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently -accumulate. Mandatory compiler correctness cannot be suppressed. - -See [Findings and exceptions](/concepts/rules/findings-and-exceptions) for configuration examples. - -### Performance is part of the design - -Model Rules are evaluated and cached per model. Editing one model invalidates affected subjects -rather than every invocation. Project Rules run once over the complete project view and are the -explicit choice for genuinely project-wide requirements, with broader invalidation. SQLBuild tracks -the inputs each Rule depends on so cached findings are reused only while those inputs remain valid. - -In required CI, 20 custom Rules over a production-shaped 5,000-model project complete their first -compile in just over 10 seconds and an unchanged compile in under six. A 100-Rule stress profile -completes an unchanged compile in just over seven seconds. - -See [Benchmarks](/benchmarks#rules-performance) for the complete workload, methodology, and results, -and [Execution and caching](/concepts/rules/execution-and-caching) for the Rules lifecycle. - -### Continue - - Select exact Rules or families and configure project-owned options. - Understand diagnostics, exact exceptions, path ignores, and stale checks. - Learn evaluation order, focused execution, and dependency-aware reuse. - List, inspect, run, and generate guidance for configured Rules. - -## Configuration and selection - -Source: `concepts/rules/configuration-and-selection.mdx` - -Select built-in and custom Rules by exact code or family prefix. - -Configure built-in and custom Rules together: - -```toml -[rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] -ignore = ["SQBRSQL004"] -``` - -An empty `select` disables configurable Rules. It does not disable mandatory compiler correctness. - -### Codes and families - -Built-in codes use `SQBR`, such as `SQBRSQL004`, `SQBRMODEL101`, and -`SQBRGRAPH101`. Custom codes use `XSQBR`, such as -`XSQBRARCH001`. - -A family is always the code with its final three digits removed: - -```text -SQBRSQL004 → SQBRSQL -XSQBRARCH001 → XSQBRARCH -XSQBR001 → XSQBR -``` - -Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching -semantics and takes precedence over `select`. - -Use the catalogue to inspect available codes and metadata: - -```bash -sqb rules list -sqb rules show SQBRSQL004 -``` - -### Rule options - -Custom Rules can declare typed options. Configure them by exact code: - -```toml -[rules.rule_options.XSQBRNAME001] -required_prefix = "order" -``` - -Unknown Rule codes, option names, and invalid option values fail configuration. - -### Focused selection - -Run one exact Rule or family while developing or adopting it: - -```bash -sqb rules run SQBRSQL004 -sqb rules run SQBRSQL -sqb rules run XSQBRARCH --select customer_orders -``` - -Focused execution constructs the compiler facts needed by the selected Rules. It does not replace -the complete configured enforcement performed by `sqb compile`. - - See every `sqb rules` command and exit code. - -## Findings and exceptions - -Source: `concepts/rules/findings-and-exceptions.mdx` - -Understand Rule diagnostics and record intentional exceptions safely. - -A Rule finding identifies an exact code, project-relative path, line, column, message, and -remediation. Error findings make compilation fail and prevent artifact completion. - -### Exact exceptions - -Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: - -```toml -[[rules.rule_exceptions]] -rule = "SQBRSQL004" -path = "models/examples/sample_orders.sql" -reason = "This example intentionally demonstrates one sampled row." -``` - -An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports -the stale exception so obsolete configuration does not accumulate silently. - -### Path- and resource-scoped ignores - -Use a path ignore when a documented project area intentionally follows a different convention: - -```toml -[[rules.rule_ignores]] -rules = ["SQBRSQL"] -paths = ["models/examples/**"] -reason = "Examples retain intentionally minimal SQL." -``` - -Path ignores accept exact codes and family prefixes. Keep their scope narrow and explain why the -project differs from the selected requirement. - -Use `selectors` when the exception follows graph-resource identities or lineage rather than files: - -```toml -[[rules.rule_ignores]] -rules = ["SQBRSQL021"] -selectors = ["intermediate_*"] -reason = "These intermediate interfaces intentionally preserve upstream columns." -``` - -Selectors use the same grammar as SQLBuild commands, including exact names, name globs, tags, -resource paths, and graph expansion such as `+intermediate_*`. Use `paths` for authored-file glob -matching, including SQL tests and audits that are not graph resources: - -```toml -[[rules.rule_ignores]] -rules = ["SQBRSQL021"] -paths = ["models/**/intermediate_*.sql"] -reason = "These intermediate SQL files intentionally preserve upstream columns." -``` - -`paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. - -### Mandatory correctness - -Mandatory compiler correctness is not configurable and cannot be suppressed. A project must first -compile into a trustworthy representation before any selected Rule can run. - -### Machine-readable findings - -Use JSON when another tool consumes findings: - -```bash -sqb compile --json -sqb rules --json run SQBRSQL -``` - -Machine-readable output remains on stdout. Lifecycle progress and terminal status are written to -stderr. - -## Execution and caching - -Source: `concepts/rules/execution-and-caching.mdx` - -Understand Rule enforcement order, focused runs, and dependency-aware reuse. - -SQLBuild evaluates a project in this order: - -1. discovery, expansion, and mandatory compiler correctness -2. selected native built-in Rules -3. selected custom Python Rules -4. artifact completion - -Native built-ins always run before custom Rules. Structured compile output reports built-in and -custom durations separately, together with cache hits and misses. - -### Authoritative enforcement - -`sqb compile` runs the complete configured Rules set. Build and execution commands use the same -compiler path and reject Rule findings before opening a warehouse connection. - -`sqb rules run` is intentionally focused. Use it to inspect one exact code or family, but run the -normal compiler before treating the complete project as valid. - -### Dependency-aware caching - -Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and -imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project -observations, and compatibility versions. - -Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model -subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for -genuinely project-wide invariants, with correspondingly broader invalidation. - -Positive and negative project-tree observations are dependencies. If a Rule observes that a glob -has no matches, adding a matching file invalidates that result. - -### Built-in-only execution - -When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every -custom result is cached, the host is not launched for evaluation. - -### Agent guidance - -Generate project-specific guidance from the configured Rules: - -```bash -sqb rules skills -sqb rules skills --check -``` - -SQLBuild manages `.agents/skills/sqlbuild-rules/SKILL.md`, -`.claude/skills/sqlbuild-rules/SKILL.md`, and `.opencode/skills/sqlbuild-rules/SKILL.md` without -overwriting divergent or unowned content. - -## Testing - -Source: `concepts/testing.mdx` - -SQL unit tests and multi-model tests with macro support, assertions, and model chaining. - -SQLBuild supports SQL-native unit tests that validate model logic by comparing actual query results against expected values. Tests can chain across multiple models (multi-model tests), use macros for reusable mock data, and include zero-row assertions. - -For end-to-end testing across many models with physical warehouse relations, see [Scenarios](/concepts/scenarios). - -### How tests work - -A test file defines mock inputs and expected outputs using CTEs. SQLBuild substitutes the mock data into the real model SQL, executes it, and compares the result against the expected CTE using `EXCEPT` queries. Zero mismatched rows means the test passes. - -```sql --- tests/unit/test_stg_orders.sql -TEST(); - -WITH -__source__raw__orders AS ( - SELECT - 1 AS id, - 100 AS customer_id, - 2 AS waffle_type_id, - 3 AS quantity, - '2026-04-01 10:00:00' AS ordered_at, - 'completed' AS status -), -__expected__stg_orders AS ( - SELECT - 1 AS order_id, - 100 AS customer_id, - 2 AS waffle_type_id, - 3 AS quantity, - '2026-04-01 10:00:00' AS ordered_at, - 'completed' AS status -) -SELECT 1 -``` - -The test: -1. Mocks the `raw__orders` source with the `__source__raw__orders` CTE -2. Runs the real `stg_orders` model SQL with the mock substituted in -3. Compares the output against `__expected__stg_orders` -4. Passes if row counts match and there are zero mismatched rows - -When an expected-output comparison fails, SQLBuild reports the unexpected and missing row counts -and best-effort samples from each direction. Samples are deliberately bounded to three rows, 12 -columns, and 120 characters per value, and pass through diagnostic redaction. They are also present -as structured `unexpected_samples` and `missing_samples` in JSON output. Sampling failure never -replaces the known test failure. - -When exactly one row exists in each difference direction and the bounded samples have the same -columns, SQLBuild also aligns the rows and reports only changed columns with their redacted actual -and expected values. It does not guess an alignment when several rows differ or sampling is -incomplete. - -Before opening a warehouse connection, SQLBuild statically checks fixture shapes when SQL analysis -is enabled. Missing columns are reported together with the test path, fixture resource, and models -that read them. Statically provable collection-versus-scalar type conflicts identify the affected -column and suggest explicit `CAST`, `ARRAY_CONSTRUCT`, or `PARSE_JSON` expressions. SQLBuild never -invents missing values. - -The trailing `SELECT 1` is required as a ceremonial closing statement. - -### CTE conventions - -| Prefix | Purpose | -|--------|---------| -| `__source__` | Mock a source. Replaces `__source("")` in the model SQL. | -| `__ref__` | Mock a model. Replaces `__ref("")` in the model SQL. | -| `__seed__` | Mock a seed. Replaces `__seed("")` in the model SQL. | -| `__expected__` | Define expected output for a model. SQLBuild resolves the model's real SQL and compares against this. | -| `__assert__` | Zero-row assertion. Passes if the query returns no rows; fails with the returned rows as diagnostics. | -| `__macro__` | Mock a macro. Replaces every `@(...)` call with the mock value. | - -Any CTE without one of these prefixes is treated as a helper CTE, available to all mock and model SQL in the test. - -### Multi-model tests - -Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL. - -```sql --- Mock two sources, assert on the final mart. --- stg_orders and stg_payments resolve automatically from their real SQL. -TEST(); - -WITH -__source__raw__orders AS ( - SELECT 1 AS id, 100 AS customer_id, 2 AS waffle_type_id, 3 AS quantity, - '2026-04-01 10:00:00' AS ordered_at, 'completed' AS status -), -__source__raw__payments AS ( - SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, - 'credit_card' AS method, '2026-04-01 10:01:00' AS paid_at, 'success' AS status -), -__expected__fact_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 1500 AS payment_amount_cents, - 'credit_card' AS payment_method -) -SELECT 1 -``` - -SQLBuild topologically sorts the expected models, resolves each intermediate model's real SQL with mocks substituted, and chains the outputs forward. Every model between the mocked sources and the expected model is computed automatically. - -Inspect that boundary without opening a warehouse connection: - -```bash -sqb test --select fact_orders --inspect -``` - -The resolved plan lists mocked sources, refs, seeds, real models in execution order, expected -models, and any unsatisfied leaf dependencies. A mocked ref is called out explicitly because its -real model SQL will not execute. Inspection exits non-zero when the plan contains unresolved -dependency errors. - -### Mocking refs and seeds - -You can mock models directly with `__ref__` and seeds with `__seed__`, not just sources. This skips the model's real SQL (or the seed's real CSV data) and provides controlled data instead: - -```sql -TEST(); - -WITH -__ref__stg_orders AS ( - SELECT - 1 AS order_id, - 100 AS customer_id, - 2 AS waffle_type_id, - 3 AS quantity, - CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, - 'completed' AS status -), -__ref__stg_payments AS ( - SELECT - 10 AS payment_id, - 1 AS order_id, - 2850 AS amount_cents, - 'card' AS payment_method, - CAST('2026-04-01 10:05:00' AS TIMESTAMP) AS paid_at, - 'success' AS payment_status -), -__seed__waffle_types AS ( - SELECT - 2 AS waffle_type_id, - 'Liege' AS waffle_name, - 'sweet' AS category, - 950 AS price_cents -), -__expected__fact_orders AS ( - SELECT - 1 AS order_id, - 100 AS customer_id, - 2 AS waffle_type_id, - 'Liege' AS waffle_name, - 'sweet' AS waffle_category, - 3 AS quantity, - 2850 AS line_total_cents, - CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, - 'completed' AS order_status, - TRUE AS is_completed_order, - 'card' AS payment_method, - 'success' AS payment_status, - 2850 AS payment_amount_cents -) -SELECT 1 -``` - -You can mix `__source__`, `__ref__`, and `__seed__` mocks in the same test. As long as every leaf dependency is satisfied (either by a source mock, a ref mock, a seed mock, or by being in the expected chain), the test resolves. - -### Multiple expected models - -A single test can assert on multiple models. SQLBuild resolves and compares each one independently: - -```sql -TEST(); - -WITH -__source__raw__orders AS ( - SELECT 1 AS id, 100 AS customer_id, 2 AS waffle_type_id, 3 AS quantity, - '2026-04-01 10:00:00' AS ordered_at, 'completed' AS status -), -__source__raw__payments AS ( - SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, - 'credit_card' AS method, '2026-04-01 10:01:00' AS paid_at, 'success' AS status -), -__source__raw__customers AS ( - SELECT 100 AS id, 'Leslie' AS first_name, 'Knope' AS last_name, - 'leslie@pawnee.gov' AS email, '2026-01-15 09:00:00' AS created_at -), -__expected__fact_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 1500 AS payment_amount_cents, - 'credit_card' AS payment_method -), -__expected__dim_customers AS ( - SELECT 100 AS customer_id, 1 AS total_orders, 1500 AS lifetime_spend_cents -) -SELECT 1 -``` - -If the expected models form a chain (e.g. `stg_orders` feeds into `fact_orders` which feeds into `dim_customers`), SQLBuild resolves them in dependency order, using the output of earlier steps as input to later ones. - -When a test defines `__expected__model_name` output, it may also use the public enums and constants -available to that model. A test that checks several models may use the public enums and constants -available to each of them. Public names are unique, so those values cannot conflict. - -Only explicit expected CTEs create grants. A matching test filename, `__ref__` mock, or nearby model path does not. Model-private declarations and macros are never granted through expected models. - -### Macro-powered mocks - -Because unit tests are written in SQL, they support macro calls. This lets you write reusable mock generators instead of copy-pasting mock data across test files: - -```sql -TEST(); - -WITH -__source__raw__orders AS ( - @mock_orders() -), -__source__raw__payments AS ( - SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, 'credit_card' AS method -), -__expected__fact_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 1500 AS total_cents, - 'credit_card' AS payment_method -) -SELECT 1 -``` - -The `@mock_orders()` call expands at compile time to whatever SQL the Python macro function returns. - -Test SQL uses macros, enums, and constants available from the test file's directory under -`tests/unit/`. Public enums and constants available to a model are also available when the test -defines `__expected__model_name` output for that model. Model-private values are not available to -tests. See [How Visibility Works](/concepts/declaration-scopes/visibility#tests-and-expected-output). - -### Macro mocking - -When a model uses macros that you want to control in tests (e.g. target-specific logic, dynamic SQL generation), you can override their output with `__macro__` CTEs: - -```sql -TEST(); - -WITH -__macro__country_filter AS ( - SELECT 'country_code = ''US''' -), -__source__raw__orders AS ( - SELECT 1 AS id, 100 AS customer_id, 'US' AS country_code, 'completed' AS status -), -__expected__stg_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status -) -SELECT 1 -``` - -When a `__macro__` mock is defined, every call to `@country_filter(...)` in any model SQL resolved by the test is replaced with the mock value (`country_code = 'US'`). The macro's actual Python function is not called, and the arguments are ignored. - -The mock value must be a single `SELECT` with one string literal. Use doubled single quotes for quotes within the value (standard SQL escaping). - -This is useful for: -- Testing models that use target-specific macros without depending on target config -- Controlling dynamic SQL generation to produce predictable test inputs -- Isolating model logic from macro implementation details - -### Assertions - -Unit tests can include `__assert__` CTEs for property-based checks. An assertion passes if the query returns zero rows - any returned rows are failing examples. - -```sql -TEST(); - -WITH -__ref__stg_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 3 AS quantity, - CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, 'completed' AS status -), -__assert__order_ids_are_not_null AS ( - SELECT * FROM __ref("stg_orders") WHERE order_id IS NULL -) -SELECT 1 -``` - -Assertions can be mixed with `__expected__` CTEs in the same test, or used on their own. They are useful when the natural check is "no rows should violate this rule" rather than "the output should exactly equal these rows" - for example, duplicate checks, negative-value constraints, or conditional business rules. - -Expected results and assertions are independent checks. They may share fixture or helper CTEs, but an `__expected__*` CTE must not depend on an `__assert__*` CTE, and an `__assert__*` CTE must not depend on an `__expected__*` CTE, directly or through helpers. - -During `sqb test` and `sqb build`, assertion results appear as nested check rows alongside expected comparisons. - -### Test modes - -By default, `TEST()` runs in model mode - mocking sources/refs and comparing model outputs. Three additional modes let you test reusable logic directly without needing a model chain. - -#### Macro tests - -Test macro output by calling the macro in `__macro_actual__` and comparing against `__macro_expected__`: - -```sql -TEST (mode macro, name "calculates_line_total_cents"); - -WITH -input_values AS ( - SELECT 950 AS price_cents, 3 AS quantity -), -__macro_actual__ AS ( - SELECT @line_total_cents("price_cents", "quantity") AS line_total_cents - FROM input_values -), -__macro_expected__ AS ( - SELECT 2850 AS line_total_cents -) -SELECT 1 -``` - -Macros are compile-time code, so macro tests expand the macro at compile time and compare the results. During `sqb build`, macro tests run before any model that uses the tested macro. - -#### UDF tests - -Test scalar UDFs by calling them in `__udf_actual__` and comparing against `__udf_expected__`: - -```sql -TEST (mode udf, name "detects_completed_orders"); - -WITH -input_values AS ( - SELECT 'completed' AS order_status - UNION ALL - SELECT 'pending' AS order_status -), -__udf_actual__ AS ( - SELECT - order_status, - __udf("udf__is_completed_order")(order_status) AS is_completed_order - FROM input_values -), -__udf_expected__ AS ( - SELECT 'completed' AS order_status, TRUE AS is_completed_order - UNION ALL - SELECT 'pending' AS order_status, FALSE AS is_completed_order -) -SELECT 1 -``` - -UDFs are warehouse objects, so the function is created before the test runs. During `sqb build`, UDF tests run after the function is created but before any model that uses it. - -#### Table function tests - -Test table functions by calling them in `__table_fn_actual__` and comparing against `__table_fn_expected__`: - -```sql -TEST (mode table_fn, name "returns_customer_orders"); - -WITH -__table_fn_actual__ AS ( - SELECT order_id, order_status, is_completed_order - FROM __table_fn("table_fn__customer_orders")(1) -), -__table_fn_expected__ AS ( - SELECT 1 AS order_id, 'completed' AS order_status, TRUE AS is_completed_order - UNION ALL - SELECT 2 AS order_id, 'completed' AS order_status, TRUE AS is_completed_order -) -SELECT 1 -``` - -Table function tests run after the function is created. Since table functions are terminal (models cannot depend on them), these tests validate the function independently. - -#### Mode rules - -Each mode has strict CTE validation: - -| Mode | Actual CTE | Expected CTE | Allowed | -|------|-----------|--------------|---------| -| `model` (default) | existing model-chain syntax | `__expected__` | `__source__`, `__ref__`, `__seed__`, `__assert__`, `__macro__` | -| `macro` | `__macro_actual__` | `__macro_expected__` | Helper CTEs, `@macro()` calls in actual | -| `udf` | `__udf_actual__` | `__udf_expected__` | Helper CTEs, `__udf()` calls in actual | -| `table_fn` | `__table_fn_actual__` | `__table_fn_expected__` | Helper CTEs, `__table_fn()` calls in actual | - -CTE prefixes from other modes are not allowed. For example, `__source__` in a macro test or `__macro_actual__` in a model test will produce a clear error pointing you to the right mode. Expected CTEs must not call macros, UDFs, or table functions - they should be independent, inspectable expected data. - -### Multiple tests per file - -A single test file can contain multiple `TEST()` blocks. Each block must have a unique `name`: - -```sql -TEST (name "completed_orders_only"); - -WITH -__source__raw__orders AS ( - SELECT 1 AS id, 100 AS customer_id, 'completed' AS status -), -__expected__stg_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status -) -SELECT 1 - -TEST (name "cancelled_orders_excluded"); - -WITH -__source__raw__orders AS ( - SELECT 1 AS id, 100 AS customer_id, 'cancelled' AS status -), -__expected__stg_orders AS ( - SELECT 1 AS order_id, 100 AS customer_id, 'cancelled' AS status -) -SELECT 1 -``` - -A file with a single test can omit the `name` field. Files with multiple tests require names on every block. - -### Repeating test logic - -SQLBuild supports three repetition patterns: - -| Pattern | Use when | Result identity | -|---------|----------|-----------------| -| Multiple `TEST` blocks | Cases need different SQL shapes | One result per block | -| Macro or `VALUES` case table | One aggregate comparison is sufficient | One result for the table | -| Native parameters and cases | Cases share a template but need independent status | One result per named case | - -#### Native independent cases - -Declare a typed schema and ordered named cases in one header: - -```sql -TEST ( - name "order_status_maps_source_states", - parameters ( - source_status string, - expected_status string, - ), - cases ( - completed (source_status "completed", expected_status "completed"), - cancelled (source_status "cancelled", expected_status "excluded"), - pending (source_status "pending", expected_status "open"), - ), -); - -WITH -__source__raw__orders AS ( - SELECT 1 AS id, @param("source_status") AS status -), -__expected__stg_orders AS ( - SELECT 1 AS id, @param("expected_status") AS status -) -SELECT 1 -``` - -The cases report independently as `order status: maps source states [completed]`, `[cancelled]`, -and `[pending]`. Authored order controls display order. Selecting `stg_orders` selects every case by -default. Add `--case` to run one named case without changing the test file: - -```bash -sqb test --select stg_orders --case cancelled -``` - -If the case does not exist within the parent/model selection, SQLBuild reports the available case -names. - -Supported scalar types are `string`, `integer`, `boolean`, `float`, and exact `decimal`. Decimal -values are quoted, such as `tax_rate "0.2000"`, so they never pass through binary float. Boolean -and integer remain distinct. Raw SQL and collection parameters are rejected. - -Nullable parameters use an expanded declaration and are valid where SQL can infer the null type: - -```sql -parameters ( - cancellation_reason (type string, nullable true), -), -cases ( - completed (cancellation_reason null), -) -``` - -Every case provides exactly the declared parameters, and every declaration must appear as -`@param("name")` in the template. SQLBuild rejects missing, extra, incompatible, undeclared, and -unused values before execution. - -Expansion is deterministic: - -```text -parse TEST template and cases - -> render typed @param values with the active adapter - -> expand @const and @enum references - -> expand macros - -> validate SQL and compile expected models/assertions -``` - -Values can therefore feed macro arguments, for example -`@order_fixture(@param("source_status"))`. Parameters work in model, macro, UDF, and table-function -modes and preserve every expected model in multi-model tests. - -Text and JSON output include parent/case identity and safe typed values. Compile JSON, manifests, -DAG checks, and compiled/runtime SQL artifacts retain source, block, case, and content-fingerprint -provenance. Changing one case changes that case's fingerprint without reassigning another case's -stable identity. - -Scenarios are intentionally not parameterized. Keep one scenario per coherent business world so -capture and replay identity remains explicit. - -#### Aggregate SQL case tables - -When independent status is unnecessary, a normal SQL case table remains concise: - -```sql -WITH cases(case_name, source_status, expected_status) AS ( - VALUES - ('completed', 'completed', 'completed'), - ('cancelled', 'cancelled', 'excluded') -), -__expected__status_mapping AS ( - SELECT case_name, expected_status FROM cases -) -SELECT 1 -``` - -This produces one aggregate test result. Use native cases when each row must pass or fail -independently. - -Quoted strings in `VALUES` rows may contain commas, brackets, and parentheses. SQLBuild preserves -these literals while extracting test CTEs; use normal SQL quote escaping for embedded quotes. - -### Test placement - -Place unit test files under `tests/unit/` in your project directory. SQLBuild discovers all `.sql` files in this directory recursively. - -``` -tests/ - unit/ - test_stg_orders.sql # model test - test_fact_orders.sql # model test with assertion - test_daily_revenue_chain.sql # multi-model test - test_line_total_cents_macro.sql # macro test - test_is_completed_order_udf.sql # UDF test - test_customer_orders_table_fn.sql # table function test - scenarios/ - ... -``` - -### Running tests - -Tests run automatically during `sqb build` in DAG order, before their target model is materialized. - -Run tests standalone: - -```bash -sqb test -``` - -Scope to specific models: - -```bash -sqb test --select stg_orders -``` - -## Scenarios - -Source: `concepts/scenarios.mdx` - -End-to-end tests that build real project graphs against coherent fixture data. - -Scenarios are end-to-end tests for coherent slices of your warehouse. You define fixture inputs, SQLBuild builds the real project graph against them in isolated physical relations, and you assert that the result is correct. - -### When to use scenarios vs unit tests - -[SQL unit tests](/concepts/testing) compile to a single comparison query. They are fast, inline, and work well for checking individual model transformations. But when you need to test across many models with related entities - customers that have orders, orders that have payments, refunds that reference original orders - a single query can become unwieldy or hit SQL size limits. - -Scenarios materialize physical relations in the warehouse and build the actual model graph. This means: - -- Fixture inputs are real tables, not CTEs inside one query -- Models execute with real SQL, real materializations, and real dependency ordering -- You can test end-to-end business logic across an entire pipeline slice -- Debugging is easier because you can inspect intermediate tables with `--retain` - -Use unit tests for transformation logic. Use scenarios for coherent end-to-end validation. - -### Authoring scenarios - -Scenario files live under `tests/scenarios/` and use one file per scenario: - -``` -tests/ - scenarios/ - revenue/ - daily_revenue_minimal.sql - daily_revenue_multi_order.sql - fulfillment/ - fulfillment__late_shipment.sql -``` - -Folders are organizational only. The scenario name is the filename stem and must be globally unique across all discovered scenario files. - -#### Scenario format - -Each file has a `SCENARIO()` header followed by CTEs that define inputs, expected outputs, and assertions: - -```sql -SCENARIO ( - description "Daily revenue includes only successful payments", - tags ["revenue", "example"] -); - -WITH -__ref__stg_orders AS ( - SELECT - 1 AS order_id, 10 AS customer_id, 1 AS waffle_type_id, - 2 AS quantity, CAST('2026-04-01 09:15:00' AS TIMESTAMP) AS ordered_at, - 'completed' AS status -), - -__ref__stg_payments AS ( - SELECT - 1 AS payment_id, 1 AS order_id, 1700 AS amount_cents, - 'credit_card' AS payment_method, - CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, - 'success' AS payment_status - UNION ALL - SELECT - 2 AS payment_id, 2 AS order_id, 1050 AS amount_cents, - 'credit_card' AS payment_method, - CAST('2026-04-01 10:01:00' AS TIMESTAMP) AS paid_at, - 'failed' AS payment_status -), - -__expected__daily_revenue AS ( - SELECT - CAST('2026-04-01' AS DATE) AS revenue_date, - 1 AS order_count, 2 AS waffles_sold, - 1700 AS total_revenue_cents -), - -__assert__all_orders_have_payments AS ( - SELECT * - FROM __ref("fact_orders") - WHERE payment_amount_cents IS NULL -) - -SELECT 1 -``` - -#### CTE conventions - -| Prefix | Purpose | -|--------|---------| -| `__source__` | Fixture source input. Materializes as a scenario table. | -| `__ref__` | Fixture model boundary. Prevents SQLBuild from building that model upstream. | -| `__seed__` | Fixture seed data. | -| `__expected__` | Full expected output. Compared order-insensitively against the scenario-built model. | -| `__assert__` | Zero-row assertion. Passes if the query returns no rows; fails with the returned rows as diagnostics. | - -For sources with two-part identity, use double underscores: `__source__raw__orders`. - -Every scenario must have at least one fixture CTE and at least one `__expected__` or `__assert__` CTE. - -Expected results and assertions are independent checks. They may share fixture or helper CTEs, but neither check kind may depend on the other directly or through helpers. - -Scenario SQL uses macros, enums, and constants available from the scenario file's directory under -`tests/scenarios/`. Public enums and constants available to a model are also available when the -scenario defines `__expected__model_name` output for that model. Model-private values and macros -available only to the model are not included. See -[How Visibility Works](/concepts/declaration-scopes/visibility#tests-and-expected-output). - -#### SCENARIO() header - -The header is metadata only: - -| Field | Description | -|-------|-------------| -| `description` | Optional description string | -| `tags` | Optional list of string tags | - -### How scenarios execute - -#### Graph inference - -SQLBuild infers which models to build from your scenario: - -1. Target models come from `__expected__` CTE names and `__ref(...)` calls inside `__assert__` CTEs -2. SQLBuild walks upstream from targets, building every required model -3. `__ref__` fixtures act as boundaries - upstream traversal stops there -4. All required sources must be provided by `__source__` fixtures -5. Required project seeds are loaded automatically unless overridden by `__seed__` fixtures - -#### Isolation - -Scenario artifacts are physically isolated from production: - -- Every scenario-owned relation uses a deterministic prefixed name (based on a hash of the project and scenario identity) -- All `ref()`, `source()`, and `seed()` calls resolve to scenario-owned relations, never production tables -- Fixture CTEs can read from existing warehouse relations if you choose, but downstream models always read the materialized scenario fixture, not the original - -#### Execution flow - -1. Clean any existing scenario artifacts from a previous run -2. Materialize source, ref, and seed fixtures as physical tables -3. Load required project seeds -4. Build required models in dependency order (incremental models run as full-refresh in scenarios) -5. Run expected-output comparisons (order-insensitive) -6. Run zero-row assertions -7. Clean up all scenario-owned artifacts (unless `--retain`) - -#### Inspecting with `--retain` - -When a scenario fails or you want to inspect intermediate state: - -```bash -sqb scenario test --select daily_revenue_minimal --retain -``` - -This keeps all scenario-owned relations in the warehouse and prints a relation map showing the logical-to-physical name mapping. You can then query the scenario tables directly to debug. - -Runtime artifacts (fixture SQL, model lifecycle SQL, check SQL, cleanup SQL) are always written to `target/run/scenarios//` regardless of `--retain`. - -### Local scenario testing - -Scenarios can run locally against DuckDB using captured JSONL snapshots - no warehouse connection needed. This is useful for CI pipelines and fast developer iteration. - -#### Capture - -First, capture scenario inputs from the real warehouse: - -```bash -sqb scenario capture --select daily_revenue_minimal -``` - -This: - -1. Materializes scenario input fixtures in the warehouse -2. Runs a preflight `COUNT(*)` on each fixture to check row counts against safety limits -3. Inspects column types and maps them to DuckDB-compatible types -4. Downloads rows as JSONL files (enforcing byte limits during writing) -5. Writes a `scenario.json` manifest with column metadata, types, row counts, and an input fingerprint -6. Cleans up warehouse artifacts - -Snapshots are written to `tests/_scenario_snapshots//` and can be committed to version control. JSONL is human-readable and git-diffable. - -#### Capture safety limits - -Capture enforces row and byte limits to prevent accidentally saving large datasets. Limits can be set in `sqlbuild_project.toml`: - -```toml -[scenario.snapshot_limits] -max_rows_per_relation = 10000 -max_total_rows = 50000 -max_bytes_per_relation = 10485760 # 10 MB -max_total_bytes = 52428800 # 50 MB -``` - -Or per-command via CLI flags: - -```bash -sqb scenario capture --max-snapshot-rows 5000 --max-snapshot-total-rows 20000 -sqb scenario capture --max-snapshot-bytes 5242880 -``` - -CLI flags override TOML config. Use `--force` to bypass all limits: - -```bash -sqb scenario capture --force -``` - -When a limit is exceeded, capture fails with a clear error and suggests narrowing the fixture query, raising the limit, or using `--force`. - -#### Local replay - -Run scenarios locally against DuckDB: - -```bash -sqb scenario test --local -``` - -This: - -1. Checks snapshot freshness via the input fingerprint (missing/stale snapshots are skipped by default) -2. Creates a temporary DuckDB database at `target/run/scenarios//local.duckdb` -3. Loads JSONL snapshots into typed DuckDB tables using column metadata from `scenario.json` -4. Transpiles model and check SQL from the project adapter dialect to DuckDB -5. Builds functions, models, and runs expected/assertion checks in DuckDB -6. Keeps the local DuckDB file for inspection (it lives under `target/`, so it's always retained) - -#### Sync snapshots - -Instead of running capture and test separately, sync snapshots in one command: - -```bash -# Capture missing/stale snapshots, then run locally -sqb scenario test --local --sync-snapshots - -# Recapture all snapshots, then run locally -sqb scenario test --local --refresh -``` - -With `--sync-snapshots`, fresh snapshots are reused. With `--refresh`, all selected snapshots are recaptured even if fresh. - -#### Strict mode - -By default, missing or stale snapshots are skipped with a warning. Use `--strict` to treat them as errors: - -```bash -sqb scenario test --local --strict -``` - -#### Local type overrides - -When the automatic warehouse-to-DuckDB type conversion produces an incompatible type, you can override it in `sqlbuild_project.toml`: - -```toml -[scenario.local_type_overrides.snowflake] -"OBJECT" = "JSON" -"ARRAY" = "JSON" -"NUMBER(*,0)" = "BIGINT" -``` - -Override keys are structural type patterns. Values can reference matched type arguments with `{1}`, `{2}`, etc: - -```toml -[scenario.local_type_overrides.bigquery] -"NUMERIC({1},{2})" = "DECIMAL({1},{2})" -``` - -These overrides are written into `scenario.json` during capture and used when loading snapshots into DuckDB. - -### Cleanup - -By default, remote scenario artifacts are dropped after each run (pass or fail). The janitor command also recognizes scenario artifacts and can clean retained or orphaned scenario relations: - -```bash -sqb janitor -``` - -See the [CLI reference](/cli/scenario) for full command documentation. - -### Limitations - -- Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. -- Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. - -## Selectors - -Source: `concepts/selectors.mdx` - -Target specific models, paths, tags, or DAG subsets with select and exclude flags. - -Selectors let you scope commands to specific subsets of your project. They work with `plan`, `build`, `test`, `audit`, `seed`, `clone`, and `diff`. - -### Basic usage - -```bash -sqb build --select daily_revenue -sqb build --select daily_revenue customer_status_snapshot -sqb build --exclude stg_customers -``` - -`--select` (or `-s` for short) accepts one or more names. Multiple values are unioned. Space-separated names within one `--select` are also unioned. `--exclude` subtracts from the selected set. - -When no `--select` is provided, all models are selected. - -### Selector types - -#### Name - -Select a single resource by name: - -```bash -sqb build --select fact_orders -``` - -Bare names accept glob patterns. This selects every resource whose name starts with -`intermediate_`: - -```bash -sqb build --select "intermediate_*" -``` - -Name globs compose with graph expansion. For example, `+intermediate_*` selects every matching -resource and all of their upstream dependencies. Quote patterns in shell commands so your shell -does not expand `*` against files in the current directory. - -#### Tag - -Select all models with a specific tag: - -```bash -sqb build --select tag:staging -sqb build --select tag:acceptance -``` - -#### Path - -Select all models under a directory path: - -```bash -sqb build --select path:models/marts -sqb build --select models/marts -sqb build --select path:models/intermediate -``` - -Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/`, `tasks/`, `assets/`, `checks/`, or `loaders/`. Nested paths work too: `models/staging/orders`. - -#### Seed and source - -```bash -sqb build --select seed:waffle_types -sqb build --select source:raw__orders -``` - -### Graph expansion - -#### Upstream - -Select a model plus all its upstream dependencies: - -```bash -sqb build --select +daily_activity_rollup -``` - -#### Downstream - -Select a model plus all its downstream dependents: - -```bash -sqb build --select fact_orders+ -``` - -#### Bidirectional - -```bash -sqb build --select +fact_orders+ -``` - -Graph expansion works with all selector types: - -```bash -sqb build --select +tag:marts -sqb build --select path:models/staging+ -``` - -### Path-between selectors - -Select all models on the shortest path between two nodes: - -```bash -sqb build --select fact_orders~daily_activity_rollup -``` - -With endpoint expansion: - -```bash -sqb build --select +fact_orders~daily_activity_rollup+ -``` - -This selects: -- All upstreams of `fact_orders` -- Every model on the path between `fact_orders` and `daily_activity_rollup` -- All downstreams of `daily_activity_rollup` - -This is useful for rebuilding a specific slice of the DAG without manually listing every model in between. - -### Intersection - -Use commas to intersect selector results: - -```bash -sqb build --select "tag:staging,path:models/finance" -``` - -This selects only models that match *both* conditions - in this case, models tagged `staging` that are also under the `models/finance` directory. - -### Combining select and exclude - -```bash -# Build all marts except daily_revenue -sqb build --select path:models/marts --exclude daily_revenue - -# Build everything upstream of fact_orders, excluding staging models -sqb build --select +fact_orders --exclude tag:staging -``` - -### Error handling - -Unknown resource names, name patterns with no matches, empty paths, and malformed selectors produce -clear error messages: - -``` -unknown selector name 'nonexistent_model' -unknown selector pattern 'missing_*' -no models found under path 'models/nonexistent'. -no models found with tag 'nonexistent_tag' -path selector 'fact_orders~' requires names on both sides of '~' -``` - -If a path selector omits the root directory, SQLBuild asks for the explicit form: - -``` -path selectors require an explicit root: use 'models/', 'tasks/', 'assets/', 'checks/', or 'loaders/' -``` - -## Column Lineage - -Source: `concepts/column-lineage.mdx` - -Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. - -### Why column lineage matters - -**Impact analysis** - Before changing a source column, see exactly which downstream models and columns are affected. A rename or type change in `raw__orders.id` can be traced through every model that consumes it, even indirectly. - -**Debugging data issues** - When a column has unexpected values, trace it upstream to find where the data originates and what transformations it passes through. Instead of reading SQL files and mentally joining dependencies, ask SQLBuild to show the path. - -**Documentation** - Column lineage provides machine-readable metadata about your pipeline. The JSON output can feed data catalogs, governance tools, or custom dashboards. - -### How it works - -SQLBuild analyzes column lineage statically at compile time. No warehouse connection is needed. The analyzer parses each model's SQL, resolves `ref()` and `source()` calls, and traces columns through `SELECT` lists, CTEs, JOINs, subqueries, and expressions. - -Column lineage requires SQL analysis to be enabled in project settings (it is by default). - -### Transform types - -Each lineage edge is classified by how the column is transformed: - -| Transform | Description | Example | -|-----------|-------------|---------| -| `direct` | Column passes through unchanged | `SELECT order_id FROM ...` | -| `cast` | Column is explicitly cast to a different type | `SELECT CAST(id AS BIGINT)` | -| `expression` | Column is used in a computed expression | `SELECT amount * 100 AS amount_cents` | -| `aggregation` | Column is used inside an aggregate function | `SELECT SUM(amount) AS total` | -| `star` | Column is included via `SELECT *` | `SELECT * FROM ...` | -| `constant` | Output column is a literal value with no upstream dependency | `SELECT 'active' AS status` | - -Transform classification helps you understand the nature of each dependency. A `direct` edge means the column is a simple passthrough - safe to rename if you rename the source. An `expression` or `aggregation` edge means the column is computed - the upstream value is an input to a calculation, not a 1:1 mapping. - -### Confidence levels - -Each edge also carries a confidence level indicating how certain the analyzer is about the traced dependency: - -| Confidence | Meaning | -|------------|---------| -| `high` | The lineage path is fully resolved through known SQL constructs | -| `medium` | The path is likely correct but involves constructs the analyzer handles with heuristics | -| `low` | The path is best-effort - complex SQL patterns or unsupported constructs may reduce accuracy | - -### Analysis modes - -Column lineage supports two analysis modes that trade off speed against depth of analysis. - -**Rich mode** uses the SQL analysis optimizer to resolve columns through CTEs, subqueries, and multi-level nesting with full transform classification. Thorough, but slower because the optimizer runs per column per model. - -**Fast mode** parses the SQL AST directly to extract column mappings, resolve CTE references, and classify transforms. It handles the same SQL patterns that most column lineage tools support and is fast enough to run on every compile. - -`sqb compile` defaults to fast mode because it runs frequently and analyzes the entire project. `sqb lineage` defaults to rich mode because it targets a specific column in a scoped slice of the DAG, where the deeper analysis is worth the cost. Both are overridable: - -```bash -sqb compile --lineage-mode rich -sqb lineage fact_orders.payment_amount_cents --mode fast -``` - -### Using column lineage - -#### Interactive tracing with `sqb lineage` - -The target syntax is `model_name.column_name`. Trace a column upstream to see where its values come from: - -```bash -sqb lineage daily_revenue.total_revenue_cents -``` - -``` -Column trace daily_revenue.total_revenue_cents upstream - -└── stg_payments.amount_cents (aggregation) - └── raw__payments.amount_cents (direct) -``` - -Each hop is annotated with how the value was derived. Here `total_revenue_cents` is an aggregation of `stg_payments.amount_cents`, which is a direct passthrough from the source. - -Use `--direction downstream` to trace the other way - every column derived from this one: - -```bash -sqb lineage stg_payments.amount_cents --direction downstream -``` - -``` -Column trace stg_payments.amount_cents downstream - -├── daily_revenue.avg_order_value_cents (aggregation) -├── daily_revenue.total_revenue_cents (aggregation) -├── daily_revenue.total_revenue_dollars (aggregation) -├── dim_customers.lifetime_spend_cents (aggregation) -└── fact_orders.payment_amount_cents (direct) -``` - -Column lineage supports `upstream` (default) and `downstream` directions (not `both` - model lineage supports `both`). - -#### Model lineage - -`sqb lineage` also traces model-level dependencies when the target has no column (no dot): - -```bash -sqb lineage fact_orders -``` - -``` -Lineage model fact_orders models/marts/fact_orders.sql upstream - -├── model stg_orders models/staging/stg_orders.sql -│ └── source raw__orders sources/raw.yml -├── model stg_payments models/staging/stg_payments.sql -│ └── source raw__payments sources/raw.yml -├── seed waffle_types seeds/waffle_types.csv -└── udf udf__is_completed_order -``` - -Each node is tagged with its resource type (`model`, `source`, `seed`, `udf`) and file path. Use `--direction both` to show upstream and downstream together: - -```bash -sqb lineage daily_revenue --direction both -``` - -``` -Lineage model daily_revenue models/marts/daily_revenue.sql both - -upstream -├── model stg_orders models/staging/stg_orders.sql -│ └── source raw__orders sources/raw.yml -└── model stg_payments models/staging/stg_payments.sql - └── source raw__payments sources/raw.yml -downstream -``` - -#### Options - -| Flag | Description | -| --------------- | --------------------------------------------------------------------------------- | -| `--direction` | `upstream` (default), `downstream`, or `both`. `both` is model lineage only. | -| `--depth` | How many hops to traverse: an integer or `all` (default `all`). | -| `--format` | `tree` (default), `list` (an edge list of `a -> b` pairs), or `json`. | -| `--mode` | Column lineage mode: `rich` (default) or `fast`. | - -See the [lineage CLI reference](/cli/lineage) for full flag documentation and output format examples. - -#### Batch analysis with `sqb compile` - -Every compile run computes column lineage for analysis and reports a summary: - -```bash -# Default: fast column lineage -sqb compile - -# Skip column lineage -sqb compile --lineage-mode none - -# JSON report includes per-model lineage summary -sqb compile --json -``` - -In the JSON compile report, each model includes a `lineage` field with `column_count`, `edge_count`, and `has_star` metadata. It does not contain the full edge graph; use `sqb lineage [.] --format json` when you need structured lineage details. - -See the [compile CLI reference](/cli/compile) for details on the compile report format. - -#### Integration with contract validation - -Column lineage feeds into compile-time contract validation. When a model declares columns in its `MODEL()` header, the compiler uses inferred column information to check that: - -- Every declared column exists in the query output -- Column types match the declared types (when `type_enforcement` is enabled) - -These checks run automatically during `sqb compile` and report diagnostics with source-annotated error messages. - -### Limitations - -- Column lineage requires SQL analysis to be enabled (`sql_analysis = true` in settings, which is the default) -- Complex SQL patterns (deeply nested correlated subqueries, dynamic SQL, adapter-specific functions) may reduce accuracy or confidence -- `SELECT *` is tracked as a `star` transform - the analyzer knows the column passes through but the mapping is less precise than explicit column references -- Column lineage is computed statically from SQL text. Runtime-only column additions (e.g. from dynamic UDFs) are not tracked - -## Data Diffs - -Source: `concepts/diff.mdx` - -Compare schemas and data between targets or virtual environments to validate changes before promotion. - -SQLBuild can compare schemas and row-level data between two build contexts. This lets you validate that changes produce the expected results before promoting them. - -`sqb diff` supports two input modes: - -- **model mode:** `sqb diff FROM:TO --select ...` compares two targets (for example `prod:dev`) in - direct mode, or two virtual environments when virtual environments are enabled; -- **raw-query mode:** inline SQL or SQL files identify the two sides and execute through the active - target connection or an explicit `--target`. - -The mechanics below are identical for both; only what `FROM` and `TO` refer to changes. - -In direct mode, `FROM` and `TO` resolve the authoritative database/schema namespaces to compare. -SQLBuild uses the `TO` target's named connection for the complete comparison and accesses both -namespaces through fully qualified relations. It does not resolve or open the `FROM` target's -connection. The `TO` connection must therefore be able to read both namespaces; missing credentials -for that execution connection still fail before warehouse inspection. - -```bash -sqb diff prod:dev --full --select customer_status_snapshot -``` - -Raw-query mode omits `FROM:TO` and requires repeated `--key` values or explicit `--unkeyed` for a -row comparison. Inputs must each be one literal read-only `SELECT`, `WITH`, or `VALUES` expression; -SQLBuild templates such as `ref()` are not expanded. - -```bash -sqb diff \ - --left-query 'SELECT order_id, amount_cents FROM analytics.orders_before' \ - --right-query 'SELECT order_id, amount_cents FROM analytics.orders_after' \ - --left-label before \ - --right-label after \ - --key order_id - -sqb diff \ - --left-query-file queries/orders_before.sql \ - --right-query-file queries/orders_after.sql \ - --target dev \ - --key account_id \ - --key order_id -``` - -Query mode defaults to a full comparison and also supports `--schema-only`. It does not accept -`--bounded`; put the desired filter in both SQL inputs instead. - -Raw-query mode loads configuration and the selected adapter without importing unrelated project -sinks, hooks, loaders, or providers. - -### Comparison modes - -Model mode requires exactly one mode. Raw-query mode defaults to full when no mode is supplied: - -#### Full diff - -Compares both schema and row-level data for the selected models: - -```bash -sqb diff prod:dev --full --select fact_orders -``` - -Rows are joined on the model's `unique_key` and compared column by column. The output shows: -- Row counts for each side -- How many rows are equal, unequal, or only in one side -- Which columns have mismatches with match percentages -- Example values showing what changed - -#### Schema-only diff - -Compares column names and types without looking at row data: - -```bash -sqb diff prod:dev --schema-only --select fact_orders -``` - -Useful for quick structural checks or when row comparison would be too expensive. - -#### Bounded diff - -Compares only a recent window of data using the model's cursor: - -```bash -sqb diff prod:dev --bounded 14d --select hourly_order_activity -``` - -For timestamp cursors, the bound is a duration (`14d`, `6h`, `30m`). For integer cursors, the bound is an integer value. If the model has no cursor configured, the diff falls back to a full row comparison. - -### Deterministic key sampling - -A cursor bound limits the range of data but does not guarantee a predictable number of rows. Use -deterministic key sampling to cap the wide value comparison for a high-volume model: - -```toml -[defaults] -row_diff_sample_rows = 100000 -row_diff_sample_seed = 0 - -[path_defaults.intermediate] -row_diff_sample_rows = 25000 -``` - -The settings use the normal configuration precedence: project `defaults`, matching -`path_defaults`, the model's `MODEL()` header, and finally CLI overrides. A model can disable an -inherited sample and request exhaustive comparison with `row_diff_sample_rows 0`: - -```sql -MODEL ( - materialized table, - unique_key [order_id, line_id], - row_diff_sample_rows 0, -); -``` - -Override the effective policy for one invocation with `--sample-rows` and `--sample-seed`, or force -an exhaustive comparison with `--exhaustive`: - -```bash -sqb diff prod:dev --bounded 14d --sample-rows 50000 --sample-seed 7 --select order_lines -sqb diff prod:dev --bounded 14d --exhaustive --select order_lines -``` - -SQLBuild applies the complete cursor bound first. It then selects the lowest deterministic hashes -from the union of unique keys found on either side and uses that same key set for both relations. -This avoids false side-only rows caused by independently sampling each side. Composite keys are -encoded with component lengths before hashing, and key columns break hash ties deterministically. - -Schema comparison, bounded row counts, cursor minimum/maximum values, and null/duplicate key checks -remain exhaustive. Only the wide column-by-column comparison is sampled. If the bounded union has -no more keys than the configured limit, SQLBuild reports the comparison as exhaustive. - -Sampled success means that no differences were found in the sampled keys; it is never reported as -complete table equality. The output shows the bounded key population, compared key count, seed, and -percentage evaluated. - -### Cursor coverage - -Before comparing values, SQLBuild reports exact bounded `COUNT`, `MIN(cursor)`, and `MAX(cursor)` -for each side. If cursor extents differ, it warns and continues with the requested comparison. It -does not ask for confirmation, silently narrow to the overlap, or hide rows that exist on only one -side. - -### Row matching - -Rows are matched using the model's `unique_key` or repeated invocation-level `--key` overrides. -For inputs without stable row identity, `--unkeyed` performs an exhaustive exact multiset -comparison in which duplicate rows retain their multiplicity. SQLBuild never guesses a key. - -The diff output categorises rows as: -- **Equal** - same key, same values on both sides -- **Unequal** - same key, different values (with per-column breakdown) -- **Left only** - exists in the FROM side but not TO -- **Right only** - exists in the TO side but not FROM - -### Tolerances - -Numeric columns can have tolerance rules to avoid false positives from floating-point differences or acceptable variance. Configure tolerances in the model's `MODEL()` header: - -```sql -MODEL ( - materialized incremental, - ... - row_diff_tolerances ( - by_column ( - total_revenue_cents ( - absolute 1, - ), - ), - ), -); -``` - -Tolerance rules support: -- **`absolute`** - maximum allowed absolute difference (e.g. `1` means values differing by 1 or less are treated as equal) -- **`relative`** - maximum allowed relative difference as a decimal (e.g. `0.01` for 1%) - -Tolerances can be set per-column (`by_column`) or per-type (`by_type`). - -Use repeated `--tolerance COLUMN:absolute=VALUE` or `--tolerance COLUMN:relative=VALUE` arguments -to override per-column rules for one model or raw-query invocation. Unkeyed comparison is exact and -does not accept tolerances. - -### Excluded columns - -Columns that are expected to differ between the two sides (like timestamps or context-specific values) can be excluded from the row comparison: - -```sql -MODEL ( - materialized incremental, - ... - row_diff_exclude_columns [latest_order_status], -); -``` - -Excluded columns are still shown in the schema comparison but skipped during row-level diffing. A column cannot be in both `row_diff_exclude_columns` and `unique_key`. - -Use repeated `--exclude-column COLUMN` arguments for invocation-level exclusions. - -### Raw-query artifacts and crash recovery - -SQLBuild materializes each raw query once in the active target schema, compares the stable results, -and drops both artifacts in a `finally` path. Every successfully created artifact appends an -immutable `query_diff_artifact` fingerprint containing ownership and expiry metadata. Query -execution and comparison never depend on historical fingerprints; they are ownership evidence for -later cleanup, not workflow status. - -Before a query diff, and independently through `sqb janitor`, SQLBuild removes an expired artifact -only when its exact physical identity and reserved name match its fingerprint. A reserved-name -relation without matching ownership evidence is reported and retained for manual investigation. -Configure the crash-recovery TTL, which defaults to 24 hours, in `sqlbuild_project.toml`: - -```toml -[diff] -query_artifact_ttl = "24h" -``` - -Historical ownership inspection failures are warnings and do not prevent the current query pair -from running. Current-run ownership publication and immediate cleanup remain mandatory. - -### Verbose output - -Add `--verbose` or `-v` to see more example rows for mismatches and side-only rows: - -```bash -sqb diff prod:dev --full --select customer_status_snapshot --verbose -``` - -Default example limits are 3 per category. Verbose mode increases this to 10. You can also set exact limits: - -```bash -sqb diff prod:dev --full --select customer_status_snapshot --max-column-examples 20 --max-row-only-examples 5 -``` - -Example limits only control diagnostic values printed after comparison. They are separate from -`row_diff_sample_rows`, which controls how many unique keys receive the wide value comparison. - -Long values default to 160 rendered characters. SQLBuild shows context around the first difference, -marks truncation, and reports the original length without changing exact comparison counts. Control -value evidence independently from example counts: - -```bash -sqb diff \ - --left-query-file queries/orders_before.sql \ - --right-query-file queries/orders_after.sql \ - --key order_id \ - --max-value-length 500 - -sqb diff \ - --left-query-file queries/orders_before.sql \ - --right-query-file queries/orders_after.sql \ - --key order_id \ - --no-example-values - -sqb diff \ - --left-query-file queries/orders_before.sql \ - --right-query-file queries/orders_after.sql \ - --key order_id \ - --full-example-values -``` - -`--verbose` increases the example count but does not remove the value-length bound. - -### Structured output - -Use `--json-output PATH` to write a stable structured result while retaining the normal terminal -summary, or `--json` to write one JSON document to stdout with progress on stderr: - -```bash -sqb diff prod:dev --bounded 14d --sample-rows 50000 --select order_lines --json-output diff.json -sqb diff --left-query-file before.sql --right-query-file after.sql --key order_id --json -``` - -Each model records `schema_only`, `exhaustive`, or `sampled` comparison scope; requested and observed -cursor coverage; bounded population and compared key counts; seed and configured limit; row result -counts; and changed-column counts. Query output additionally includes named inputs, -`query_comparisons`, sparse nonzero `column_mismatches`, lifecycle timings, and explicit evidence -truncation metadata. Existing `models` output remains for compatibility. - -### Invocation safety limits - -Use `--max-models` and `--max-columns` to put explicit hard limits around a broad selector. SQLBuild -fails visibly instead of truncating the selected models or compared columns: - -```bash -sqb diff prod:dev --bounded 14d --sample-rows 50000 --max-models 10 --max-columns 80 --select path:models/intermediate -``` - -These limits are optional and apply to the complete invocation. `--max-columns` checks the larger -observed schema for each model before starting its row comparison. - -### Selectors - -Model diff requires `--select` in the current version. You can use any selector syntax: - -```bash -# Diff a single model -sqb diff prod:dev --full --select customer_status_snapshot - -# Diff all models in a path -sqb diff prod:dev --schema-only --select path:models/marts - -# Diff models with a specific tag -sqb diff prod:dev --full --select tag:acceptance -``` - -### Exit codes - -Complete comparisons return `0` when no differences are found and `1` for findings. Raw-query mode -returns `2` when safety, schema, or key preflight prevents complete value evidence, and `3` for -setup, connection, materialization, comparison execution, ownership publication, or cleanup -failure. Human and JSON output carry the same outcome. - -## Execution Observability - -Source: `concepts/observability.mdx` - -Choose authoritative lifecycle facts, readable logs, or command-output records. - -SQLBuild keeps execution facts separate from human output. Do not parse terminal text to determine -whether work started, completed, failed, skipped, or retried. - -| Record | Purpose | Authority | -|--------|---------|-----------| -| `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | -| `DiagnosticLog` | Structured framework diagnostics | Explanatory only | -| `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | -| `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | -| Final JSON output | End-of-command aggregate | Result projection, not event history | - -Human-readable CLI output remains the normal operator interface. Structured records are a separate -contract for integrations and durable consumers. - -### Lifecycle envelope - -New events use lifecycle schema version 2: - -```json -{ - "event_id": "9fb6d899754d4bd79c703067a8d5046c", - "event_type": "statement_completed", - "schema_version": 2, - "producer": "sqlbuild", - "producer_version": "0.91.0", - "occurred_at": "2026-09-07T18:15:30.125000Z", - "invocation_id": "4cc757dd93fe466aad2a220f1e76625e", - "invocation_sequence": 17, - "run_id": "run-20260907-181500", - "resource_id": "model:orders", - "resource_attempt_id": "97933c4aa29441cda133641345515e20", - "operation_id": "c978e47367494051bdde68ad03e56db4", - "statement_id": "5113ebc5afe247eb9c41dc0b27ef1a9a", - "external_context": { - "integration": { - "name": "dagster", - "run_id": "dagster-run-1", - "job_name": "daily_models", - "step_key": "sqlbuild_assets", - "retry_number": 0, - "partition_key": "2026-09-07" - } - }, - "payload": { - "adapter": "snowflake", - "duration_ms": 318.4, - "query_id": "01b6f1c2-0000-0000-0000-000000000000", - "sql_digest": "881b77b00a75e07936d7f20ea308a55e9ef98f2f1fd42c144a3a998d32094c1b" - } -} -``` - -`event_id` is the immutable deduplication identity. `invocation_sequence` is unique and increasing -in synchronous publication order within one invocation, including concurrent workers. A sink may -prioritize delivery, so consumers reconstruct invocation chronology by sorting this field rather -than relying on arrival timestamps. - -Schema-version-1 events remain readable. Unknown event names and future schema versions are retained -as opaque envelopes instead of being assigned invented semantics. - -### Correlation and privacy - -Every fact has an invocation ID. Nested IDs identify a run, logical resource and attempt, non-SQL -operation, and SQL statement. Orchestrators may provide bounded JSON-compatible -`external_context`; the Dagster integration supplies run, job, step, retry, and partition values -when its execution context exposes them. - -Lifecycle payloads exclude full SQL, parameter values, credentials, arbitrary user messages, and -raw process output. Command output and local logs can contain sensitive data and require an explicit -retention and access policy. - -### Failure semantics - -A start without a terminal fact is unknown or presumed lost. Consumers must not fabricate success -or failure. Sink delivery is bounded and best effort; destination failure does not change successful -warehouse work, and successful delivery does not prove the command succeeded. - -See [Typed Sinks](/concepts/observability/sinks) to export lifecycle and command-output records. - -## Typed Sinks - -Source: `concepts/observability/sinks.mdx` - -Export lifecycle facts and command output through project-owned providers. - -Project sinks consume one declared record type. Lifecycle and command-output streams remain separate -even when they share a provider and destination. - -### Lifecycle sink - -```python -# sinks/publish.py -from providers.destination_client import DestinationClient -from sqlbuild.sinks import ( - LifecycleEvent, - LifecycleEventKind, - lifecycle_event_sink, - lifecycle_event_to_json, -) - -@lifecycle_event_sink( - event_kinds={ - LifecycleEventKind.INVOCATION, - LifecycleEventKind.RUN, - LifecycleEventKind.RESOURCE, - LifecycleEventKind.AUDIT, - } -) -def publish_lifecycle(event: LifecycleEvent, destination_client: DestinationClient) -> None: - destination_client.publish( - route="sqlbuild.lifecycle.v2", - key=event.invocation_id, - payload=lifecycle_event_to_json(event).encode("utf-8"), - ) -``` - -Using `invocation_id` as a partition key keeps one invocation on one Kafka partition. Keep -`event_id` in the payload as the deduplication identity. Confirm the destination topic is not -compacted when complete lifecycle history must remain replayable. - -### Command-output sink - -```python -from sqlbuild.sinks import CommandOutputRecord, command_output_sink, command_output_to_json - -@command_output_sink(streams={"stdout", "stderr"}) -def publish_output(record: CommandOutputRecord, destination_client: DestinationClient) -> None: - destination_client.publish( - route="sqlbuild.command_output.v1", - key=record.record_id, - payload=command_output_to_json(record).encode("utf-8"), - ) -``` - -Command output groups adjacent text into bounded byte chunks and flushes on elapsed time, stream -change, size, or close. A loss record reports bounded-queue drops. This stream is useful for remote -transcripts but is not lifecycle evidence. - -### Configuration - -Runtime configuration can narrow declaration filters: - -```toml -[sinks.lifecycle] -event_kinds = ["run", "resource", "operation", "statement", "audit"] -min_severity = "info" - -[sinks.lifecycle.named.publish_lifecycle] -event_kinds = ["resource", "statement"] -min_severity = "warning" -``` - -SQLBuild owns envelope validation, local queueing, filtering, and bounded dispatch. The project owns -destination credentials, serialization, routes/topics, acknowledgements, retries, retention, and -durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. - -### Delivery behavior - -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. -- Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. -- A failing or timed-out sink is isolated from command correctness. -- Provider setup and declaration errors fail before execution because the project configuration is - invalid. -- Providers shared by multiple sinks are set up and torn down once per command. - -Use lifecycle facts—not command-output records—to build execution state and timelines. - -## Overview - -Source: `concepts/declaration-scopes.mdx` - -Limit enums, constants, and macros to the parts of a project that use them. - -Most projects can keep enums, constants, and macros in their ordinary top-level directories. Those -declarations are available throughout the project. - -When an enum, constant, or macro should be available only within one folder or its child folders, -you can keep it near the SQL that uses it. SQLBuild uses the declaration directory's location to -decide which files can access it. No TOML configuration is required. - -| Where the declaration lives | Who can use it | -|-------------------------------------|--------------------| -| Top-level `macros/`, `constants/`, or `enums/` | The whole project | -| Nested `_sqlbuild/macros/`, `_sqlbuild/constants/`, or `_sqlbuild/enums/` | Files in the owner folder and folders below it | -| Nested `_sqlbuild/_macros/`, `_sqlbuild/_constants/`, or `_sqlbuild/_enums/` | Files directly in the owner folder only | -| An underscored constant or enum inside `MODEL()` | That model and its inline SQL hooks only | - -Macro declarations are Python files (`.py`). Constant and enum declarations are SQL files -(`.sql`). - -### Example - -The annotation beside each directory shows where its contents are available: - -```text -models/ -├── constants/ published throughout models/ -│ └── warehouse.sql -└── commerce/ - ├── _sqlbuild/ - │ ├── macros/ published throughout commerce/ - │ │ └── currency.py - │ └── _enums/ exact commerce/ directory only - │ └── grain.sql - ├── orders.sql sees warehouse, currency, and grain - ├── finance/ - │ ├── _sqlbuild/ - │ │ └── macros/ published throughout finance/ - │ │ └── tax.py - │ └── revenue.sql sees warehouse, currency, and tax - └── fulfillment/ - └── shipments.sql sees warehouse and currency -``` - -The folder containing `_sqlbuild/` is the declaration owner. An unprefixed role applies to that -owner and everything below it. An underscored role applies only to files directly in the owner. -Legacy declaration roles directly below an owner remain supported. - -`_sqlbuild/` is reserved for the six declaration-role directories shown above. Other direct files or -folders are rejected. It must also sit below a concrete owner directory. A project-root -`_sqlbuild/` or authored-root path such as `models/_sqlbuild/` is invalid; declarations at that -boundary belong in the project-wide `macros/`, `enums/`, or `constants/` roots. - -| Resource | Visible from the example tree | Not visible | -|----------|-------------------------------|-------------| -| `orders.sql` | Warehouse constant, commerce macro, commerce-folder-only enum | Finance macro | -| `finance/revenue.sql` | Warehouse constant, commerce macro, finance macro | Commerce-folder-only enum | -| `fulfillment/shipments.sql` | Warehouse constant, commerce macro | Commerce-folder-only enum, finance macro | - -Declarations do not flow upward or sideways into sibling directories. - -### Terms used in text output - -The CLI's text output uses short labels for the same four rules: - -| Canonical label | Plain-language meaning | -|-----------------|------------------------| -| `project` | Available throughout the project | -| `descendant-public` | Available in this folder and folders below it | -| `exact-owner-private` | Available directly in this folder only | -| `model-private` | Available to one model only | - -Here, an **owner folder** is simply the folder directly containing a model, test, hook, function, or -other authored resource. The [Scope Explorer](/concepts/declaration-scopes/explorer) shows the owner -folder, its parents, and project-wide declarations separately. - -JSON keeps the underlying enum values for integrations. Its `scope` field uses `global`, -`inherited`, `local`, or `private`, while derived `visibility` values use underscores, such as -`descendant_public` and `exact_owner_private`. Consumers should use the fields documented in the -versioned JSON schema rather than parsing text labels. - -### Choose a location - -| Where the value is needed | Placement | -|---------------------------|-----------| -| One model only | In that model's `MODEL()` header | -| Files directly in one folder | In an underscored role under the owner's `_sqlbuild/` folder | -| Files across one folder tree | In an unprefixed role under the nearest shared owner's `_sqlbuild/` folder | -| Different resource trees, such as models and tests | In a top-level declaration role | - -SQLBuild computes the lowest common owner of every declaration's runtime consumers. A project-wide -declaration is valid only when no narrower supported owner contains all consumers. This keeps the -top-level roles as a genuine project API instead of a neutral dumping ground. - -### Explore the feature - - See which declarations a model, test, hook, or function can use. - Choose the narrowest folder that contains every real use. - Ask SQLBuild what a file can use and preview how moving it would change that answer. - -Learn the features themselves in [Enums](/concepts/enums), [Constants](/concepts/constants), and -[Writing Macros](/concepts/macros). See [Interpolation](/concepts/interpolation) for project, -environment, and runtime context values, which are separate from declarations. - -## How Visibility Works - -Source: `concepts/declaration-scopes/visibility.mdx` - -See which enums, constants, and macros are available to each SQL file. - -For most SQL, the rule is simple: - -> A file can use declarations available to the whole project, declarations in unprefixed roles -> under `_sqlbuild/` folders owned beside or above it, and declarations in an underscored role owned -> directly beside it. - -### Start from the SQL file - -SQLBuild starts from the file containing the SQL and walks up that file's directory tree. The folder -directly containing the file is its **owner folder**. - -```text -models/ -├── constants/ available throughout models/ -│ └── warehouse.sql -└── commerce/ - ├── _sqlbuild/ - │ ├── enums/ available throughout commerce/ - │ │ └── order_status.sql - │ └── _constants/ available directly in commerce/ only - │ └── minimum_value.sql - ├── orders.sql - └── history/ - └── archived_orders.sql -``` - -From this tree: - -| File | Can use | -|------|---------| -| `orders.sql` | `warehouse`, `order_status`, and `minimum_value` | -| `history/archived_orders.sql` | `warehouse` and `order_status` | - -`minimum_value` is not available in `history/` because `_sqlbuild/_constants/` applies only to files -directly in the `_sqlbuild/` owner's folder. - -### Supported resource trees - -Grouped declaration directories can be placed below these SQL resource roots: - -| Root | Contents | -|------|----------| -| `models/` | Models and inline model hooks | -| `tests/unit/` | Unit tests | -| `tests/scenarios/` | Scenarios | -| `hooks/sql/` | Named SQL hooks | -| `functions/sql/` | SQL functions | -| `audits/` | Audits | -| `sources/` | Inline source expressions | - -Each root is a separate tree. For example, `models/constants/` does not make declarations available -under `tests/`. Put a declaration in the top-level `constants/`, `enums/`, or `macros/` directory -when it must be available across different resource trees. Project-root `_constants/`, `_enums/`, -and `_macros/` directories are invalid because there is no owner folder at the project root for -them to be private to. - -Existing scoped declaration roles directly below an owner remain supported. `_sqlbuild/` is the -preferred layout because it keeps all declarations together without changing their visibility. -Only the six public/private declaration-role directories are valid directly under `_sqlbuild/`; -other entries are rejected. `_sqlbuild/` must sit below a concrete owner directory. A project-root -`_sqlbuild/` or authored-root path such as `models/_sqlbuild/` is invalid; use the project-wide -`macros/`, `enums/`, or `constants/` roots at that boundary. - -### Which file controls visibility? - -| SQL being compiled | SQLBuild starts from | -|--------------------|----------------------| -| Model query | The model file | -| Inline SQL hook in a model | The model file | -| Unit test SQL | The test file, plus inferred tested-resource relationships | -| Scenario SQL | The scenario file, plus expected-model enum and constant relationships | -| Named SQL hook | The hook file under `hooks/sql/` | -| SQL function | The function file under `functions/sql/` | -| Audit | The audit file | -| Inline source expression | The source definition | - -This means a reusable named hook does not change meaning depending on which model calls it. The -hook uses declarations available where the hook itself is stored. - -### Tests and expected output - -A test first sees declarations available from its own folder tree. It may also use file-based -macros, enums, and constants available to a model for which it defines expected output. - -```sql -TEST(); - -WITH -__expected__orders AS ( - SELECT - 1 AS order_id, - @normalize_order_status("'completed'") AS normalized_status, - @enum("order_status").COMPLETED AS status -) -SELECT 1 -``` - -Because the test defines `__expected__orders`, the test may use file-based macros, enums, and -constants available to `orders`, including declarations in exact-owner-private roles. -Scope Explorer describes this as **available through expected output for model `orders`**. This is -an additional relationship, not another visibility level. - -This additional access does **not** include: - -- Enums or constants declared privately inside the model's `MODEL()` header -- Declarations from a model merely mentioned by filename or directory layout - -Only an explicit `__expected__model_name` section adds the model's eligible file-based declarations. -When a test checks several models, SQLBuild combines the declarations available through all -expected models and makes that deterministic union available while compiling the entire test. - -Macro-mode tests receive the tested macro and the file-based declarations available from that -macro's production owner. SQLBuild infers tested macros from calls in `__macro_actual__`; the test's -filename or directory does not grant production visibility. - -```text -models/orders/_sqlbuild/_macros/order_policy.py -tests/unit/macros/models/orders/test_order_policy__returns_quantity.sql -``` - -The macro test can call `order_policy` without promoting it to the project-wide `macros/` root. A -test can also use helper macros scoped to its own test directory: - -```text -tests/unit/orders/ -├── _sqlbuild/ -│ └── macros/ -│ └── fixtures.py -└── test_orders__calculates_total.sql -``` - -Those helpers remain unavailable to production resources. Mocked model, source, seed, and dbt -fixture CTEs do not grant declarations from the mocked resource's production scope. - -### Macros importing macros - -A macro file may import another project macro file when the imported macro is available from the -importing file's location. The same directory rules apply as they do to SQL. - -```python -# models/commerce/macros/orders.py -from macros.currency import round_money - -def formatted_total(expression: str) -> str: - return round_money(expression) -``` - -A broadly available macro cannot import a macro from a narrower child or sibling directory. See -[Composition and Context](/concepts/macros/composition-and-context) for complete examples. - -### Names do not shadow - -Public names must be unique within their feature: - -- One public macro name cannot be defined twice. -- One public enum name cannot be defined twice. -- One public constant name cannot be defined twice. - -Moving a declaration into a narrower directory changes where it is available; it does not create a -second version that overrides another declaration. - -Macros, enums, and constants use separate namespaces, so these three references may coexist: - -```sql -@format_currency() -@const("format_currency") -@enum("format_currency").USD -``` - - Choose the simplest directory that matches where a declaration is used. - -## Where to Put Declarations - -Source: `concepts/declaration-scopes/placement.mdx` - -Choose the narrowest folder that contains every real use. - -Start with the ordinary project-wide directories unless you have a reason to limit access: - -```text -macros/ -enums/ -constants/ -``` - -Move a declaration closer to its users when it should be available only in one folder or one folder -tree. - -### Choose a location - -| Where it is needed | Location | -|--------------------|----------| -| One model only | Inside that model's `MODEL()` header, for enums and constants | -| SQL files directly in one directory | `_sqlbuild/_macros/`, `_sqlbuild/_enums/`, or `_sqlbuild/_constants/` | -| A directory and its descendants | `_sqlbuild/macros/`, `_sqlbuild/enums/`, or `_sqlbuild/constants/` | -| Different trees, such as models and tests | Top-level `macros/`, `enums/`, or `constants/` | - -### One directory - -These two models use the same constant and both sit directly in `commerce/`: - -```text -models/ -└── commerce/ - ├── _sqlbuild/ - │ └── _constants/ - │ └── minimum_value.sql - ├── orders.sql - └── customers.sql -``` - -Use `_sqlbuild/_constants/` because no descendant directory needs the value. - -### One directory tree - -These models use the same constant across two child directories: - -```text -models/ -└── commerce/ - ├── _sqlbuild/ - │ └── constants/ - │ └── reporting_day.sql - ├── finance/ - │ └── revenue.sql - └── fulfillment/ - └── shipments.sql -``` - -Use `_sqlbuild/constants/` in `commerce/` so both child directories can use it. - -### Different resource trees - -When a declaration is used directly from unrelated trees, make it project-wide: - -```text -my_project/ -├── constants/ -│ └── order_status.sql -├── models/ -│ └── commerce/orders.sql -└── tests/ - └── scenarios/order_lifecycle.sql -``` - -Top-level placement is valid when consumers genuinely cross resource trees or otherwise have no -shared owner folder. SQLBuild applies the same nearest-shared-folder analysis to project-wide -declarations, so a declaration used only under one narrower folder must move closer to those users. - -### What SQLBuild checks - -Every declaration must be used by real compiled SQL. A name appearing only in a comment, quoted -string, mock identity, or documentation does not count as use. - -For every declaration, including project-wide declarations, SQLBuild finds the nearest shared owner -folder of the files that use it. If the declaration is in a broader location, the error shows: - -- Where the declaration is now -- Which files use it -- Which directory form is required -- The destination directory - -These checks use the complete project rather than only the models selected by the current command. - - Use Scope Explorer to see what a file can access or preview moving the file. - -## Scope Explorer - -Source: `concepts/declaration-scopes/explorer.mdx` - -Inspect visibility, explain resolution, browse declarations, and preview moves offline. - -Scope Explorer answers questions about declaration visibility through `sqb scope`. It is read-only -and offline: it never connects to the warehouse, moves files, or edits configuration. - -Use this page for common workflows. See the [`sqb scope` CLI reference](/cli/scope) for every flag, -filter, text section, and JSON field. - -### Example project - -The examples below use two domains with project-wide currency helpers: - -```text -project/ -├── macros/ -│ └── currency.py add_tax, round_money -├── models/ -│ ├── commerce/ -│ │ ├── constants/limits.sql minimum_order_value -│ │ ├── enums/status.sql order_status -│ │ ├── macros/orders.py formatted_order_total -│ │ ├── orders.sql -│ │ └── returns/returns.sql -│ └── finance/ -│ ├── enums/status.sql finance_status -│ ├── macros/margin.py calculate_margin -│ ├── finance_summary.sql -│ └── reports/margin_report.sql -└── tests/ - └── unit/orders__completed_only.sql -``` - -`formatted_order_total` composes the project-wide functions through ordinary Python calls: - -```python -from macros.currency import add_tax, round_money - -def formatted_order_total(expression: str) -> str: - return round_money(add_tax(expression)) -``` - -`orders.sql` uses the commerce constant and macro. The returns model also uses all three commerce -declarations, so they are correctly published throughout that domain. The two finance models use -both finance declarations. This gives every scoped declaration a real consumer and valid placement. - -### See available and used declarations - -Start with a model, test, scenario, hook, function, audit, source, or authored resource path to -inspect its directory-derived scope. A declaration identity instead opens its explanation. This -excerpt of the main report sections puts actual usage beside the complete directory-derived scope: - -```console -$ sqb scope model:orders -Scope - Target: model:orders - Resource: model:orders - Path: models/commerce/orders.sql - -Used (2) - ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - -Scope chain - ├─ exact-owner-private models/commerce (3) - ├─ descendant-public models (0) - └─ project global (2) - -Available (3 of 5, 2 collapsed) - ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 - ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - … 2 globals collapsed; run sqb scope model:orders --globals all - -Diagnostics (0) - (none) -Completeness: complete -``` - -`●` marks a declaration included in `Used`: a direct usage by default, or a followed declaration -dependency when `--dependency-depth` is nonzero. `○` marks a declaration present in the section but -not in `Used`. - -The compact labels translate to ordinary folder rules: - -| Output label | Meaning in this report | -|--------------|------------------------| -| `project` | Available throughout the project | -| `descendant-public` | Available from a folder at or above this resource | -| `exact-owner-private` | Available directly in this resource's owner folder only | -| `inherited_ancestor` | Visible because an unprefixed role publishes it down the folder tree | -| `role models/commerce/macros` | The declaration role begins at this path | - -In the scope chain, the first path is the resource's owner folder, followed by parent folders and -the project declaration roles. The counts show declarations defined at each path; the chain label -describes how SQLBuild reaches that path, not the visibility of every declaration counted there. -Unused project-wide declarations stay collapsed by default so a large project API does not hide the -owner-folder facts. - -### Follow composed dependencies - -`Used` is direct by default. Add `--dependency-depth` to follow declarations used by those -declarations. The expanded `Used` section is: - -```console -$ sqb scope model:orders --used-only --dependency-depth 1 -Used (4) - ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 - ├─ ● macro:add_tax [macro; project; dependency; params 1; role macros] macros/currency.py:1:1 - ├─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - └─ ● macro:round_money [macro; project; dependency; params 1; role macros] macros/currency.py:5:1 -``` - -SQLBuild derives these edges from actual Python calls, including nested calls and calls reached -through private helpers. Merely importing a macro does not make it used. Increase the depth to -follow longer chains. - -Explain the composed macro to see its direct dependencies, consumers, and required placement. The -ordinary report appears first; its `Explanation` section is: - -```console -$ sqb scope model:orders --explain macro:formatted_order_total -Explanation - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - Owner: (none) - Owning path: models/commerce - Consumers: model:orders, model:returns - Dependencies: macro:add_tax, macro:round_money - Grants: (none) - Required scope: descendant-public - Required path: models/commerce - Promotion impact: (none) -``` - -#### Catch placement that is too broad - -If `returns.sql` stopped using `formatted_order_total`, only `orders.sql` would consume it. The same -explanation would then show that descendant-public placement is broader than necessary: - -```console -Explanation - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - Owner: (none) - Owning path: models/commerce - Consumers: model:orders - Dependencies: macro:add_tax, macro:round_money - Grants: (none) - Required scope: exact-owner-private - Required path: models/commerce - Promotion impact: model:orders - -Diagnostics (1) - ERROR S008 models/commerce/macros/orders.py: Declaration 'macro:formatted_order_total' is currently descendant-public at 'models/commerce' (models/commerce/macros/orders.py); required exact-owner-private at 'models/commerce'. Consumers: model:orders. Move it to 'models/commerce/_sqlbuild/_macros/' -Completeness: complete -``` - -This is actionable placement guidance, not only a visibility lookup. - -### Find declarations outside the scope - -Nearby discovery is useful when you know a declaration exists but not why the current resource -cannot use it. The relevant section is: - -```console -$ sqb scope model:orders --include-nearby -Nearby unavailable (2 of 2) - ├─ ○ enum:finance_status [enum; descendant-public; sibling_scope; members 3; type VARCHAR; role models/finance/enums] models/finance/enums/status.sql:1:1 - └─ ○ macro:calculate_margin [macro; descendant-public; sibling_scope; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 -``` - -The declarations are known, but they belong to a different folder branch. Here, the inspected -resource belongs to `models/commerce`, while the declarations belong to `models/finance`. -`sibling_scope` is the compact output label for that situation. Ask for one complete explanation -when you know a declaration's identity. After the ordinary report, the command adds: - -```console -$ sqb scope model:orders --explain macro:calculate_margin -Explanation - └─ ○ macro:calculate_margin [macro; descendant-public; sibling_scope; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 - Owner: (none) - Owning path: models/finance - Consumers: model:finance_summary, model:margin_report - Dependencies: (none) - Grants: (none) - Required scope: descendant-public - Required path: models/finance - Promotion impact: (none) -``` - -An explanation distinguishes a known but inaccessible declaration from an unknown name. It also -shows whether the declaration is placed more broadly than its real consumers require. - -### See declarations available through expected output - -Tests can use file-based macros, enums, and constants available to a model when they define that -model's expected output. Scenarios receive the model's file-based enums and constants but not its -macros. Neither receives declarations inside `MODEL()`. The access applies to the whole test or -scenario and remains separate from declarations visible through its own folder tree. The relevant -report sections are: - -```console -$ sqb scope test:orders__completed_only --used-only -Used (1) - └─ ● enum:order_status [enum; descendant-public; expected_model through model:orders; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 - -Relationship grants (1 of 1) - └─ ● enum:order_status [enum; descendant-public; expected_model through model:orders; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 -``` - -The compact reason `expected_model through model:orders` means **available through expected output -for model `orders`**. With multiple expected models, SQLBuild combines their eligible file-based -declarations into one deterministic set. Tests include macros in that union; scenarios include only -enums and constants. Neither receives declarations defined inside a model's `MODEL()` header. - -Macro-mode tests also report `tested_macro through macro:` relationships. Those grants contain -the tested macro and scoped file-based declarations available from its production owner. - -### Preview a resource move - -Use `--as-path` to see the scope delta before moving an existing model, test, hook, or function. The -ordinary report is followed by: - -```console -$ sqb scope model:orders --as-path models/finance/orders.sql -Move preview - Resource: model:orders - Destination: models/finance/orders.sql - Ownership root: models - Retained (2) - ├─ ○ macro:add_tax [macro; project; global; params 1; role macros] macros/currency.py:1:1 - └─ ○ macro:round_money [macro; project; global; params 1; role macros] macros/currency.py:5:1 - Gained (2) - ├─ ○ enum:finance_status [enum; descendant-public; inherited_ancestor; members 3; type VARCHAR; role models/finance/enums] models/finance/enums/status.sql:1:1 - └─ ○ macro:calculate_margin [macro; descendant-public; inherited_ancestor; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 - Lost (3) - ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 - ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - Private retained (0) - (none) - Relationship retained (0) - (none) - Invalidated usages (2) - - constant:minimum_order_value - - macro:formatted_order_total -``` - -The move would gain finance declarations and lose commerce declarations. More importantly, -`Invalidated usages` separates the losses that break the model from declarations that happened to -be available but were never used. The preview does not move or rewrite any file. - -### Check a prospective path - -Inspect visibility before a resource exists: - -```console -$ sqb scope --at models/commerce/returns/new_return.sql -Scope - Target: models/commerce/returns/new_return.sql - Path: models/commerce/returns/new_return.sql - Status: prospective - -Used (0) - (none) - -Scope chain - ├─ exact-owner-private models/commerce/returns (0) - ├─ descendant-public models/commerce (3) - ├─ descendant-public models (0) - └─ project global (2) - -Available (3 of 5, 2 collapsed) - ├─ ○ constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 - ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 - └─ ○ macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 - … 2 globals collapsed; run sqb scope --at models/commerce/returns/new_return.sql --globals all - -Relationship grants (0 of 0) - (none) - -Nearby unavailable (0 of 0) - (none) - -Diagnostics (1) - ERROR S013 models/commerce/returns/new_return.sql: Runtime usage and relationship facts are unavailable for a prospective path -Completeness: partial -``` - -Visibility is available from the proposed path. Actual usage and expected-output relationships do -not exist yet, so the command preserves the useful static result while clearly marking it partial. - -### Browse large declaration sets - -In a larger project, browse returns folder summaries rather than an arbitrary prefix of the -declaration inventory. The `used` counts are direct usages by this target; browse does not apply -dependency-depth expansion: - -```console -$ sqb scope model:orders --browse global -Scope folders - Path: global - - ├─ constants/ 146 declarations, 18 used, 4 children; constant 146 - sqb scope model:orders --browse global/constants - sqb scope model:orders --list global/constants - ├─ enums/ 84 declarations, 11 used, 3 children; enum 84 - sqb scope model:orders --browse global/enums - sqb scope model:orders --list global/enums - └─ macros/ 231 declarations, 27 used, 8 children; macro 231 - sqb scope model:orders --browse global/macros - sqb scope model:orders --list global/macros - -Diagnostics (0) - (none) -Completeness: complete -``` - -Choose a bounded folder, then use `--list`. Definition-path, kind, glob, and actual-usage filters -can be combined, and paged sections use stable declaration identities as cursors. - -### Automation - -Use `--json` for editor integrations and repository tooling. Output has a versioned schema, stable -ordering, filters, pagination, and move-preview results. Constant values, credentials, and secret -connection settings are not included. - - Review the directory rules behind the report. - See all selectors, filters, pagination options, output sections, and JSON behavior. - -## Overview - -Source: `concepts/rules/custom-rules/overview.mdx` - -Define repository-owned checks with one typed Python API. - -Custom Rules are ordinary Python under `rules/**/*.py`. Only functions decorated with `@rule` -register; helpers, constants, dataclasses, and classes remain ordinary Python. - -### Model-subject Rules - -```python -from sqlbuild.rules import Finding, Model, RuleContext, rule - -@rule( - code="XSQBRARCH001", - message="Final models must declare an order identifier", - remediation="Declare order_id in the model contract.", -) -def final_order_identifier(*, model: Model, ctx: RuleContext) -> list[Finding]: - declared = {column.name for column in ctx.columns.declared(model)} - return [] if "order_id" in declared else [ctx.finding(subject=model)] -``` - -Annotations determine invocation. The signature must contain exactly one typed `Model` or `Project` -subject and one typed `RuleContext`, all keyword-only. Parameter names and position are not semantic. -A model-subject Rule can still inspect project-wide facts. - -### Project-subject Rules - -Use `Project` when an invariant has no natural model subject: - -```python -from sqlbuild.rules import Finding, Project, RuleContext, rule - -@rule( - code="XSQBRARCH002", - message="The project must contain a final model directory", - remediation="Add models/final and place final outputs beneath it.", -) -def final_directory(*, ctx: RuleContext, project: Project) -> list[Finding]: - del project - paths = ctx.project.tree.glob("models/final/**") - return [] if paths else [ctx.finding(subject="models")] -``` - -Rules cannot mutate SQL, resources, adapter lowering, or compiler output. They return diagnostics -only. - -### Typed options - -```python -from sqlbuild.rules import Finding, Model, RuleContext, RuleOption, rule - -REQUIRED_PREFIX = RuleOption.string( - name="required_prefix", - default="customer", - description="Required model-name prefix.", -) - -@rule( - code="XSQBRNAME001", - message="The model name uses the wrong prefix", - remediation="Rename the model with the configured prefix.", - options=(REQUIRED_PREFIX,), -) -def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: - prefix = ctx.option(REQUIRED_PREFIX) - return [] if model.name.startswith(prefix) else [ctx.finding(subject=model)] -``` - -```toml -[rules.rule_options.XSQBRNAME001] -required_prefix = "order" -``` - -Unknown codes, option names, or invalid values fail configuration. - -### Continue - - Inspect SQL, graph, contracts, tests, audits, declarations, and project structure. - Test findings and keep helper and project observations cacheable. - -## RuleContext and compiler facts - -Source: `concepts/rules/custom-rules/rule-context.mdx` - -Inspect stable compiler-owned facts and opt into lazy SQL AST access. - -`RuleContext` exposes typed, read-only views over the compiled project: - -- `ctx.sql`: authored and expanded SQL -- `ctx.graph`: compiler-resolved dependencies and dependents -- `ctx.columns`: declared and inferred output columns -- `ctx.contracts`: enforcement and grain facts -- `ctx.tests` and `ctx.audits`: checks associated with a model -- `ctx.declarations`: public and model-scoped enums and constants -- `ctx.project`: compiled resources and deterministic project-tree observations - -### Authored and expanded SQL - -Choose the SQL representation that matches the requirement: - -```python -sql = ctx.sql.for_model(model) - -authored_text = sql.authored.source -expanded_text = sql.expanded.source -``` - -An authoring convention may inspect what a developer wrote. A compiler-output convention may need -the expanded SQL after interpolation, declarations, and macros. - -### Common SQL structures - -Common structures are available through typed source nodes: - -```python -for cte in sql.expanded.ctes(): - ... - -for star in sql.authored.star_projections(): - ... -``` - -### Lazy Polyglot access - -Use the full Polyglot AST only when common projections are insufficient: - -```python -ast = sql.expanded.polyglot_ast() -for node in ast.walk(): - ... -``` - -AST construction is lazy. A path or contract Rule does not pay for SQL parsing merely because AST -access exists elsewhere. - -### Project structure - -Use the compiler-owned tree rather than direct filesystem calls: - -```python -parts = ctx.project.tree.relative_parts(path=model.path, under="models") -models = ctx.project.tree.resources_under("models/orders") -config = ctx.project.tree.read_text("rules/requirements.yaml") -``` - -Paths remain project-relative, and observations participate in cache invalidation. - -### Choose the subject deliberately - -A model-subject Rule is evaluated and cached per model. A project-subject Rule runs once and can -iterate `ctx.project.models`. Use a project subject only when the invariant genuinely needs a -project-wide view; its cache invalidation is intentionally broader. - -## Testing and deterministic Rules - -Source: `concepts/rules/custom-rules/testing-and-determinism.mdx` - -Test custom findings and keep implementations safe for dependency-aware caching. - -### Test Rules - -Use the public harness to exercise discovery, compilation, and Rules evaluation: - -```python -from sqlbuild.rules.testing import RuleCase, evaluate_rule -from rules.final_outputs import final_order_identifier - -def test_given_missing_order_id_when_evaluating_then_reports_finding() -> None: - result = evaluate_rule( - rule=final_order_identifier, - test_case=RuleCase( - description="missing order identifier", - source='MODEL (description "Orders");\nSELECT 1 AS customer_id\n', - path="models/final/customer_orders.sql", - expected_finding_count=1, - ), - ) - - assert result.finding_count == 1 -``` - -`RuleCase.files` adds supporting project files. `RuleCase.config` supplies option values. Include -passing cases, failing cases, near misses, and exact source-position assertions. - -Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. - -### Helpers - -Custom Rules can import repository-owned helpers under `rules/` and supported pure Python modules. -Changing an imported helper invalidates the Rules that depend on it. - -Only decorated functions register. Ordinary functions, constants, dataclasses, and classes remain -helpers. - -### Deterministic inputs - -Direct filesystem, environment, clock, randomness, network, and subprocess access is rejected. -Those inputs cannot be reproduced safely by the Rules cache. - -Read supported project text and structure through `ctx.project.tree`: - -```python -text = ctx.project.tree.read_text("rules/requirements.yaml") -matches = ctx.project.tree.glob("models/*/interface/*.sql") -``` - -Both positive and negative observations are tracked. If `glob` returns no paths, adding a matching -path invalidates the cached result. - -### Cache granularity - -Cache identity incorporates the Rule implementation, imported helper closure, configured options, -subject, accessed compiler facts, tracked project observations, dialect, and compatibility -versions. Prefer model subjects for independent per-model checks and project subjects for genuine -cross-project invariants. - -## Overview - -Source: `concepts/python-nodes/overview.mdx` - -Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph. - -Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. - -There are four kinds, authored in dedicated top-level folders: - -| Kind | Decorator | Folder | Purpose | -|------|-----------|--------|---------| -| Loader | `@loader` | `loaders/` | Load external data into a managed source | -| Task | `@task` | `tasks/` | Run Python computation or side effects | -| Asset | `@asset` | `assets/` | Produce or observe an external artifact | -| Check | `@check` | `checks/` | Validate tasks, assets, and loaders | - -All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: - -- [Loaders](/concepts/python-nodes/loaders) - load data into managed sources -- [Tasks](/concepts/python-nodes/tasks) - Python computation and side effects -- [Assets](/concepts/python-nodes/assets) - external artifacts -- [Checks](/concepts/python-nodes/checks) - Python validations -- [Factories](/concepts/python-nodes/factories) - generate nodes programmatically with `@factory` -- [SQL references](/concepts/python-nodes/sql-references) - read SQL models and sources from Python - -Nodes can also be generated programmatically with [`@factory`](/concepts/python-nodes/factories) instead of authored one at a time. - -### The SQL boundary - -The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. - -- SQL models depend only on other SQL resources (models, sources, seeds, functions). -- Python nodes may depend on other Python nodes. -- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`. -- Python nodes may **read** SQL models and sources at runtime through typed references (see [SQL references](/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency. - -This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. - -``` - depends_on - task ----------------> task - | | - | (read only) v - | asset - v | - source <-- loader | check (validates tasks/assets/loaders) - | - v - model (SQL) ----> model (SQL) -``` - -### Decorators - -Every decorator accepts the same organizational metadata: - -| Argument | Description | -|----------|-------------| -| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()` references) | -| `tags` | Labels for selection, filtering, and catalog grouping | -| `group` | A display/catalog grouping string | -| `description` | Human-readable docs (defaults to the function docstring) | -| `meta` | Freeform JSON metadata for catalogs and integrations | - -`@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. - -### Identity tracking - -Every Python node is fingerprinted by source code hash, decorator config hash, and transitive dependency hashes (scoped to the git root, so third-party package updates don't affect identities). The plan shows source and dependency diffs when a node's identity changes, giving you visibility into what changed in your Python code. - -Unlike SQL models, Python nodes may depend on external inputs the framework cannot observe (APIs, files, third-party services). Skip/run decisions are therefore user-controlled via `ctx.skip()`: the node's own logic decides whether it needs to run. See [Planning and Change Detection](/concepts/planning#python-nodes) for details. - -### Runtime context - -Each node receives a context object as its first argument (`TaskContext`, `AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: - -| Helper | Description | -|--------|-------------| -| `ctx.run_id` | Unique identifier for this run | -| `ctx.target` | Active target name | -| `ctx.vars` | Project variables | -| `ctx.is_reload` | `True` when `--reload` was passed | -| `ctx.adapter` / `ctx.connection` | Adapter and live connection | -| `ctx.log(message)` | Log to the run output | -| `ctx.query(sql)` / `ctx.execute_sql(sql)` | Run SQL on the connection | -| `ctx.qualify_name(name)` | Qualify a relation name | -| `ctx.relation(ref)` | Resolve a declared `model()`/`source()` reference to a relation | -| `ctx.result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | -| `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `ctx.providers` | Access discovered [providers](/concepts/python-nodes/providers) by name | - -Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. - -Providers can also be injected directly as function parameters by name. See [Providers](/concepts/python-nodes/providers) for details. - -### Returns and skips - -Tasks and assets return through `ctx.result(...)`: - -```python -@task -def export_orders(ctx): - return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) -``` - -- A plain value or `None` is also accepted and normalized to a successful result. -- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. -- Assets may pass `materialized=True`/`False` to record whether an artifact was produced. - -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. - -### Result persistence - -Node results (payload, metadata, status, errors) are persisted after each execution. In direct mode, results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. In virtual mode, results are stored in the VDE state backend scoped per environment. Results persist across runs, so they are available for observability, debugging, and downstream consumption. - -### Selection - -Python nodes are selected like SQL resources, by bare name or typed selector: - -```bash -sqb build --select export_orders # bare name -sqb build --select task:export_orders # typed -sqb build --select asset:orders_export -sqb build --select check:check_orders -sqb build --select tag:exports # by tag -sqb build --select +orders_export # with upstreams -``` - -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, and hooks. - -### Lifecycle: run, build, check - -Python nodes run in two phases relative to SQL: - -- **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. -- **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. - -The commands differ in what they include by default: - -| Command | SQL | Loaders / tasks / assets | Checks | Audits | -|---------|-----|--------------------------|--------|--------| -| `sqb build` | Yes | Yes | Yes | Yes | -| `sqb build --no-tests --no-audits` | Yes | Yes | No | No | -| `sqb check` | No | No | Selected checks only | No | - -- `sqb build` is the complete build-and-validate command: it runs SQL, the required Python nodes, SQL audits, and Python checks. -- `sqb build --no-tests --no-audits` executes the DAG without validation, for fast iteration. -- `sqb check` runs Python checks only. See [`sqb check`](/cli/check). - -Use `--no-python` on `plan` and `build` to suppress read-side tasks/assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](/cli/build) reference. - -### Try it - -The `python_nodes` playground is a small working project with a task, loader, model, asset, and check: - -```bash -sqb playground --template python_nodes -cd sqlbuild-playground -sqb build --select +fact_orders --select +orders_export -sqb check --select +check_orders_export -``` - -See [`sqb playground`](/cli/playground). - -## Loaders - -Source: `concepts/python-nodes/loaders.mdx` - -Load external data into source tables with Python functions. - -Loaders are Python functions that load data into source tables. They replace expression sources and manual ETL scripts with code that lives inside your project, runs as part of the build, and supports incremental write strategies. Loaders are one of the four [Python node](/concepts/python-nodes/overview) kinds, and the only one that writes into a SQL source. - -### How it works - -1. Write a Python function under `loaders/` decorated with `@loader` -2. Declare a managed source in `sources/*.yml` with `managed: true` and **the same name as the loader function** -3. SQLBuild calls the function, writes returned rows to a staging table, then applies the configured write strategy to the target - -Loaders participate in the build lifecycle. When `sqb build` runs, managed sources are loaded before any dependent model is materialized. - -### Defining a loader - -Place Python files under `loaders/` in your project directory. Each file can contain one or more loader functions: - -```python -# loaders/raw_sources.py -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: - return [ - {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, - {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, - ] -``` - -The function receives a `LoaderContext` and returns rows as a list of dicts, an iterator of dicts, or `None` for self-managed loaders. - -### Binding to a source - -Declare a managed source in `sources/*.yml`. A managed source is bound to the loader function **with the same name** - there is no separate `loader` field: - -```yaml -sources: - - name: raw_customers - managed: true - write_strategy: table - columns: - - name: id - type: INTEGER - - name: name - type: VARCHAR - - name: email - type: VARCHAR -``` - -Setting `managed: true` makes this a **managed source** - SQLBuild owns both the loading and the schema. The binding is by name: the source `raw_customers` is populated by the `@loader` function named `raw_customers`. SQLBuild raises an error if a managed source has no loader function of the same name. - -Models reference managed sources the same way as any other source: - -```sql -SELECT id, name FROM __source("raw_customers") -``` - -### Write strategies - -The `write_strategy` field controls how returned rows are written to the target table. - -#### table - -Full replace. The target is dropped and recreated from the loader output on every run. - -```yaml -sources: - - name: raw_countries - managed: true - write_strategy: table - columns: - - name: country_id - type: INTEGER - - name: country_code - type: VARCHAR -``` - -#### append - -Insert all returned rows into the target. No deduplication. - -```yaml -sources: - - name: raw_webhook_events - managed: true - write_strategy: append - columns: - - name: event_id - type: INTEGER - - name: event_name - type: VARCHAR -``` - -#### delete_insert - -Delete rows in the cursor range, then insert replacements. Requires `cursor_column`. - -```yaml -sources: - - name: raw_order_events - managed: true - write_strategy: delete_insert - cursor_column: event_at - columns: - - name: event_id - type: INTEGER - - name: event_at - type: TIMESTAMP - - name: amount_cents - type: INTEGER -``` - -The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_column)` from the target, so it can fetch only new or updated data. Its function name matches the source name (`raw_order_events`): - -```python -@loader -def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: - if ctx.current_cursor_value is None: - return fetch_all_events() - return fetch_events_since(ctx.current_cursor_value) -``` - -#### merge - -Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. - -```yaml -sources: - - name: raw_customers - managed: true - write_strategy: merge - unique_key: customer_id - cursor_column: updated_at - columns: - - name: customer_id - type: INTEGER - - name: plan_name - type: VARCHAR - - name: updated_at - type: TIMESTAMP -``` - -Existing rows matching the unique key are updated; new rows are inserted. - -### Self-managed loaders - -If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader is responsible for writing data to the target itself, using whatever approach makes sense - `ctx.execute_sql()`, an external library, a subprocess, or anything else: - -```python -@loader -def raw_status(ctx: LoaderContext) -> None: - ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") - ctx.execute_sql( - f"CREATE TABLE {ctx.destination} AS " - "SELECT 1 AS status_id, 'loaded' AS status_name" - ) -``` - -The source is still declared as managed, just without a `write_strategy`: - -```yaml -sources: - - name: raw_status - managed: true - columns: - - name: status_id - type: INTEGER - - name: status_name - type: VARCHAR -``` - -Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool like [dlt](/integrations/dlt), or handle writes in a way that doesn't fit the dict-return pattern. - -### Loader context - -Every loader function receives a `LoaderContext` as its first argument. It provides access to the destination relation, cursor state, active target, and helper methods. - -#### Properties - -| Property | Type | Description | -|----------|------|-------------| -| `destination` | `str` | Fully-qualified destination relation name (where rows are written) | -| `destination_database` | `str \| None` | Destination database | -| `destination_schema` | `str \| None` | Destination schema | -| `destination_name` | `str` | Unqualified destination table name | -| `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | -| `run_id` | `str` | Unique identifier for this execution run | -| `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | -| `vars` | `dict` | Project variables (merged from project, target, and local config) | -| `is_reload` | `bool` | `True` when `--reload` was passed | -| `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | -| `end_cursor_ts` | `datetime \| None` | Timestamp cursor end override from `--end-cursor-ts` | -| `start_cursor_int` | `int \| None` | Integer cursor start override from `--start-cursor-int` | -| `end_cursor_int` | `int \| None` | Integer cursor end override from `--end-cursor-int` | -| `adapter` | `BaseAdapter` | The database adapter instance | -| `connection` | `object` | The active database connection | -| `logger` | `Logger` | Python logger scoped to the loader | - -#### Methods - -| Method | Description | -|--------|-------------| -| `execute_sql(sql)` | Execute a SQL statement against the connection | -| `query(sql)` | Execute a SQL query and return the cursor | -| `log(message)` | Log a message to the execution lifecycle output | -| `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | -| `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | -| `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | -| `result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | -| `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency | -| `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | - -#### LoaderRelationRef - -Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream relation: - -| Property / Method | Description | -|-------------------|-------------| -| `destination` | Fully-qualified relation name | -| `current_cursor_value` | Current `MAX(cursor_column)` from the relation | -| `max(column)` | Return the `MAX` of any column from the relation | - -### Loader dependencies - -Loaders can depend on other loaders using `depends_on`. Dependencies are executed first, and their destination relations are available via `ctx.loader()`: - -```python -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: - return [ - {"account_id": 1, "account_name": "Pawnee Parks"}, - {"account_id": 2, "account_name": "Eagleton"}, - ] - -@loader(depends_on=[raw_accounts]) -def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: - accounts = ctx.loader(raw_accounts) - rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") - return [ - {"account_id": row[0], "metric": "active"} - for row in rows.fetchall() - ] -``` - -Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. - -Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: - -```python -@loader(destination="staging.shared_accounts") -def raw_accounts(ctx: LoaderContext): - ... -``` - -### Decorator parameters - -The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence. - -| Parameter | Description | -|-----------|-------------| -| `depends_on` | List of loader functions this loader depends on | -| `destination` | Override the destination relation name (can include schema or database) | -| `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | -| `cursor_column` | Column used for incremental cursor tracking | -| `unique_key` | Column(s) used as the merge key (string or list of strings) | -| `columns` | Column specifications with name, type, nullable, and description | -| `contract` | `enforced` or `none` | - -### Auto-load during builds - -By default, `sqb build` automatically loads managed sources before building dependent models. This is controlled by the `auto_load_sources` setting: - -```toml -[settings] -auto_load_sources = true # default -``` - -You can also control this per-run with CLI flags: - -```bash -# Explicitly load sources before building -sqb build --load - -# Skip source loading -sqb build --no-load - -# Reload sources (passes is_reload=True to loaders) -sqb build --reload -``` - -When `--reload` is passed, `ctx.is_reload` is `True` in the loader function. This lets loaders implement different behavior for full reloads versus normal incremental loads. - -### Source deferral - -Loader writes and managed source reads are resolved separately. `loader_schema` controls -where a target's loaders write; `defer_sources_to` optionally selects another target whose -managed sources models should read. - -```toml -[targets.dev] -schema = "analytics_dev" -loader_schema = "raw_dev" -defer_sources_to = "prod" - -[targets.prod] -schema = "analytics_prod" -loader_schema = "raw_prod" -``` - -With this config: - -- `sqb load --target dev` writes to `raw_dev`. -- `sqb load --target prod` writes to `raw_prod`. -- Models built in dev read from `raw_prod` because dev defers source reads to prod. -- Models built in prod read from `raw_prod` because omitted deferral defaults to the active target. - -If `loader_schema` is omitted, loaders fall back to the active target's model `schema`, then -the adapter default schema. An explicit `schema` on a managed source overrides the target -default. SQLBuild validates final managed-loader write namespaces and rejects two targets -on the same warehouse/database that could write to the same schema. - -`defer_sources_to` never redirects loader writes. It only changes `__source()` and Python -`source()` reads. This lets developers test loaders in an isolated schema without making -development models consume partial development source data. - -### Schema evolution - -When a loader returns rows with columns not present in the existing target table, SQLBuild detects the schema change and adds the new columns automatically. Type mismatches between the staging table and the existing target raise an error. - -### Project structure - -``` -my-project/ - loaders/ - raw_sources.py # loader functions - api_sources.py # more loader functions - sources/ - raw.yml # managed source declarations (managed: true) - models/ - staging/ - stg_customers.sql # __source("raw_customers") -``` - -SQLBuild discovers all `.py` files under `loaders/` recursively (excluding `__init__.py` and files starting with `_`). Each file is scanned for functions decorated with `@loader`. - -### Config reference - -#### Source YAML fields for managed sources - -| Field | Description | -|-------|-------------| -| `managed` | Set to `true` to bind the source to the `@loader` function of the same name | -| `write_strategy` | `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | -| `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | -| `unique_key` | Merge key column(s) (required for `merge`) | -| `columns` | Column declarations with types | -| `contract` | `enforced` or `none` | - -#### Validation rules - -- `append` cannot have `unique_key` -- `merge` requires `unique_key` -- `table` cannot have `cursor_column` or `unique_key` -- `delete_insert` requires `cursor_column` and cannot have `unique_key` -- `cursor_column` requires one of `append`, `delete_insert`, or `merge` - -## Tasks - -Source: `concepts/python-nodes/tasks.mdx` - -Run Python computation and side effects as nodes in the SQLBuild graph. - -Tasks are Python functions that run as part of the DAG. Use them for computation, side effects, and orchestration steps that are not loading data into a source and not producing a tracked artifact. See [Python Nodes](/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. - -### Defining a task - -Place Python files under `tasks/` and decorate functions with `@task`: - -```python -# tasks/orders.py -from sqlbuild.tasks import task, TaskContext - -@task -def export_orders(ctx: TaskContext): - rows = fetch_orders() - return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) -``` - -The task receives a `TaskContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. - -### Dependencies - -Tasks declare dependencies with `depends_on`, accepting a single function, a tuple, or a list: - -```python -@task -def fetch_orders(ctx: TaskContext): - return ctx.result(payload=download_orders()) - -@task(depends_on=fetch_orders) -def summarize_orders(ctx: TaskContext): - result = ctx.result_of(fetch_orders) - return ctx.result(metadata={"count": len(result.payload)}) -``` - -`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. - -Tasks may depend on other tasks, assets, and loaders. They may **not** depend on SQL models or sources as graph dependencies, but they can read them at runtime with typed references - see [SQL references](/concepts/python-nodes/sql-references). - -### Returning results - -```python -@task -def build_export(ctx: TaskContext): - return ctx.result( - payload={"path": "/exports/orders.csv"}, - metadata={"rows": 1200}, - ) -``` - -- `payload` is the value downstream nodes read with `ctx.result_of(...)`. -- `metadata` is structured JSON for catalogs and downstream reads. -- Tasks cannot set `materialized` - that is for [assets](/concepts/python-nodes/assets). - -### Skipping - -Return `ctx.skip(...)` to skip a task: - -```python -@task -def export_if_present(ctx: TaskContext): - if not new_files_available(): - return ctx.skip("no new files") - return ctx.result(payload=do_export()) -``` - -- `"soft"` (default) skips only this task; dependents may still run if another upstream succeeded. -- `"hard"` skips this task and blocks its dependents. - -`mode` accepts either a plain string or the `SkipMode` enum: - -```python -from sqlbuild.tasks import task, SkipMode - -@task -def optional_step(ctx): - return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" -``` - -### Retries - -`@task` accepts a `retry` policy for transient failures: - -```python -from sqlbuild.retries import RetryPolicy -from sqlbuild.tasks import task - -@task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) -def call_api(ctx): - return ctx.result(payload=fetch_from_flaky_api()) -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `max_attempts` | `3` | Total attempts including the first | -| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | -| `initial_delay_seconds` | `1.0` | Delay before the first retry | -| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | -| `max_delay_seconds` | `30.0` | Cap on any single delay | -| `max_elapsed_seconds` | `None` | Overall time bound for retries | -| `jitter` | `True` | Randomize delays to avoid synchronized retries | - -The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. - -### Decorator parameters - -| Parameter | Description | -|-----------|-------------| -| `name` | Override the node name (defaults to the function name) | -| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs | -| `tags` | Labels for selection and grouping | -| `group` | Display/catalog grouping | -| `description` | Docs (defaults to docstring) | -| `meta` | Freeform JSON metadata | -| `retry` | A `RetryPolicy` | - -### Running tasks - -```bash -# Run a task and its required graph -sqb build --select export_orders --no-tests --no-audits - -# Include in a full build -sqb build --select export_orders -``` - -Tasks run during `sqb build`. They are not validated by checks unless you also write a [check](/concepts/python-nodes/checks) that depends on them. - -## Assets - -Source: `concepts/python-nodes/assets.mdx` - -Produce or observe external artifacts as nodes in the SQLBuild graph. - -Assets are Python nodes that produce or observe an external artifact - an exported file, a published dataset, a dashboard refresh, an ML model. They behave like [tasks](/concepts/python-nodes/tasks) but add dataset-like metadata (columns, column lineage) and a materialization flag. See [Python Nodes](/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. - -### Defining an asset - -Place Python files under `assets/` and decorate functions with `@asset`: - -```python -# assets/exports.py -from sqlbuild.assets import asset, AssetContext - -@asset -def orders_export(ctx: AssetContext): - path = write_orders_csv() - return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) -``` - -The asset receives an `AssetContext` and returns through `ctx.result(...)`. - -### Materialization - -Assets record whether they actually produced an artifact via `materialized`: - -```python -@asset -def orders_export(ctx: AssetContext): - if nothing_changed(): - return ctx.result(metadata={"status": "unchanged"}, materialized=False) - return ctx.result(metadata={"path": export()}, materialized=True) -``` - -- `materialized=True` - the artifact was produced this run. -- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. - -Only assets have `materialized`; tasks do not. - -### Dependencies - -Assets declare dependencies with `depends_on` (a single function, tuple, or list), and may depend on tasks, assets, and loaders: - -```python -from sqlbuild.assets import asset -from tasks.orders import export_orders - -@asset(depends_on=export_orders) -def orders_dashboard(ctx): - result = ctx.result_of(export_orders) - return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) -``` - -To read a SQL model or source, declare a typed reference and resolve it at runtime - see [SQL references](/concepts/python-nodes/sql-references): - -```python -from sqlbuild.assets import asset -from sqlbuild.refs import model - -@asset(depends_on=model("fact_orders")) -def orders_extract(ctx): - relation = ctx.relation(model("fact_orders")) - rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] - return ctx.result(metadata={"rows": rows}, materialized=True) -``` - -### Columns and column lineage - -Assets can declare a schema for catalog and lineage purposes. This does not enforce anything at runtime; it describes the artifact: - -```python -@asset( - columns=[ - {"name": "order_id", "type": "INTEGER"}, - {"name": "total_cents", "type": "INTEGER", "nullable": False}, - ], - column_lineage={ - "order_id": [{"node": "fact_orders", "column": "order_id"}], - "total_cents": [{"node": "fact_orders", "column": "total_cents"}], - }, -) -def orders_export(ctx): - ... -``` - -- `columns` - column declarations with `name`, optional `type`, `nullable`, `description`, and `meta`. -- `column_lineage` - maps each asset column to upstream `{node, column}` references, surfaced in the DAG artifact and integrations. - -Tasks and checks do not support `columns` or `column_lineage`. - -### Skipping and retries - -Assets support the same `ctx.skip(...)` and `retry` behavior as tasks: - -```python -from sqlbuild.assets import asset, SkipMode -from sqlbuild.retries import RetryPolicy - -@asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) -def export(ctx): - if not ready(): - return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT - return ctx.result(metadata={"path": do_export()}, materialized=True) -``` - -See [Tasks](/concepts/python-nodes/tasks#retries) for the full retry policy fields. - -### Decorator parameters - -| Parameter | Description | -|-----------|-------------| -| `name` | Override the node name (defaults to the function name) | -| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs | -| `columns` | Column declarations for the produced artifact | -| `column_lineage` | Map of asset column to upstream `{node, column}` references | -| `tags` | Labels for selection and grouping | -| `group` | Display/catalog grouping | -| `description` | Docs (defaults to docstring) | -| `meta` | Freeform JSON metadata | -| `retry` | A `RetryPolicy` | - -### Running assets - -```bash -# Run an asset and its required graph -sqb build --select orders_export --no-tests --no-audits - -# Include in a full build with its upstreams -sqb build --select +orders_export -``` - -Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. - -## Checks - -Source: `concepts/python-nodes/checks.mdx` - -Validate tasks, assets, and loaders with Python checks. - -Checks are Python nodes that validate other Python nodes. They are the Python analog of SQL [audits](/concepts/audits): audits validate SQL relations, checks validate the output of tasks, assets, and loaders. See [Python Nodes](/concepts/python-nodes/overview) for the shared model. - -Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`. - -### Defining a check - -Place Python files under `checks/` and decorate functions with `@check`. `depends_on` is required: - -```python -# checks/orders.py -from sqlbuild.checks import check, CheckContext -from tasks.orders import export_orders - -@check(depends_on=export_orders) -def check_orders_exported(ctx: CheckContext): - result = ctx.result_of(export_orders) - if result.metadata.get("rows", 0) == 0: - return ctx.fail("no orders exported") - return ctx.pass_("orders exported") -``` - -The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. - -### Results - -Return a result through the context helpers, or a bool shorthand: - -```python -@check(depends_on=orders_asset) -def rows_present(ctx): - return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail -``` - -| Return | Meaning | -|--------|---------| -| `ctx.pass_(message=None, metadata=None)` | Passing | -| `ctx.fail(message, metadata=None)` | Failing, using the check's severity | -| `ctx.warn(message, metadata=None)` | Warning, regardless of severity | -| `True` | Pass | -| `False` | Fail | - -Returning `None` is not allowed - checks must be explicit. - -### Severity - -`@check` takes a `severity` of `error` (default) or `warn`: - -```python -@check(depends_on=export_orders, severity="warn") -def orders_freshness(ctx): - if stale(): - return ctx.fail("export is stale") # recorded as a warning, does not fail the build - return ctx.pass_() -``` - -- `error` (default) - a failing check fails `sqb build`. -- `warn` - a failing check is reported but does not fail the build. - -`ctx.warn(...)` always produces a warning regardless of the declared severity. - -### What checks can depend on - -- Checks may depend on **tasks, assets, and loaders**. -- Checks may **not** depend on SQL models, sources, seeds, or functions. Use SQL [audits](/concepts/audits) to validate SQL relations. -- Checks may **not** depend on other checks. -- Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead. - -A check that depends on a single node is displayed grouped under that node. Multi-dependency checks are shown as standalone validation nodes, grouped by `group`, tags, or path. - -### Decorator parameters - -| Parameter | Description | -|-----------|-------------| -| `depends_on` | Required. Tasks/assets/loaders to validate (function, tuple, or list) | -| `name` | Override the node name (defaults to the function name) | -| `severity` | `error` (default) or `warn` | -| `tags` | Labels for selection and grouping | -| `group` | Display/catalog grouping | -| `description` | Docs (defaults to docstring) | -| `meta` | Freeform JSON metadata | - -Checks do not support `columns`, `column_lineage`, or `retry`. - -### Running checks - -Checks run automatically during `sqb build` when their Python dependencies run. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/cli/check): - -```bash -# Run all checks -sqb check - -# Run a specific check (and its required dependencies) -sqb check --select +check_orders_exported - -# Run checks by tag -sqb check --select tag:exports -``` - -`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. - -### Checks vs audits - -| | Checks | Audits | -|---|--------|--------| -| Validates | Python tasks, assets, loaders | SQL relations | -| Authored in | `checks/` (Python) | `MODEL()` headers / `audits/` (SQL) | -| Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | -| Severity | `error`, `warn` | `error`, `warn` | - -`sqb build` runs both. `sqb audit` runs SQL audits only; `sqb check` runs Python checks only. - -## Factories - -Source: `concepts/python-nodes/factories.mdx` - -Generate Python nodes programmatically with @factory. - -A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [tasks](/concepts/python-nodes/tasks), [assets](/concepts/python-nodes/assets), [loaders](/concepts/python-nodes/loaders), or [checks](/concepts/python-nodes/checks) from a list, a config, or a loop, rather than hand-writing each one. - -Factories are an advanced feature. Most projects author nodes directly; reach for a factory when you find yourself copy-pasting near-identical node definitions. - -### Defining a factory - -A `@factory` function takes **no arguments** and returns one or more decorated node functions: - -```python -# factories/exports.py -from sqlbuild.factories import factory -from sqlbuild.assets import asset - -TABLES = ["orders", "customers", "payments"] - -@factory -def export_assets(): - nodes = [] - for table in TABLES: - @asset(name=f"export_{table}", tags=("export",)) - def export(ctx, table=table): - return ctx.result(metadata={"table": table}, materialized=True) - nodes.append(export) - return nodes -``` - -The returned nodes are discovered and added to the graph exactly as if you had written them by hand. They participate in selection, lifecycle, the DAG artifact, and integrations like any other node. - -A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. - -### Folder rules - -Where a factory lives determines what it is allowed to emit: - -| Location | May emit | -|----------|----------| -| `loaders/` | loaders only | -| `tasks/` | tasks only | -| `assets/` | assets only | -| `checks/` | checks only | -| `factories/` | any kind, including a mix | - -- A **single-kind** factory can live in that kind's folder (e.g. an asset-only factory in `assets/`), keeping it next to the nodes it generates. It can also live in `factories/`. -- A factory that emits **more than one kind** must live in `factories/`. - -This keeps each kind folder honest: everything in `assets/`, whether hand-written or factory-generated, is an asset. - -SQLBuild enforces this at discovery time. A factory in a kind folder that returns a foreign kind raises an error pointing you to `factories/`: - -``` -Factory export_pipeline in assets/ returned a loader 'raw_orders'; -mixed-kind factories must live in factories/. -``` - -The `factories/` folder is not created by `sqb init` - add it when you need it. - -### Mixed-kind factories - -A factory in `factories/` can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: - -```python -# factories/orders.py -from sqlbuild.factories import factory -from sqlbuild.loaders import loader -from sqlbuild.assets import asset -from sqlbuild.checks import check - -@factory -def orders_pipeline(): - @loader(name="raw_orders") - def load(ctx): - return fetch_orders() - - @asset(name="orders_export", depends_on=load) - def export(ctx): - return ctx.result(materialized=True) - - @check(depends_on=export) - def orders_export_check(ctx): - return ctx.pass_("export ready") - - return [load, export, orders_export_check] -``` - -Generated nodes follow the same rules as directly-authored ones, including the [SQL boundary](/concepts/python-nodes/overview#the-sql-boundary): a factory-generated loader still binds to a managed source, and a factory-generated check still may not depend on SQL models. - -### Naming - -Generated nodes need unique names across the project. Pass an explicit `name=` to each node a factory creates (factories almost always generate names from a loop variable or config), since relying on the function's own name would produce duplicates. - -## Providers - -Source: `concepts/python-nodes/providers.mdx` - -Shared runtime services for Python nodes and hooks. - -Providers are shared runtime services that SQLBuild discovers, configures, and injects into your Python nodes and hooks. Use them for external connections, API clients, or any stateful service that multiple nodes need access to. - -### Defining a provider - -Create a Python file under `providers/` in your project. A provider is a class that subclasses `Provider` from `sqlbuild.providers`: - -```python -# providers/warehouse_client.py -from sqlbuild.providers import Provider - -class WarehouseClient(Provider): - api_key: str - endpoint: str = "https://api.example.com" - - def setup(self, ctx): - self.session = create_session(self.api_key, self.endpoint) - - def teardown(self): - self.session.close() -``` - -`Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. - -### Provider name - -Each provider has a runtime name used for injection. By default, the name is derived from the class name by converting to `lower_snake_case`: - -- `WarehouseClient` becomes `warehouse_client` -- `SlackNotifier` becomes `slack_notifier` - -Override the name explicitly with `provider_name`: - -```python -class WarehouseClient(Provider): - provider_name = "warehouse" - api_key: str -``` - -Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and hooks. - -### Using providers in Python nodes - -Providers are injected into Python node functions by **parameter name**. Add a parameter whose name matches the provider's runtime name: - -```python -from sqlbuild.tasks import task - -@task -def export_orders(ctx, warehouse_client): - warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) -``` - -SQLBuild matches the parameter name `warehouse_client` to the discovered provider with that name, sets it up if it hasn't been already, and passes it to the function. - -You can also type-annotate the parameter for IDE support and compile-time validation: - -```python -from sqlbuild.tasks import task -from providers.warehouse_client import WarehouseClient - -@task -def export_orders(ctx, warehouse_client: WarehouseClient): - warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) -``` - -When a type annotation is present, SQLBuild validates that the discovered provider is an instance of the annotated class. A mismatch raises a compile-time error. - -Provider injection works in all Python node types: - -- **Loaders** (`@loader`) -- **Tasks** (`@task`) -- **Assets** (`@asset`) -- **Checks** (`@check`) - -### Using providers in hooks - -Python lifecycle hooks also support provider injection by parameter name: - -```python -# hooks/python/notify.py -from sqlbuild.hooks import hook - -@hook -def notify_complete(ctx, slack_notifier): - slack_notifier.send(f"Model {ctx.model_name} built successfully") -``` - -```sql -MODEL ( - materialized table, - post_hooks [python("notify_complete")], -); -``` - -Providers are also available on the `HookContext` via `ctx.providers`: - -```python -@hook -def notify_complete(ctx): - notifier = ctx.providers.slack_notifier - notifier.send(f"Model {ctx.model_name} built successfully") -``` - -### Using providers via context - -All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: - -```python -@task -def export_orders(ctx): - client = ctx.providers.warehouse_client - client.session.upload(ctx.query("SELECT * FROM orders")) -``` - -Both approaches (parameter injection and `ctx.providers`) are equivalent. Parameter injection is more explicit and enables type checking; `ctx.providers` is useful when provider access is conditional or dynamic. - -### Lifecycle - -Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: - -1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). -2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. -3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. - -```python -class WarehouseClient(Provider): - api_key: str - - def setup(self, ctx): - # Called once, the first time any node accesses this provider - self.connection = connect(self.api_key) - - def teardown(self): - # Called after the command completes - self.connection.close() -``` - -Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. - -### Configuration from environment variables - -Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: - -```python -class SlackNotifier(Provider): - slack_token: str # reads SLACK_TOKEN from environment - channel: str = "#builds" # has a default, environment variable is optional -``` - -See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. - -### Discovery rules - -- Provider classes are discovered from `.py` files under `providers/` recursively -- Files named `__init__.py` or starting with `_` are skipped -- Each concrete (non-abstract) subclass of `Provider` is registered -- Provider names must be globally unique across project resources -- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime - -### Plan output - -When providers are used by Python nodes or hooks, `sqb plan` shows a Providers section listing each provider and its consumers: - -``` -Providers - warehouse_client 2 nodes - slack_notifier 1 node -``` - -Use `--verbose` to see which specific nodes consume each provider. - -### Project layout - -``` -my-project/ - providers/ - warehouse_client.py - slack_notifier.py - hooks/ - sql/ - record_notification.sql - python/ - notify.py - loaders/ - load_orders.py - tasks/ - export_orders.py -``` - -## SQL References - -Source: `concepts/python-nodes/sql-references.mdx` - -Read SQL models and sources from Python nodes without creating SQL dependencies. - -Python nodes can **read** SQL models and sources at runtime, but they cannot depend on them as graph edges (see the [SQL boundary](/concepts/python-nodes/overview#the-sql-boundary)). Typed references make this read-only access explicit and safe across direct and virtual modes. - -### Declaring a reference - -Use `model()` and `source()` from `sqlbuild.refs` in a node's `depends_on`: - -```python -from sqlbuild.refs import model, source -from sqlbuild.tasks import task - -@task(depends_on=model("fact_orders")) -def export_orders(ctx): - ... - -@task(depends_on=source("raw_orders")) -def inspect_raw(ctx): - ... -``` - -Declaring the reference does two things: - -1. It tells SQLBuild the node reads that SQL resource, so the node is scheduled **after** the resource is built (read-side). -2. It authorizes `ctx.relation(...)` to resolve that reference at runtime. - -It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse. - -### Resolving with ctx.relation - -Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name: - -```python -@task(depends_on=model("fact_orders")) -def export_orders(ctx): - relation = ctx.relation(model("fact_orders")) - rows = ctx.query(f"SELECT * FROM {relation}").fetchall() - return ctx.result(metadata={"rows": len(rows)}) -``` - -`ctx.relation(...)` returns the adapter-qualified relation name for the current run. Passing a reference that was not declared in `depends_on` raises an error. - -Do not hardcode model or source names in raw SQL. In virtual mode a model resolves to a versioned VDE relation (for example `marts__feature_x.fact_orders`), not the bare name. A query like `ctx.query("SELECT * FROM fact_orders")` will fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`. - -### model vs source - -| Reference | Resolves to | -|-----------|-------------| -| `model("name")` | The built model relation (the VDE logical relation in virtual mode) | -| `source("name")` | The source read relation, following deferred-source semantics | - -`source(...)` respects `defer_sources_to`, so a node reading a source in `dev` can read the deferred target's data just like SQL models do. - -### Where references are allowed - -- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`. -- **Checks** may not reference SQL resources. Validate SQL with [audits](/concepts/audits). -- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. - -## Overview - -Source: `concepts/virtual-environments.mdx` - -Version-controlled SQL pipeline environments with instant promotion and rollback. - -Virtual environments are in alpha. The core workflow (build, promote, rollback, reconcile) is functional and tested across supported adapters. The API and CLI surface may evolve based on feedback. Do not use virtual environments for production workloads yet. - -Virtual environments (VDEs) let you build, preview, and promote SQL pipeline changes without recomputing models. Each VDE is a set of pointers to versioned physical relations. Creating a new VDE is instant (pointer copy, no data duplication), and promoting one VDE into another is a view swap, not a rebuild. - -### When to use virtual environments - -- **PR preview environments** - build a VDE per pull request against a production warehouse, validate with audits and tests, then promote the built versions to production without rebuilding -- **Blue/green deployments** - build into a staging VDE, promote to production atomically -- **Multi-developer isolation** - each developer works in their own VDE without conflicting with others, sharing physical versions when code is identical -- **Instant rollback** - revert production to a prior finalized state by restoring a checkpoint's pointer set - -Virtual environments are opt-in via `virtual_environments = true` (under `[settings]`) and require a state store. Projects that don't need environment isolation or promotion workflows should use the default direct mode. - -### How it works - -#### Targets and virtual environments - -In virtual mode there are two separate concepts: - -**Targets** are the existing SQLBuild build contexts from `sqlbuild_project.toml` (e.g. `dev`, `prod`). They choose the warehouse connection, schema, and state database. In virtual mode they are sometimes called *physical targets* to distinguish them from VDEs. - -**Virtual data environments (VDEs)** are versioned pointer sets stored in the state database. They choose which model versions the logical views point to. - -```bash -sqb --target prod build --virtual-env pr_123 -``` - -This means: use the `prod` physical target (warehouse, state DB), but build into the `pr_123` virtual environment. - -#### Version identity - -Model versions are identified by content hashes, not sequence numbers. The hash includes the model's query SQL, version-identity config, and upstream version hashes. If two developers compile identical code with identical upstream versions, they get the same hash and reuse the same physical relation. No data is duplicated. - -#### Physical and logical relations - -Virtual mode creates two types of warehouse objects: - -- **Physical version relations** store actual data: `__sqb_physical.__v_` -- **Logical VDE views** point to physical versions: `__.` as `SELECT * FROM ` - -Users query the logical views. SQLBuild manages the physical layer. - -#### Zero-copy branching - -Creating a new VDE from a baseline copies only pointer rows in the state database, not data. Unchanged models share the same physical relations across VDEs. - -#### Instant promotion - -Promoting VDE `pr_123` to `prod` updates the pointer rows and refreshes the logical views. No models are rebuilt. If every model in the source VDE has already been built and validated, promotion is a metadata operation. - -### Try it - -```bash -sqb playground --template virtual -cd sqlbuild-playground -sqb state init -sqb build -sqb build --virtual-env pr -sqb diff dev:pr --schema-only --allow-partial-diff -sqb promote --from pr --to dev -sqb state checkpoints list -sqb rollback -``` - -This creates a self-contained DuckDB project with loaders, models, tests, scenarios, and a local state store. No warehouse credentials required. - -### Example workflow - -```bash -# Initialize state store -sqb state init - -# First build creates physical versions and default VDE -sqb build - -# Create a PR preview VDE -sqb build --virtual-env pr_123 - -# Modify a model, rebuild in the PR VDE -sqb build --virtual-env pr_123 - -# Compare VDEs -sqb diff dev:pr_123 - -# Promote PR versions to the default VDE -sqb promote --from pr_123 --to dev -``` - -### What's next - -- [Setup](/concepts/virtual-environments/setup) - configuration and state initialization -- [Building](/concepts/virtual-environments/building) - virtual builds, partial builds, seeded incrementals -- [Promotion](/concepts/virtual-environments/promotion) - promoting VDEs -- [Rollback](/concepts/virtual-environments/rollback) - checkpoints and rollback -- [Clone](/concepts/virtual-environments/clone) - hydrating physical versions from a source warehouse -- [Diff](/concepts/virtual-environments/diff) - comparing VDE ref sets -- [Adopt and Detach](/concepts/virtual-environments/adopt-detach) - migrating existing projects -- [Reconcile](/concepts/virtual-environments/reconcile) - diagnosing and repairing drift -- [Locks](/concepts/virtual-environments/locks) - concurrent access control -- [Janitor](/concepts/virtual-environments/janitor) - cleanup and retention -- [Recovery](/concepts/virtual-environments/recovery) - what to do when things break - -## Setup - -Source: `concepts/virtual-environments/setup.mdx` - -Configure virtual environments and initialize the state store. - -Virtual environments require two things: `virtual_environments = true` in your project settings, and a state store configured for your active target. - -### Project configuration - -```toml -name = "my_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connections.local] -database = "warehouse.duckdb" - -[targets.dev] -connection = "local" -schema = "dev" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -``` - -The `virtual_environments` setting switches the project from direct mode (default) to virtual mode. All state, plan, build, promote, rollback, and reconcile commands route through the virtual path when this is enabled. - -### State configuration - -Each physical target that uses virtual mode needs a `[targets..state]` block: - -| Field | Required | Description | -|-------|----------|-------------| -| `backend` | Yes | State store engine: `duckdb` or `postgres` | -| `schema` | Yes | Schema name for state tables | -| `connection` | Yes | Backend-specific connection config | -| `allow_reset` | No | Whether `sqb state reset` is permitted (default: `false`) | -| `unsuffixed_virtual_env` | No | VDE name that uses the base schema without a suffix (for [adopt/detach](/concepts/virtual-environments/adopt-detach)) | - -#### DuckDB state backend - -```toml -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -``` - -DuckDB state stores are file-based. Relative paths are resolved against the project directory. Suitable for local development and single-user workflows. Not recommended when multiple processes or CI jobs need concurrent state access. - -#### Postgres state backend - -```toml -[targets.prod.state] -backend = "postgres" -schema = "sqlbuild_state" - -[targets.prod.state.connection] -host = "state-db.internal" -port = 5432 -user = "sqlbuild_state" -password = "${ENV:STATE_DB_PASSWORD}" -dbname = "sqlbuild_state" -``` - -Postgres is recommended for production. It supports concurrent access from multiple developers or CI jobs. - -### State initialization - -Before using virtual commands, initialize the state store: - -```bash -sqb state init -``` - -This creates the state tables in the configured schema. Run it once per physical target. - -### State lifecycle - -| Command | Description | -|---------|-------------| -| `sqb state init` | Create state tables | -| `sqb state migrate` | Back up current state and re-initialize tables | -| `sqb state rollback` | Restore from the latest backup | -| `sqb state rollback --backup-id ` | Restore from a specific backup | -| `sqb state reset --auto-approve` | Drop all state tables (requires `allow_reset = true`) | - -`state migrate` creates a backup schema (e.g. `sqlbuild_state__backup_`) before re-initializing. This lets you roll back if a migration causes problems. - -`state reset` is destructive and requires both `allow_reset = true` in config and `--auto-approve` on the command line. - -### Local overrides - -Use `sqlbuild_local.toml` to override state connection config per developer: - -```toml -[targets.dev.state.connection] -database = "local-state.duckdb" -``` - -This is useful when each developer uses a local state file while the project config points to a shared state database. - -### Per-target state - -Different physical targets can use different state backends: - -```toml -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" - -[targets.prod.state] -backend = "postgres" -schema = "sqlbuild_state" - -[targets.prod.state.connection] -host = "prod-state.internal" -dbname = "sqlbuild_state" -``` - -Each physical target has its own state store. VDEs are scoped to their physical target and cannot be promoted or compared across different physical targets. - -### Unsuffixed VDE naming - -By default, all VDE logical views use a suffixed schema: `dev__kevin.fact_orders`. For the primary VDE that consumers query directly (e.g. the production VDE), you typically want clean unsuffixed names: `dev.fact_orders`. - -Configure this with `unsuffixed_virtual_env`: - -```toml -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -unsuffixed_virtual_env = "dev" - -[targets.dev.state.connection] -database = "state.duckdb" -``` - -With this config, VDE `dev` uses `dev.fact_orders` while other VDEs like `kevin` use `dev__kevin.fact_orders`. - -This setting is also required for [adopt and detach](/concepts/virtual-environments/adopt-detach) operations so that existing consumer queries continue to work after migrating to virtual mode. - -### Janitor configuration - -When using virtual environments, configure the janitor to run periodically to clean up expired VDEs, old checkpoints, and unreferenced physical versions: - -```toml -[janitor] -enabled = true -retention_days = 30 -max_checkpoints = 20 -``` - -See [Janitor](/concepts/virtual-environments/janitor) for details on what gets cleaned up and how retention works. - -### State tables - -The state store contains current-state tables and append-only history tables. You do not need to interact with these directly, but understanding what is stored helps when debugging: - -**Current state:** `state_versions`, `model_versions`, `function_versions`, `physical_relations`, `physical_relation_ancestry`, `virtual_environments`, `virtual_environment_refs`, `virtual_environment_function_refs`, `virtual_environment_checkpoints`, `virtual_environment_checkpoint_refs`, `virtual_environment_checkpoint_function_refs`, `locks`, `state_operations` - -**History:** `plan_runs`, `virtual_environment_ref_events`, `reconcile_events`, `state_migration_events`, `state_operation_events` - -## Building - -Source: `concepts/virtual-environments/building.mdx` - -Virtual builds, VDE creation, partial builds, and seeded incrementals. - -Virtual builds create versioned physical relations and update VDE pointer sets. The build lifecycle is the same as direct mode (seeds, tests, models, audits), but model outputs are written to versioned physical tables and exposed through logical VDE views. - -Virtual builds run ingress (loaders and Python nodes that feed sources) as a separate phase before SQL model execution. This means independent SQL models that do not depend on loaders will wait for all ingress to complete before starting. Direct mode does not have this limitation. This keeps VDE state persistence simpler and safer but may add wall time when ingress is slow and independent SQL work is available. A future optimization may allow overlapping ingress with independent SQL execution. - -### Default VDE - -When you run `sqb build` in virtual mode without `--virtual-env`, the VDE name defaults to the active physical target name: - -```bash -sqb build -``` - -If the active environment is `dev`, this builds into VDE `dev`. On first run, it creates physical version relations and logical VDE views for all models. - -### Explicit VDE - -Use `--virtual-env` to build into a named VDE: - -```bash -sqb build --virtual-env pr_123 -``` - -If VDE `pr_123` does not exist, SQLBuild creates it by inheriting refs from the baseline VDE (the default VDE for the active physical target). Unchanged models share the same physical relations as the baseline - no data is copied. - -Only models with changed code, config, or upstream versions get new physical version relations. - -### Physical naming - -Physical version relations are stored in a dedicated schema: - -``` -__sqb_physical.__v_ -``` - -For example: `dev__sqb_physical.fact_orders__v_8f3a9c12` - -Logical VDE views are created in a VDE-suffixed schema: - -``` -__. -``` - -For example: `dev__pr_123.fact_orders` - -When `unsuffixed_virtual_env` is configured, the named VDE uses the base schema directly: - -``` -dev.fact_orders -``` - -### VDE status - -| Status | Meaning | -|--------|---------| -| `finalized` | All models match their expected version hashes. The VDE is coherent and eligible for promotion. | -| `active` | Some models are stale (partial build or pending changes). The VDE is a work in progress. | -| `detached` | The VDE has been detached via `sqb state detach`. Build, promote, and rollback are blocked. | - -### Partial builds - -Build a subset of models with `--select`: - -```bash -sqb build --virtual-env pr_123 --select fact_orders -``` - -Partial builds leave the VDE in `active` (working) status if downstream models remain stale. A -follow-up `sqb build --virtual-env pr_123 --changes-only` without `--select` builds the remaining -stale models to finalize the VDE. A plain `sqb build --virtual-env pr_123` runs the whole selection. - -#### Stale upstream coherence - -If a selected model has stale required upstream models, the build blocks by default: - -```bash -sqb build --virtual-env pr_123 --select fact_orders -# error: selected models have stale required upstreams: stg_orders -``` - -Pass `--include-stale-upstreams` to expand the selection to the minimal set of stale ancestor models needed to make the selected scope coherent: - -```bash -sqb build --virtual-env pr_123 --select fact_orders --include-stale-upstreams -``` - -### Stale-driven execution - -Virtual environment builds run the full selection by default. Add `--changes-only` to limit a plan -or build to work whose expected version is not already bound in the target VDE: - -```bash -sqb plan --virtual-env pr_123 --changes-only -sqb build --virtual-env pr_123 --changes-only -sqb build --virtual-env pr_123 --select path:models/marts --changes-only -``` - -With selectors, SQLBuild intersects the selected scope with the stale set. Models with changed SQL or -configuration, pending backfills, changed functions, or affected upstreams remain in the plan; -models already bound to their expected version do not need another physical version. - -`--changes-only` is a virtual-mode execution option. Direct mode always runs its selected build -scope, although its plan still reports change reasons and uses them to choose incremental actions. -See [Planning and change detection](/concepts/planning) for the shared identity and cascade model. - -You can make stale-driven execution the default for virtual plans and builds: - -```toml -[settings] -virtual_environments = true -changes_only = true - -[targets.dev] -changes_only = true -``` - -The CLI flag takes precedence, followed by the selected target, local settings, and project -settings. - -#### Run despite unchanged - -`run_despite_unchanged` lets a table remain eligible during stale-driven execution even when its -version identity is unchanged. This is useful when external data changes cannot be represented by -source freshness. - -```sql -MODEL ( - materialized table, - run_despite_unchanged "always", -); -``` - -- `always` runs whenever the model is selected. -- A duration such as `24h`, `30d`, or `90m` keeps the model eligible while the newest timestamp data - version from an upstream source is no older than that duration; it is not a periodic scheduler. - -Only table materializations support this setting. When it triggers, downstream models are also -stale. - -### Stale detection - -SQLBuild determines which models and seeds need rebuilding by comparing expected version hashes against bound version hashes in the VDE refs: - -1. **Expected local hash** - derived from the node's query SQL (for models), content hash (for seeds), version-identity config, and source freshness hashes -2. **Expected version hash** - local hash propagated through upstream dependencies (upstream hash changes cascade downstream) -3. **Bound version hash** - the hash currently stored in the VDE refs from the last successful build - -A node is stale when `bound != expected`. Stale nodes whose own local hash changed are roots (`query changed`, `config changed`, `function changed`). Others are stale due to `upstream changed`. - -Seeds participate in version identity the same way as models. They are loaded into versioned physical tables with logical VDE views on top, and their version hashes and refs are tracked per virtual environment in the state backend. Unchanged seeds are not reloaded. - -Source freshness observations and Python node identities are also persisted per virtual environment and included in version hash computation. When a source's observed data version changes, its downstream models become stale. See [Sources: Source freshness](/concepts/sources#source-freshness) for configuration. - -### Seeded incremental builds - -When an incremental model's version hash changes, SQLBuild seeds the new physical version from the prior physical version before running the incremental delta. This avoids full rebuilds of large incremental tables. - -| Adapter | Seed strategy | -|---------|--------------| -| Snowflake | Zero-copy clone | -| BigQuery | Table clone | -| Databricks | Deep clone | -| DuckDB, Postgres, SQL Server | CTAS copy | - -For append models with bounded replay (`replay_on_change bounded-7d`), the seed copies only rows before the replay window cutoff. The incremental delta then appends the bounded range without duplicating rows. - -### Custom materializations - -Custom materializations are supported in virtual mode. By default, SQLBuild seeds new physical versions using the direct clone/copy strategy before calling the custom `materialize` function. - -For custom materializations that need different seeding behavior, define a `prepare_version` function alongside `materialize`: - -```python -from sqlbuild.virtual.executor.models import VersionPrepareContext -from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult - -def prepare_version(ctx: VersionPrepareContext) -> None: - """Prepare the new physical destination from the prior version.""" - ctx.execute_sql(f"CREATE TABLE {ctx.destination} AS SELECT * FROM {ctx.prior_relation}") - -def materialize(ctx: MaterializationContext) -> MaterializationResult: - """Run the custom materialization logic against the prepared destination.""" - ... -``` - -If `prepare_version` is not defined, the framework uses the default clone/copy. Most custom materializations do not need to define it. - -`VersionPrepareContext` provides `prior_relation` (the source physical version), `destination` (the new physical destination relation), `adapter`, `connection`, `execute_sql()`, `config`, and `vars`. - -Custom materializations in virtual mode must write only to `ctx.destination`. Side-effect writes to other relations are not tracked by virtual state, not cleaned up by janitor, and not restored by rollback. - -### Functions - -Functions are published into the logical VDE schema, not the physical layer. Each VDE has its own copy of function definitions. Function versions are tracked in state and participate in promotion and rollback. - -### Plan - -Use `sqb plan` to preview what a virtual build would do without executing: - -```bash -sqb plan -sqb plan --virtual-env pr_123 -sqb plan --select fact_orders -``` - -Virtual plan output shows: -- Virtual environment name and status (finalized/working) -- Stale root count and root set -- Stale model count -- Query diffs from prior bound versions -- Remaining stale models after partial selection - -## Promotion - -Source: `concepts/virtual-environments/promotion.mdx` - -Promote VDE refs and diff virtual environments. - -Promotion copies model version refs from one VDE to another and refreshes the target's logical views. No models are rebuilt - it is a pointer swap. - -### Basic usage - -```bash -sqb promote --from pr_123 --to dev -``` - -This updates VDE `dev` to point at the same physical versions that VDE `pr_123` uses. The target's logical views (`dev__dev.*` or `dev.*` if unsuffixed) are refreshed to point at the promoted physical relations. - -### What happens during promotion - -1. Target VDE lock is acquired -2. Source VDE is validated (finalized, current with workspace) -3. Source model refs are copied to target VDE refs -4. Source function refs are copied and function definitions are republished in the target schema -5. Target logical VDE views are refreshed -6. A checkpoint is created if the target is finalized -7. Target VDE lock is released - -### Partial promotion - -Promote a subset of models with `--select`: - -```bash -sqb promote --from pr_123 --to dev --select fact_orders -``` - -Partial promotion can leave the target VDE with stale downstream models - models that still point at older versions than the promoted scope expects. When this happens, promotion is blocked by default so you don't accidentally leave the target in a working state. - -To accept a working target, pass `--allow-partial-promotion`: - -```bash -sqb promote --from pr_123 --to dev --select fact_orders --allow-partial-promotion -``` - -The target VDE is marked `working` after the promotion. You can finalize it later by promoting or building the remaining models. - -If the models you select depend on upstream models that are themselves stale in the source, the promoted scope would not be coherent on its own. Pass `--include-stale-upstreams` to expand the selection to include the minimal set of required upstream models: - -```bash -sqb promote --from pr_123 --to dev --select fact_orders --include-stale-upstreams -``` - -### Source VDE requirements - -**Whole promotion** requires the source VDE to be finalized and current with the workspace (no stale models vs current code). If the source VDE has been built but the code has since changed, you need to rebuild the source VDE first or use partial promotion. - -**Partial promotion** does not require a finalized source. A working source VDE is allowed when using `--select` for a coherent scope. - -### Guards - -| Condition | Behavior | -|-----------|----------| -| Source VDE not finalized (whole promotion) | Blocks. Use `--select` for partial promotion from a working source. | -| Source VDE has stale models vs workspace | Blocks. Rebuild the source VDE or use `--select`. | -| Target VDE is locked | Blocks. Wait or clear the lock with `sqb state locks clear`. | -| Target VDE is detached | Blocks. Detached VDEs cannot be promoted to. | -| Source VDE is detached | Blocks. Detached VDEs cannot be promoted from. | -| Partial promotion leaves target working | Blocks unless `--allow-partial-promotion` is set. | - -### Comparing VDEs before promotion - -Use `sqb diff` to compare VDE ref sets before promoting. See [Diff](/concepts/virtual-environments/diff) for details. - -## Rollback - -Source: `concepts/virtual-environments/rollback.mdx` - -Checkpoints and rollback for virtual environments. - -Rollback restores a VDE to a prior finalized state by rebinding its refs to a stored checkpoint. - -### Checkpoints - -Checkpoints are created automatically when a VDE reaches `finalized` status: - -- After a whole build where all models match expected versions -- After a whole promotion where the target VDE is finalized - -Each checkpoint stores the complete set of model refs and function refs for that VDE at that point in time. Checkpoints are retained according to `[janitor] max_checkpoints` (default: 20). - -### Basic rollback - -Restore the previous finalized checkpoint for the default VDE: - -```bash -sqb rollback -``` - -For an explicit VDE: - -```bash -sqb rollback --virtual-env pr_123 -``` - -### Explicit checkpoint - -Restore a specific checkpoint: - -```bash -sqb rollback --checkpoint-id -``` - -Use `sqb state checkpoints list` to see available checkpoints. - -### What happens during rollback - -1. Target VDE lock is acquired -2. The target checkpoint is located (previous finalized by default, or explicit id) -3. Checkpoint physical relations are validated to still exist in the warehouse -4. VDE refs are replaced with the checkpoint's ref set -5. VDE function refs are replaced with the checkpoint's function ref set -6. Logical VDE views are refreshed to point at the restored physical versions -7. Function definitions are republished from the checkpoint's function versions -8. Target VDE lock is released - -### Partial rollback - -Roll back a subset of models with `--select`: - -```bash -sqb rollback --select fact_orders -``` - -Rolling back some models but not others can leave the VDE with stale models - models whose restored versions no longer line up with the rest of the VDE. When this happens, rollback is blocked by default so you don't leave the VDE in a working state by accident. - -To accept a working VDE, pass `--allow-partial-rollback`: - -```bash -sqb rollback --select fact_orders --allow-partial-rollback -``` - -The VDE is marked `working` after the rollback. You can finalize it later by building or rolling back the remaining models. - -If the models you select depend on upstream models that also need to be restored for the scope to be coherent, pass `--include-stale-upstreams` to expand the selection to the minimal set of required upstream refs from the checkpoint: - -```bash -sqb rollback --select fact_orders --include-stale-upstreams -``` - -### Guards - -| Condition | Behavior | -|-----------|----------| -| No previous checkpoint exists | Blocks. Build the VDE first to create a finalized checkpoint. | -| Checkpoint physical relations deleted | Blocks. The physical version tables have been cleaned up by janitor. Use a more recent checkpoint or rebuild. | -| Unknown checkpoint id | Blocks with error. | -| Target VDE is locked | Blocks. Wait or clear the lock. | -| Target VDE is detached | Blocks. Detached VDEs cannot be rolled back. | -| Partial rollback leaves VDE working | Blocks unless `--allow-partial-rollback` is set. | - -### Checkpoint inspection - -List checkpoints for a VDE: - -```bash -sqb state checkpoints list -sqb state checkpoints list --virtual-env pr_123 -``` - -Show a checkpoint's model refs: - -```bash -sqb state checkpoints show -``` - -Diff current VDE refs against a checkpoint: - -```bash -sqb state checkpoints diff -``` - -## Adopt and Detach - -Source: `concepts/virtual-environments/adopt-detach.mdx` - -Migrate existing projects to and from virtual mode. - -Adopt converts an existing stateless project into virtual mode. Detach reverses the process. Both are interactive operations that require typed confirmation. - -### Adopt - -`sqb state adopt` takes existing warehouse relations (tables and views) and converts them into versioned physical relations with VDE views at the original names. - -#### Prerequisites - -1. State store must be initialized (`sqb state init`) -2. `unsuffixed_virtual_env` must be configured so existing object names are preserved: - -```toml -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -unsuffixed_virtual_env = "dev" - -[targets.dev.state.connection] -database = "state.duckdb" -``` - -Without `unsuffixed_virtual_env`, adopt blocks with a config error. This is intentional - without it, existing relations would be renamed to suffixed schemas (e.g. `dev__dev.fact_orders`), breaking existing consumers. - -#### What happens - -1. For each model, the existing table is moved/renamed into the physical schema (`dev__sqb_physical.fact_orders__v_`) -2. A logical VDE view is created at the original name (`dev.fact_orders`) pointing to the physical version -3. Model versions, physical relations, VDE record, and VDE refs are persisted in state - -#### Usage - -```bash -sqb state adopt --allow-copy -``` - -The command prints an adoption plan and requires typed confirmation: - -``` -Type "adopt dev" to confirm: adopt dev -``` - -`--allow-copy` is required when the adapter does not support native same-schema rename (cross-schema moves, some adapters). Without it, adopt blocks with `requires --allow-copy` if a copy fallback would be needed. - -#### View models - -View models are adopted the same way - a versioned physical view is created in the physical schema, and the original name becomes a logical VDE view. - -### Detach - -`sqb state detach` reverses adoption, collapsing a VDE back into normal stateless relations. - -#### Prerequisites - -The VDE must be `finalized`. If the VDE is working (has stale models), detach blocks: - -``` -error: detach requires a finalized virtual environment -``` - -Build the VDE to finalize it first, or resolve any pending changes. - -#### What happens - -1. For each table model, the physical version is moved/renamed back to the original target name -2. For each view model, the view is recreated from compiled SQL at the original target name (not copied from the physical ref) -3. The VDE is marked `detached` in state -4. VDE refs are preserved for audit/recovery and janitor protection - -#### Usage - -```bash -sqb state detach --allow-copy -``` - -Requires typed confirmation: - -``` -Type "detach dev" to confirm: detach dev -``` - -#### After detach - -A detached VDE is blocked from further virtual operations: - -- `sqb build` blocks with "virtual environment is detached" -- `sqb promote --from ` or `--to ` blocks -- `sqb rollback` on a detached VDE blocks - -The project can continue operating in direct mode, or you can re-adopt to return to virtual mode. - -#### Detached VDE cleanup - -Detached VDE refs and state rows are cleaned up by the [janitor](/concepts/virtual-environments/janitor), not by detach itself. This preserves refs for recovery if detach fails partway through. - -### Interrupted operations - -Adopt and detach are multi-step warehouse operations that cannot be wrapped in a single transaction. If an operation fails partway through: - -- A `failed` operation record is persisted in state with the error message -- VDE refs and checkpoints remain intact -- The warehouse may have partial artifacts (e.g. table moved to physical schema but view not yet created) - -Recovery path: - -1. Run `sqb reconcile` to diagnose the current state -2. Use `sqb reconcile repair-view` or manual warehouse repair as needed -3. Retry the adopt or detach operation - -There is no automatic resume command. SQLBuild records the failure state and leaves recovery to the operator. - -## Clone - -Source: `concepts/virtual-environments/clone.mdx` - -Hydrate physical versions from a source warehouse. - -In virtual mode, `sqb clone` hydrates physical version relations from a source warehouse into the target physical storage layer. It does not copy VDE pointer sets or create logical views - it copies the underlying physical data so that builds and promotions can reference those versions locally. - -This is useful for seeding a new target from an existing one, or restoring physical versions that were cleaned up by the janitor. - -As in direct mode, `--from` selects an origin namespace and clone-origin policy, not a -connection. `--to` (or the active target) supplies the only physical connection. The origin -physical namespace must be readable through that destination connection; SQLBuild never opens -the origin target connection or transfers data across accounts, servers, or files using a -second connection. - -### How it works - -Virtual clone looks up expected physical version relations in the source warehouse by their naming convention (`__sqb_physical.__v_`) and copies them into the target warehouse. It then registers the copied relations in the target state store. - -Target VDE refs and logical views are not changed. Clone only populates the physical layer. - -### Default mode - -Without `--virtual-env`, clone hydrates physical versions matching the current workspace's expected fingerprints: - -```bash -sqb clone --from prod --to dev -``` - -This computes expected version hashes from the current code, looks for matching physical relations in the source warehouse, and copies them into the target. - -### VDE ref mode - -With `--virtual-env`, clone hydrates physical versions referenced by a specific VDE in the target state store: - -```bash -sqb clone --from prod --to dev --virtual-env pr_123 -``` - -This reads VDE `pr_123`'s refs from the target state, looks for those physical relations in the source warehouse, and copies them into the target. Useful when a VDE's physical versions have been deleted (by janitor or manual cleanup) but the state refs still exist. - -### Selection - -Scope which models are hydrated: - -```bash -sqb clone --from prod --to dev --select fact_orders -``` - -### Missing source artifacts - -If a source physical relation does not exist, clone reports it as missing and continues with the remaining models. The exit code is non-zero if any models are missing. - -### Model version locks - -If a target model version is locked (another process is building that version), clone blocks by default: - -```bash -# Skip locked versions and hydrate the rest -sqb clone --from prod --to dev --skip-locked -``` - -### What clone does not do - -- Does not create or update VDE refs -- Does not create or refresh logical VDE views -- Does not read the source state database (it uses warehouse-level artifact discovery) -- Does not change the target VDE status - -Clone is a physical-layer operation. VDE pointer management is handled by [build](/concepts/virtual-environments/building) and [promote](/concepts/virtual-environments/promotion). - -### Comparison with direct-mode clone - -In direct mode, `sqb clone` copies model relations between targets using zero-copy cloning where supported. In virtual mode, clone hydrates versioned physical relations instead. The source and target are still physical targets, but the copied objects are physical version relations rather than normal model targets. - -## Diff - -Source: `concepts/virtual-environments/diff.mdx` - -Compare virtual data environments. - -In virtual mode, `sqb diff` compares VDE ref sets within the same physical target. It shows which models have different version hashes and, for changed models, reports schema and row-level differences. - -### Basic usage - -```bash -sqb diff dev:pr_123 -``` - -The format is `:`. Both VDEs must exist in the active physical target's state store. - -### What diff shows - -1. **Ref comparison** - which models have different version hashes between the two VDEs -2. **Schema differences** - column additions, removals, and type changes for changed models -3. **Row differences** - row counts, matched/unmatched rows, and changed column values - -Identical refs are skipped by default so the output focuses on models that actually differ. A model with a different version hash is included in the comparison even if its resulting data happens to be identical. - -### Options - -```bash -# Schema differences only (no row comparison) -sqb diff dev:pr_123 --schema-only - -# Full row-level comparison -sqb diff dev:pr_123 --full - -# Compare specific models -sqb diff dev:pr_123 --select fact_orders - -# No color output -sqb --no-color diff dev:pr_123 -``` - -### Working VDE guard - -If either VDE is working (has stale models that haven't been built yet), diff is blocked by default because the comparison may be incomplete: - -```bash -# Blocked -sqb diff dev:pr_123 - -# Allowed with explicit override -sqb diff dev:pr_123 --allow-partial-diff -``` - -This guard prevents misleading diff output when one VDE has pending changes that haven't been materialized yet. - -### Comparison with direct-mode diff - -In direct mode, `sqb diff prod:dev` compares physical target schemas and data directly in the warehouse. In virtual mode, `sqb diff dev:pr_123` compares VDE pointer sets within a single physical target, then inspects the physical versions those pointers reference. - -The output format is the same - schema diffs, row counts, changed columns, and example rows. The difference is what is being compared: physical targets vs virtual pointer sets. - -## Reconcile - -Source: `concepts/virtual-environments/reconcile.mdx` - -Diagnose and repair drift between state and warehouse. - -Reconcile detects and repairs inconsistencies between the virtual state store and the actual warehouse objects. - -### Report - -Run reconcile without a subcommand to get a diagnostic report: - -```bash -sqb reconcile --virtual-env dev -``` - -This inspects the VDE's refs, checks that logical views exist and point to the expected physical relations, and reports any issues without changing anything. - -### Repair view - -Recreate a logical VDE view from trusted state: - -```bash -sqb reconcile repair-view --virtual-env dev --model fact_orders -``` - -This runs `CREATE OR REPLACE VIEW` for the logical VDE view, pointing it at the physical version relation recorded in the VDE's refs. It is idempotent - running it when the view is already correct is a no-op. - -#### Guards - -| Condition | Behavior | -|-----------|----------| -| Logical target is a table (not a view) | Blocks. Drop the table manually first, then retry. | -| Physical relation is missing | Blocks. Rebuild the model with `sqb build --select `. | -| Target VDE is locked | Blocks. Wait or clear the lock. | - -No confirmation is needed. The command is explicit and idempotent. - -### Attach - -Rebind a VDE model ref to a different tracked physical relation: - -```bash -sqb reconcile attach --virtual-env dev --model fact_orders \ - --physical-relation dev__sqb_physical.fact_orders__v_8f3a9c12 -``` - -This updates the VDE ref for `fact_orders` to point at the specified physical relation and refreshes the logical view. - -#### Guards - -| Condition | Behavior | -|-----------|----------| -| Physical relation not tracked in state | Blocks. Only physical relations registered in SQLBuild state can be attached. | -| Physical relation tracked for a different model | Blocks. Cannot attach a relation that belongs to another model. | -| Logical target is a table (not a view) | Blocks. | -| Target VDE is locked | Blocks. | -| Wrong confirmation | Blocks. Refs remain unchanged. | - -Attach requires confirmation by default. Type the confirmation text when prompted, or cancel to leave refs unchanged. - -### When to use reconcile - -- **Missing logical views** after a failed build, promotion, or detach - use `repair-view` -- **Wrong physical version** if a VDE ref was corrupted or you need to manually override which version a model points to - use `attach` -- **Diagnostic inspection** before or after recovery operations - use the default report - -Reconcile records events in the `reconcile_events` state table for audit trail. - -### Limitations - -Reconcile repairs pointer/view state. It does not rebuild physical versions. If a physical relation is missing from the warehouse, the only remedy is rebuilding the model with `sqb build --select `. - -## Locks - -Source: `concepts/virtual-environments/locks.mdx` - -Advisory locks for concurrent access control. - -Virtual mode uses advisory locks with TTL to prevent concurrent operations from conflicting. Locks are stored in the state database and are scoped to specific resources. - -### Lock types - -| Lock key | Protects | Acquired by | -|----------|----------|-------------| -| `virtual_env:` | VDE pointer set and views | build, promote, rollback, reconcile, adopt, detach | -| `model_version::` | Physical version creation | build (per model version) | -| `state_migration` | State schema changes | state init, migrate, rollback, reset | - -Different VDEs can be locked concurrently. Two builds targeting different VDEs do not block each other. - -### Lock behavior - -- Locks have an expiry time (`expires_at`). Active locks are those where `expires_at > now`. -- When a lock is successfully released, the lock row is deleted. -- Acquiring a lock over an expired lock replaces it. -- Owner identity is checked on release - only the owner that acquired the lock can release it. - -### When locks block - -If a lock is active and an operation requires it, the operation fails immediately with a clear error: - -``` -error[S014]: virtual environment 'dev' is locked -``` - -The operation does not wait or retry. This is intentional - SQLBuild does not implement lock queuing. If a lock is active, either wait for the holding operation to complete or clear the lock manually. - -### Inspecting locks - -List active locks: - -```bash -sqb state locks -``` - -### Clearing stuck locks - -If a process crashed while holding a lock, the lock may remain active until it expires. To clear it immediately: - -```bash -sqb state locks clear virtual_env:dev --force -``` - -Only clear locks when you are certain the holding operation is no longer running. Clearing a lock while the operation is still active can cause state corruption. - -### Lock expiry - -Locks are acquired with a TTL (typically 10 minutes for VDE locks). If an operation takes longer than the TTL, the lock expires and another operation can acquire it. This is a safety net against abandoned locks, not a normal operating condition. - -If operations consistently exceed the lock TTL, the TTL may need to be increased in a future configuration option. - -## Janitor - -Source: `concepts/virtual-environments/janitor.mdx` - -Cleanup of virtual environment artifacts and retention policies. - -The janitor manages cleanup of virtual mode artifacts: expired VDEs, old checkpoints, unreferenced physical versions, stale state backups, and expired locks. All cleanup runs through `sqb janitor` with preview and confirmation. - -### Physical version protection - -Physical version relations are never deleted while referenced by: - -- Any active (non-detached) VDE's current refs -- Any retained checkpoint's refs - -The janitor resolves the complete set of protected physical relations before considering any deletions. - -### Checkpoint retention - -Checkpoints are retained according to `[janitor] max_checkpoints` (default: 20): - -```toml -[janitor] -max_checkpoints = 20 -``` - -Values below 1 are rejected. Checkpoint creation never prunes history - pruning is janitor-only, behind preview and confirmation. - -When old checkpoints are pruned, physical versions that were protected only by those checkpoints become eligible for deletion in the same janitor run (if not protected by active VDE refs or remaining checkpoints). - -### Expired VDE cleanup - -Non-active, non-detached VDEs older than `[janitor] retention_days` are pruned: - -```toml -[janitor] -retention_days = 30 -``` - -Active/default VDEs are always protected. This catches abandoned PR preview VDEs that were never promoted or cleaned up. - -### Detached VDE cleanup - -Detached VDEs (created by `sqb state detach`) are eligible for cleanup after `retention_days`: - -- `retention_days = 0` makes them eligible immediately -- Refs, function refs, and VDE row are deleted -- Checkpoint rows remain governed by checkpoint retention -- Physical versions newly unprotected by removed refs can be deleted in the same run - -Active VDE refs continue to protect physical versions even when detached refs are pruned. - -### State cleanup - -The janitor also prunes state-only artifacts: - -- **Migration backups**: old backup schemas are deleted, but the latest backup is always preserved -- **Expired locks**: lock rows with `expires_at` in the past are deleted; active locks are never touched - -### Execution order - -The janitor drops warehouse physical versions before pruning state rows. If a warehouse drop fails, the corresponding state refs are preserved so the janitor can retry on the next run. - -### Usage - -```bash -# Preview what would be cleaned -sqb janitor - -# Execute cleanup with confirmation -sqb janitor --auto-approve -``` - -The janitor shows a preview of all candidates (checkpoints, VDEs, physical versions, state items) and requires confirmation before executing any destructive operations. - -## Recovery - -Source: `concepts/virtual-environments/recovery.mdx` - -Diagnosing and recovering from failures in virtual mode. - -Virtual mode has explicit recovery paths for common failure scenarios. SQLBuild blocks cleanly rather than leaving ambiguous state, and error messages point to the specific recovery action needed. - -### State corruption - -| Scenario | Error | Recovery | -|----------|-------|----------| -| Missing state table | "Cannot backup invalid state schema" on `state migrate` | `sqb state reset --auto-approve` then `sqb state init` | -| Missing state column | Same | Same | -| Wrong state column type | Same | Same | -| Deleted backup schema (explicit id) | Rollback blocks with backup-id error | Use a different backup or reset | -| All backups deleted | "No state backup is available for rollback" | `sqb state reset --auto-approve` then reinitialize | - -State corruption is detected by schema validation during `state migrate`. If the current state schema is invalid, SQLBuild refuses to back it up (to avoid persisting a broken snapshot) and directs you to reset. - -### Warehouse drift - -| Scenario | Detection | Recovery | -|----------|-----------|----------| -| Missing logical VDE view | `sqb reconcile` reports missing view | `sqb reconcile repair-view` for the affected model | -| Missing physical relation | `sqb reconcile` reports it, `repair-view` blocks | Rebuild with `sqb build --select` | -| Logical target is a table, not a view | `repair-view` and `attach` block | Drop the table manually, then `repair-view` | -| Checkpoint physical relation missing | `sqb rollback` blocks | Use a more recent checkpoint, or rebuild | -| Promoted physical relation missing | `sqb promote` blocks | Rebuild the source VDE first | - -### Lock conflicts - -| Scenario | Error | Recovery | -|----------|-------|----------| -| VDE locked by another process | "virtual environment is locked" | Wait for the other process, or clear the lock | -| Lock held by crashed process | Same | `sqb state locks clear` with `--force` | - -Only clear locks when you are certain the holding operation is no longer running. - -### Interrupted operations - -| Scenario | Behavior | Recovery | -|----------|----------|----------| -| Interrupted adopt | Failed operation recorded. Physical schema may have partial artifacts. | `sqb reconcile` to diagnose, manual repair if needed, retry. | -| Interrupted detach | Failed operation recorded. VDE stays finalized. Physical version preserved. | `sqb reconcile` to diagnose, retry detach. | -| Interrupted promotion | Failed operation recorded. Target VDE lock released. | Retry promote. | - -There is no automatic resume command for interrupted operations. SQLBuild records the failure in `state_operations` and `state_operation_events`, preserves VDE refs and checkpoints, and leaves recovery to the operator. - -To inspect a failed operation: - -```sql -SELECT operation_id, operation_type, status, virtual_environment_name -FROM sqlbuild_state.state_operations -WHERE status = 'failed'; - -SELECT operation_id, action, status, message -FROM sqlbuild_state.state_operation_events -WHERE operation_id = '' -ORDER BY created_at; -``` - -### Mode guards - -SQLBuild blocks operations that don't apply to the current mode: - -| Operation | Direct mode | Virtual mode | -|-----------|-------------|--------------| -| `sqb state` subcommands | Blocked | Allowed | -| `sqb build --no-tests --no-audits` | Allowed | Blocked (use `sqb build`) | -| `sqb build --defer-to` | Allowed | Blocked (VDE refs handle upstream resolution) | -| `sqb promote` | Blocked | Allowed | -| `sqb rollback` | Blocked | Allowed | -| `sqb reconcile` | Blocked | Allowed | - -### Detached VDE guards - -After `sqb state detach`, the VDE is marked `detached` and blocked from further virtual operations: - -| Operation | Behavior | -|-----------|----------| -| `sqb build` | Blocks: "virtual environment is detached" | -| `sqb promote --from ` | Blocks | -| `sqb promote --to ` | Blocks | -| `sqb rollback` | Blocks | - -The project can continue in direct mode or re-adopt to return to virtual mode. - -### Adopt guards - -| Condition | Error | -|-----------|-------| -| No `unsuffixed_virtual_env` configured | Blocks with config guidance | -| Copy fallback needed without `--allow-copy` | `requires --allow-copy` | -| Wrong typed confirmation | Cancelled, no changes made | - -### Detach guards - -| Condition | Error | -|-----------|-------| -| VDE not finalized | "requires a finalized virtual environment" | -| Copy fallback needed without `--allow-copy` | `requires --allow-copy` | -| Wrong typed confirmation | Cancelled, no changes made | - -### General recovery strategy - -1. **Diagnose** with `sqb reconcile` or by querying state tables directly -2. **Repair views** with `sqb reconcile repair-view` for missing/broken logical views -3. **Rebuild** with `sqb build --select ` for missing physical versions -4. **Roll back** with `sqb rollback` to restore a prior finalized state -5. **Reset state** with `sqb state reset --auto-approve` as a last resort (drops all virtual state) - -When in doubt, `sqb reconcile` is the starting point. It reports what's wrong without changing anything. - -## Overview - -Source: `integrations/dagster.mdx` - -Orchestrate SQLBuild pipelines with Dagster scheduling, retries, and asset UI. - -SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, tasks, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. - -### Install - -```bash -uv pip install 'sqlbuild[dagster]' -# or -pip install 'sqlbuild[dagster]' -``` - -This installs `dagster` and `dagster-webserver` alongside SQLBuild. - -### Try it - -```bash -sqb playground --template dagster -cd sqlbuild-playground -dagster dev -f dagster/definitions.py -``` - -This creates the waffle shop project with a `dagster/definitions.py` that includes asset definitions, scenario checks, and a configured resource. Open the Dagster UI, materialize the assets, then run the scenario checks. - -### How it works - -1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges, checks) -2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, task, and asset, with dependency edges preserved -3. `SqlBuildCliResource` shells out to `sqb build`, `sqb test`, `sqb scenario test`, etc. as subprocesses -4. Execution results (materializations, audit pass/fail, scenario outcomes) are read from SQLBuild's structured integration-result stream and emitted as Dagster `MaterializeResult` and `AssetCheckResult` events - -SQLBuild tests, audits, and Python checks become Dagster asset checks. Scenarios become asset checks attached to the models they exercise. - -### Quickstart - -```python -# definitions.py -from sqlbuild.integrations.dagster import ( - SqlBuildCliResource, - SqlBuildProject, - sqlbuild_assets, -) -import dagster as dg - -project = SqlBuildProject(project_dir=".") -project.prepare_if_dev() # auto-generates DAG artifact in dagster dev - -@sqlbuild_assets(project=project) -def my_sqlbuild_assets(context: dg.AssetExecutionContext, sqb: SqlBuildCliResource): - yield from sqb.cli(["build"], context=context).stream() - -defs = dg.Definitions( - assets=[my_sqlbuild_assets], - resources={"sqb": SqlBuildCliResource(project)}, -) -``` - -```bash -dagster dev -f definitions.py -``` - -Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. - -For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. - -### Asset selection - -When you select a subset of assets in the Dagster UI, the integration automatically: - -1. Maps selected Dagster asset keys back to SQLBuild node names using the DAG artifact -2. Writes the selectors to a temporary file -3. Passes `--select-file` to the `sqb` CLI so only the selected models are built - -This means Dagster's asset subsetting works naturally with SQLBuild's selector system. - -An explicit `--select`, `--select-file`, or `--exclude` supplied by application code takes -precedence; SQLBuild does not silently combine it with Dagster's implicit asset selection. - -### Checks - -SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: - -- **Unit tests** become checks attached to the models they test -- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` -- **Scenarios** become checks attached to the models they exercise -- **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate - -Check results are emitted with pass/fail status and metadata from the execution JSON. - -### Execution correlation - -When a Dagster execution context is passed to `sqb.cli`, SQLBuild propagates bounded orchestration -metadata to command-output and lifecycle records: - -- integration name (`dagster`) -- Dagster run ID and job name -- step/op key -- retry number -- partition key, when the run is partitioned - -Lifecycle events remain SQLBuild's immutable execution facts. Human stdout/stderr remains a -correlated troubleshooting transcript; the integration does not parse terminal text to establish -resource status. See [Execution Observability](/concepts/observability). - -### Scenarios as checks - -Scenarios can be included as asset checks alongside tests and audits (the default), or run separately: - -```python -from sqlbuild.integrations.dagster import sqlbuild_assets, sqlbuild_scenario_checks - -# Include scenario checks with other assets (default) -@sqlbuild_assets(project=project, include_scenario_checks=True) -def my_assets(context, sqb): - yield from sqb.cli(["build"], context=context).stream() - -# Or run scenarios separately -@sqlbuild_scenario_checks(project=project) -def my_scenario_checks(context, sqb): - yield from sqb.cli(["scenario", "test"], context=context).stream() -``` - -### Project preparation - -`SqlBuildProject.prepare()` regenerates the DAG artifact by running `sqb compile --dag`. Use `prepare_if_dev()` to only regenerate during local development: - -```python -project = SqlBuildProject(project_dir=".") -project.prepare_if_dev() # only runs when DAGSTER_IS_DEV_CLI is set -``` - -This keeps the Dagster UI in sync with your latest model changes during development without regenerating in production. - -### Translator - -Customise how SQLBuild nodes map to Dagster assets by subclassing `SqlBuildDagsterTranslator`: - -```python -from sqlbuild.integrations.dagster import SqlBuildDagsterTranslator -import dagster as dg - -class MyTranslator(SqlBuildDagsterTranslator): - def get_asset_key(self, node): - # Prefix all asset keys with the project name - return dg.AssetKey(["my_project", *node["asset_key"]]) - - def get_group_name(self, node): - # Group by materialization type instead of kind - return node.get("materialization_type", "other") - -@sqlbuild_assets(project=project, translator=MyTranslator()) -def my_assets(context, sqb): - yield from sqb.cli(["build"], context=context).stream() -``` - -See the [API reference](/integrations/dagster-reference) for all translator methods. - -## API Reference - -Source: `integrations/dagster-reference.mdx` - -Dagster integration classes, decorators, and translator hooks. - -### SqlBuildProject - -Project metadata and DAG artifact preparation. - -```python -from sqlbuild.integrations.dagster import SqlBuildProject - -project = SqlBuildProject( - project_dir=".", - target_path="target", - dag_filename="sqlbuild_dag.json", - sqb_command=("sqb",), - prepare_project_cli_args=("compile", "--dag"), -) -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `project_dir` | (required) | Path to the SQLBuild project root | -| `target_path` | `"target"` | Relative path to the target directory | -| `dag_filename` | `"sqlbuild_dag.json"` | DAG artifact filename | -| `sqb_command` | `("sqb",)` | Command to invoke SQLBuild CLI | -| `prepare_project_cli_args` | `("compile", "--dag")` | CLI args used by `prepare()` to generate the DAG | - -#### Methods - -| Method | Description | -|--------|-------------| -| `dag_path` | Property. Returns the full path to the DAG artifact (`project_dir / target_path / dag_filename`). | -| `prepare()` | Runs `sqb compile --dag ` to generate the DAG artifact. Raises `DagsterProjectPrepareError` on failure. | -| `prepare_if_dev()` | Calls `prepare()` only when the `DAGSTER_IS_DEV_CLI` environment variable is set. | - -### SqlBuildCliResource - -Dagster `ConfigurableResource` that shells out to the SQLBuild CLI. - -```python -from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject - -# From a project -resource = SqlBuildCliResource(SqlBuildProject(project_dir=".")) - -# Or with explicit paths -resource = SqlBuildCliResource( - project_dir=".", - sqb_command=["sqb"], - dag_path="target/sqlbuild_dag.json", -) -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `project_dir` | `"."` | Path to the SQLBuild project root, or a `SqlBuildProject` instance | -| `sqb_command` | `["sqb"]` | Command to invoke SQLBuild CLI | -| `dag_path` | `None` | Path to the DAG artifact. When set, enables asset selection bridging and structured event emission. | - -#### cli() - -Create a CLI invocation for the provided command arguments. - -```python -invocation = resource.cli( - ["build"], - context=context, # Dagster execution context (optional) - raise_on_error=True, # raise Failure on non-zero exit (default) -) -``` - -Returns a `SqlBuildCliInvocation`. - -### SqlBuildCliInvocation - -A running or completed SQLBuild CLI subprocess. - -#### Methods - -| Method | Description | -|--------|-------------| -| `wait()` | Wait for the process to complete. Streams stdout/stderr in real time. Returns `self`. | -| `stream()` | Wait for the process, then yield Dagster `MaterializeResult` and `AssetCheckResult` events parsed from execution JSON. | -| `is_successful()` | Returns `True` if exit code is 0. | -| `get_error()` | Returns a Dagster `Failure` if the process failed, `None` otherwise. | -| `get_artifact(name)` | Read a JSON artifact from the project's `target/` directory. | - -#### Properties - -| Property | Description | -|----------|-------------| -| `stdout` | Captured stdout after `wait()` or `stream()`. | -| `stderr` | Captured stderr after `wait()` or `stream()`. | -| `returncode` | Process exit code after completion. | -| `execution_payload` | Parsed execution JSON, if available. | - -#### stream() behavior - -`stream()` is the primary way to emit Dagster events from a SQLBuild execution: - -1. Waits for the subprocess to complete while streaming output to stdout/stderr -2. Reads the structured execution JSON (from `--json-output` tempfile) -3. Maps each completed asset to a `MaterializeResult` with metadata (status, duration, row counts) -4. Maps each check result to an `AssetCheckResult` with pass/fail, severity, and check metadata -5. If the DAG artifact is available, results are matched to the correct Dagster asset keys -6. Raises `Failure` if the process exited non-zero and `raise_on_error` is set - -### sqlbuild_assets - -Decorator that creates a Dagster multi-asset definition from a SQLBuild DAG artifact. - -```python -from sqlbuild.integrations.dagster import sqlbuild_assets - -@sqlbuild_assets( - project=project, # or dag=path_or_dict - translator=MyTranslator(), # optional - name="my_sqlbuild_assets", # optional - include_scenario_checks=True, # default: True - required_resource_keys={"sqb"}, # optional -) -def my_assets(context, sqb): - yield from sqb.cli(["build"], context=context).stream() -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `project` | `None` | `SqlBuildProject` instance. Mutually exclusive with `dag`. | -| `dag` | `None` | DAG artifact as a path, string, or pre-loaded dict. Mutually exclusive with `project`. | -| `translator` | `SqlBuildDagsterTranslator()` | Translator for asset keys, groups, tags, and metadata. | -| `name` | `None` | Override the Dagster multi-asset name. | -| `include_scenario_checks` | `True` | Whether to include scenario checks alongside test and audit checks. | -| `required_resource_keys` | `None` | Additional Dagster resource keys to require. | - -If neither `project` nor `dag` is provided, the decorator looks for `target/sqlbuild_dag.json` in the current directory. - -### sqlbuild_scenario_checks - -Decorator that creates a Dagster multi-asset-check definition for SQLBuild scenarios only. Useful when you want to run scenarios separately from the main build. - -```python -from sqlbuild.integrations.dagster import sqlbuild_scenario_checks - -@sqlbuild_scenario_checks(project=project) -def my_scenario_checks(context, sqb): - yield from sqb.cli(["scenario", "test"], context=context).stream() -``` - -Parameters are the same as `sqlbuild_assets` except there is no `include_scenario_checks` flag. - -### SqlBuildDagsterTranslator - -Customise how SQLBuild DAG nodes map to Dagster asset metadata. Subclass and override any method. - -#### Methods - -| Method | Arguments | Returns | Default behavior | -|--------|-----------|---------|------------------| -| `get_asset_key` | `node` | `AssetKey` | Authored source `meta.dagster.asset_key`, otherwise `AssetKey(node["asset_key"])` | -| `is_asset_node` | `node` | `bool` | Include the node as a Dagster asset | -| `is_asset_check` | `check` | `bool` | Include the check in the multi-asset definition | -| `get_group_name` | `node` | `str \| None` | Authored group, project name, then `sqlbuild` | -| `get_tags` | `node` | `dict[str, str]` | `sqlbuild/kind` tag plus any model tags | -| `get_metadata` | `node` | `dict[str, Any]` | SQLBuild identity/selector plus available path, target, SQL, columns, lineage, group, language, materialization, loader, and authored metadata | -| `get_description` | `node` | `str \| None` | Node description if present | -| `get_check_name` | `check` | `str` | `"{kind}__{name}"` with optional column/target suffix | -| `get_check_metadata` | `check` | `dict[str, Any]` | Check ID, kind, name, and selector | - -#### Node structure - -Each `node` dict passed to translator methods contains: - -```json -{ - "id": "model:fact_orders", - "kind": "model", - "name": "fact_orders", - "asset_key": ["dev", "fact_orders"], - "target": { - "database": null, - "schema": "dev", - "name": "fact_orders", - "qualified_name": "dev.fact_orders" - }, - "path": "models/marts/fact_orders.sql", - "description": "Order fact table with waffle and payment details.", - "tags": ["marts"], - "columns": [ - {"name": "order_id", "type": "INTEGER"} - ], - "materialization_type": "table" -} -``` - -Python node asset keys use a two-part key with the node kind as prefix: - -| Kind | Asset key example | -|------|-------------------| -| Task | `("task", "prepare_orders")` | -| Asset | `("asset", "orders_export")` | -| Loader | `("loader", "raw_orders")` | - -A source can provide a stable Dagster-native key without replacing the translator: - -```yaml -sources: - - name: raw__orders - meta: - dagster: - asset_key: [warehouse, raw, orders] -``` - -Override `is_asset_node` or `is_asset_check` when one SQLBuild project intentionally exposes only a -subset through a particular Dagster definition. Filtering changes Dagster ownership; it does not -remove the resource from SQLBuild's compiled graph. - -#### Check structure - -Each `check` dict passed to check translator methods contains: - -```json -{ - "id": "audit:not_null:model:fact_orders:order_id", - "kind": "audit", - "name": "not_null", - "checked_asset_ids": ["model:fact_orders"], - "path": "audits/generic/not_null.sql", - "severity": "error", - "attached_target_name": "fact_orders", - "attached_column_name": "order_id" -} -``` - -Scenario checks include additional fields: - -```json -{ - "id": "scenario:daily_revenue_minimal", - "kind": "scenario", - "name": "daily_revenue_minimal", - "checked_asset_ids": ["model:daily_revenue"], - "path": "tests/scenarios/revenue/daily_revenue_minimal.sql", - "assertion_names": ["no_negative_revenue"], - "expected_model_names": ["daily_revenue"], - "fixture_refs": ["stg_orders", "stg_payments"] -} -``` - -## Rivers - -Source: `integrations/rivers.mdx` - -Orchestrate SQLBuild pipelines with Rivers scheduling, jobs, and asset tracking. - -SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, tasks, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. - -### Install - -```bash -pip install 'sqlbuild[rivers]' -# or -uv pip install 'sqlbuild[rivers]' -``` - -This installs `rivers` alongside SQLBuild. - -### Try it - -```bash -sqb playground --template rivers -cd sqlbuild-playground -uv run rivers dev rivers_pipeline.definitions -``` - -This creates the waffle shop project with a `rivers_pipeline/definitions.py` that includes asset definitions and a configured job. Open the Rivers UI to inspect assets and trigger materializations. - -### How it works - -1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges) -2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, task, and asset, with dependency edges preserved -3. The decorated function runs `sqb build` as a subprocess and yields `Materialization` events for each output - -### Quickstart - -```python -# definitions.py -from __future__ import annotations - -import subprocess -from collections.abc import Iterator -from pathlib import Path -from typing import Any - -import rivers as rs - -from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets - -PROJECT_DIR = Path(__file__).resolve().parent.parent -SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) -SQLBUILD_PROJECT.prepare_if_dev() - -@sqlbuild_assets(project=SQLBUILD_PROJECT) -def my_assets(context: Any) -> Iterator[Any]: - completed = subprocess.run( - ["sqb", "build"], - cwd=PROJECT_DIR, - capture_output=True, - check=False, - text=True, - ) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - for output_name in context.output_selection: - yield rs.Materialization(output_name=output_name) - -repo = rs.CodeRepository( - assets=[my_assets], - jobs=[ - rs.Job( - name="sqlbuild_pipeline", - assets=[my_assets], - executor=rs.Executor.in_process(), - ), - ], -) -``` - -### SqlBuildProject - -`SqlBuildProject` manages paths and DAG artifact generation: - -```python -project = SqlBuildProject(project_dir=Path(".")) -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `project_dir` | required | Path to the SQLBuild project root | -| `target_path` | `target` | Directory for build artifacts | -| `dag_filename` | `sqlbuild_dag.json` | DAG artifact filename | -| `sqb_command` | `("sqb",)` | Command to invoke SQLBuild | -| `prepare_project_cli_args` | `("compile", "--dag")` | CLI arguments for DAG generation | - -#### Methods - -| Method | Description | -|--------|-------------| -| `prepare()` | Generate the DAG artifact by running `sqb compile --dag` | -| `prepare_if_dev()` | Generate the DAG artifact only when `RIVERS_DEPLOYMENT=dev` | -| `dag_path` | Property returning the full path to the DAG artifact | - -Use `prepare_if_dev()` so the DAG artifact is auto-generated during local development but not during production deployments where it should already exist. - -### sqlbuild_assets - -The `@sqlbuild_assets()` decorator creates a Rivers multi-asset definition from a SQLBuild DAG artifact: - -```python -@sqlbuild_assets(project=project) -def my_assets(context): - ... -``` - -| Parameter | Description | -|-----------|-------------| -| `project` | `SqlBuildProject` instance (generates and locates the DAG artifact) | -| `dag` | Alternative: pass a DAG path or dict directly instead of a project | -| `translator` | Optional `SqlBuildRiversTranslator` for custom name/tag/metadata mapping | -| `name` | Optional asset name override | - -Pass either `project` or `dag`, not both. - -### SqlBuildRiversTranslator - -Override the default mapping from SQLBuild DAG nodes to Rivers asset metadata: - -```python -from sqlbuild.integrations.rivers import SqlBuildRiversTranslator - -class CustomTranslator(SqlBuildRiversTranslator): - def get_group_name(self, node): - return "my_custom_group" - - def get_tags(self, node): - tags = super().get_tags(node) - tags.append("team:analytics") - return tags - -@sqlbuild_assets(project=project, translator=CustomTranslator()) -def my_assets(context): - ... -``` - -#### Translator methods - -| Method | Default behavior | -|--------|-----------------| -| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `task__prepare_orders`, `asset__orders_export`) | -| `get_group_name(node)` | Uses the project name | -| `get_tags(node)` | Includes `sqlbuild/kind:` plus model tags | -| `get_kinds(node)` | Returns `["sqlbuild", ""]` | -| `get_metadata(node)` | Includes SQLBuild name, kind, path, target, columns, etc. | - -### Running locally - -Run the pipeline directly without the Rivers UI: - -```bash -uv run python rivers_pipeline/definitions.py -``` - -Or start the Rivers development server: - -```bash -uv run rivers dev rivers_pipeline.definitions -``` - -## dlt - -Source: `integrations/dlt.mdx` - -Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders. - -[dlt](https://dlthub.com) is an open-source Python library for loading data from APIs, databases, cloud storage, and more. SQLBuild integrates with dlt two ways: - -- **Declarative dlt sources** - configure dlt directly in your source YAML, no Python code. SQLBuild runs the dlt pipeline as part of the build lifecycle. Best for the common REST API, SQL database, and filesystem cases. -- **dlt inside a Python loader** - wrap any `dlt.pipeline(...)` in a SQLBuild [source loader](/concepts/python-nodes/loaders) for full control when you need dlt features beyond the declarative surface. - -### Install - -```bash -pip install 'sqlbuild[dlt]' -# or -uv pip install 'sqlbuild[dlt]' -``` - -This installs dlt with the `duckdb`, `filesystem`, and `sql_database` extras alongside SQLBuild. - -### Declarative dlt sources (YAML) - -Declare a `dlt_sources` list in a `sources/*.yml` file. Each entry has a `type`, a `config` block passed to the dlt source, and a list of `resources` that become SQLBuild managed sources. SQLBuild generates a synthetic loader per resource and writes into the database its adapter manages, no Python and no destination setup required. - -Supported source types: `rest_api`, `sql_database`, `filesystem`. - -#### REST API - -```yaml -# sources/github.yml -dlt_sources: - - type: rest_api - config: - client: - base_url: https://api.github.com/ - headers: - Authorization: "Bearer ${github_token}" - paginator: - type: header_link - resources: - - name: raw_github_issues - write_disposition: append - endpoint: - path: "repos/${github_owner}/${github_repo}/issues" - params: - state: all - per_page: 100 -``` - -`rest_api` requires `client.base_url` in `config`. Each resource needs an `endpoint` mapping; the dlt resource name comes from `endpoint.name` if present, otherwise the resource `name`. - -#### SQL database - -```yaml -# sources/postgres.yml -dlt_sources: - - type: sql_database - config: - credentials: "${postgres_connection_string}" - resources: - - name: raw_customers - table: customers - write_disposition: merge - primary_key: id -``` - -`sql_database` requires `credentials` in `config`, and each resource requires a `table`. - -#### Filesystem - -```yaml -# sources/files.yml -dlt_sources: - - type: filesystem - config: - bucket_url: "s3://my-bucket/events/" - resources: - - name: raw_events - write_disposition: append -``` - -`filesystem` requires `bucket_url` in `config`. - -#### Resource options - -| Key | Description | -|-----|-------------| -| `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | -| `table` | Source table to replicate (`sql_database` only). Required for that type. | -| `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | -| `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | -| `primary_key` / `merge_key` | dlt keys for `merge`. | -| `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | -| `schema` | Per-resource schema override (falls back to the group `schema`). | - -Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](/concepts/interpolation): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. - -#### Destination - -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): - -```yaml -dlt_sources: - - type: rest_api - destination: - loader_file_format: parquet - config: - client: - base_url: https://api.example.com/ - resources: - - name: raw_widgets - endpoint: - path: widgets -``` - -#### Reference in models - -Declared resources are managed sources, reference them like any other source: - -```sql -SELECT id, title, state FROM __source("raw_github_issues") -``` - -### dlt inside a Python loader - -When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](/concepts/python-nodes/loaders). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. - -```python -# loaders/github_sources.py -import dlt -from dlt.sources.rest_api import rest_api_source -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_github_issues(ctx: LoaderContext): - source = rest_api_source({ - "client": { - "base_url": "https://api.github.com/", - "headers": {"Authorization": f"Bearer {ctx.vars['github_token']}"}, - "paginator": {"type": "header_link"}, - }, - "resources": [{ - "name": "issues", - "endpoint": { - "path": "repos/{owner}/{repo}/issues", - "params": { - "owner": ctx.vars["github_owner"], - "repo": ctx.vars["github_repo"], - "state": "all", - "per_page": 100, - }, - }, - }], - }) - - pipeline = dlt.pipeline( - pipeline_name="github_issues", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", - ) - pipeline.run(source) -``` - -Bind it to a source with `managed: true` in `sources/*.yml`: - -```yaml -# sources/github.yml -sources: - - name: raw_github_issues - managed: true - table: issues - columns: - - name: id - type: INTEGER - - name: title - type: VARCHAR -``` - -#### DuckDB connection sharing - -With the DuckDB adapter, pass `ctx.connection` to dlt's DuckDB destination to reuse SQLBuild's open connection, so dlt writes into the same database without a separate connection string: - -```python -pipeline = dlt.pipeline( - pipeline_name="my_pipeline", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", -) -``` - -#### Warehouse destinations - -For Snowflake, BigQuery, or Databricks, configure dlt with its own credentials. dlt writes directly to the warehouse, and SQLBuild reads the resulting tables as sources: - -```python -pipeline = dlt.pipeline( - pipeline_name="api_data", - destination="snowflake", - dataset_name=ctx.destination_schema or "public", -) -``` - -Configure dlt credentials via its own `secrets.toml` or environment variables, as in the [dlt documentation](https://dlthub.com/docs/general-usage/credentials/setup). - -### Build integration - -Whether declarative or Python, loaders run automatically during `sqb build` (when `auto_load_sources` is enabled), so dlt pipelines execute as part of the normal build lifecycle: - -```bash -sqb build # loaders run, then models build -sqb build --no-load # skip loading, use existing source data -sqb load # run loaders standalone -``` - -See [Loaders](/concepts/python-nodes/loaders) for write strategies, the loader context API, auto-load behavior, and source deferral. - -## ingestr - -Source: `integrations/ingestr.mdx` - -Declarative data ingestion from 50+ sources using ingestr. - -[ingestr](https://github.com/bruin-data/ingestr) is an open-source CLI tool by [Bruin](https://github.com/bruin-data) that copies data from any source to any destination using a single command. SQLBuild integrates with ingestr as a declarative source loader - you configure the ingestion directly in your source YAML, and SQLBuild handles execution as part of the build lifecycle. - -### Install - -```bash -pip install 'sqlbuild[ingestr]' -# or -uv pip install 'sqlbuild[ingestr]' -``` - -This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. - -### How it works - -1. Declare an `ingestr` block on a source in `sources/*.yml` -2. SQLBuild generates a synthetic loader that calls `ingestr ingest` as a subprocess -3. ingestr reads from the configured source and writes directly to the SQLBuild adapter's database -4. The destination URI is built automatically from your SQLBuild connection config - no manual destination setup - -No Python loader code is needed. The YAML declaration is the entire configuration. - -### Example: PostgreSQL to DuckDB - -Replicate a table from PostgreSQL into your local DuckDB project: - -```yaml -# sources/raw.yml -sources: - - name: raw_orders - table: orders - ingestr: - source_uri: "postgresql://user:pass@host:5432/mydb" - source_table: "public.orders" -``` - -That's it. Run `sqb load` or `sqb build` and ingestr copies the `orders` table into your project. - -### Example: Stripe with incremental merge - -Load Stripe charges with incremental merge on a primary key: - -```yaml -sources: - - name: raw_stripe_charges - table: charges - ingestr: - source_uri: "stripe://${stripe_api_key}" - source_table: "charges" - strategy: merge - primary_key: id - incremental_key: created -``` - -On subsequent runs, ingestr merges new and updated records based on the `id` column, using `created` to detect changes. - -### Configuration reference - -The `ingestr` block on a source supports the following fields: - -| Field | Required | Description | -|-------|----------|-------------| -| `source_uri` | Yes | ingestr source connection URI (e.g. `postgresql://...`, `stripe://...`, `shopify://...`) | -| `source_table` | Yes | Source table or resource name to ingest | -| `strategy` | No | Ingestion strategy: `replace`, `append`, `merge`, `delete+insert`, or `truncate+insert` | -| `incremental_key` | No | Column used for incremental change detection | -| `primary_key` | No | Primary key column(s) for merge strategy (string or list) | -| `columns` | No | Comma-separated column list to select from the source | -| `extra_args` | No | Additional CLI arguments passed to `ingestr ingest` (list of strings) | - -#### Strategy mapping - -| Strategy | Behavior | -|----------|----------| -| `replace` | Drop and recreate the destination table (default when no strategy is set) | -| `append` | Insert all rows without deduplication | -| `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | -| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements | -| `truncate+insert` | Truncate destination table, then insert all rows | - -### Template variables - -All string fields in the `ingestr` block support SQLBuild project variable substitution and context templates: - -```yaml -sources: - - name: raw_orders - ingestr: - source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" - source_table: "public.orders" -``` - -Variables are resolved from the project's merged variable config (project + target + local). - -Set sensitive values in `sqlbuild_local.toml` (gitignored): - -```toml -[vars] -pg_user = "readonly" -pg_password = "secret" -pg_host = "prod-db.example.com" -pg_database = "analytics" -stripe_api_key = "sk_live_..." -``` - -### Destination URI - -SQLBuild automatically builds the ingestr destination URI from your adapter connection config. All supported adapters work without manual destination configuration: - -| Adapter | Destination URI format | -|---------|----------------------| -| DuckDB | `duckdb:///path/to/db.duckdb` | -| MotherDuck | `motherduck://database` | -| PostgreSQL | `postgresql://user:pass@host:port/db` | -| Snowflake | `snowflake://user:pass@account/db/schema` | -| BigQuery | `bigquery://project` | -| Databricks | `databricks://token:...@host` | -| SQL Server | `mssql://user:pass@host:port/db` | - -### Reload - -When `--reload` is passed, ingestr runs with `--full-refresh`, forcing a complete reload regardless of the configured strategy: - -```bash -sqb load --reload -sqb build --reload -``` - -### Build integration - -ingestr sources are managed sources - they participate in the same lifecycle as Python loaders: - -```bash -# ingestr runs automatically before dependent models -sqb build - -# run ingestr sources standalone -sqb load - -# skip loading -sqb build --no-load -``` - -See [Loaders](/concepts/python-nodes/loaders) for details on auto-load behavior, source deferral, and the `--load` / `--no-load` / `--reload` flags. - -### Supported sources - -ingestr supports 50+ sources including databases, SaaS APIs, and file systems. See the [ingestr documentation](https://bruin-data.github.io/ingestr/) for the full list of supported sources and their URI formats. - -## init - -Source: `cli/init.mdx` - -Scaffold a new SQLBuild project. - -## sqb init - -Creates a new SQLBuild project with a minimal directory structure and configuration files. - -### Usage - -```bash -sqb init -``` - -No flags. Run in the directory where you want to create the project. - -### Project layout - -`sqb init` creates the configuration, linter settings, and empty resource directories needed for a standalone project: - -```text -my-project/ - sqlbuild_project.toml - .sqruff - models/ - staging/ - marts/ - schemas/ - sources/ - seeds/ - loaders/ - tasks/ - assets/ - checks/ - hooks/ - sql/ - python/ - tests/ - unit/ - scenarios/ - functions/ - sql/ - python/ - macros/ - audits/ - generic/ - singular/ -``` - -Empty directories contain `.gitkeep` files so the scaffold can be committed. Reusable attached -audit definitions belong in `audits/generic/`; standalone audits belong in `audits/singular/`. -Add reusable SQL lifecycle hooks to `hooks/sql/` and decorated Python lifecycle hooks to -`hooks/python/`; see [Hooks](/concepts/models/hooks). - -The generated project uses DuckDB, creates a named `local` connection shared by `dev` and -`prod`, and defaults models to table materialization. Its configuration follows this shape: - -```toml -adapter = "duckdb" -default_target = "dev" - -[connections.local] -database = "my_project.duckdb" - -[targets.dev] -connection = "local" -schema = "dev" - -[targets.prod] -connection = "local" -schema = "prod" -``` - -The project name is derived from the current directory name, with hyphens converted to -underscores. - -## playground - -Source: `cli/playground.mdx` - -Create a self-contained SQLBuild project to explore locally. - -## sqb playground - -Creates a self-contained waffle shop project with DuckDB. No warehouse credentials, no git clone, no external data - just a working project you can compile, build, test, and explore immediately. - -### Usage - -```bash -sqb playground [name] -``` - -The positional argument is the directory to create (default `sqlbuild-playground`). The template is chosen with `--template` (default `waffle_shop`). - -### Templates - -| Template | Description | -|----------|-------------| -| `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | -| `dagster` | Waffle shop project plus a `dagster/` directory with a ready-to-run `definitions.py`. | -| `rivers` | Waffle shop project plus a `rivers_pipeline/` directory with a Rivers repository definition. | -| `virtual` | Waffle shop with virtual environments enabled, a local DuckDB state store, loaders, and the full virtual lifecycle (build, promote, rollback). | -| `python_nodes` | A small DuckDB project demonstrating [Python nodes](/concepts/python-nodes/overview): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | - -### What it creates - -A complete DuckDB-backed project with: - -- Staging views, fact/dimension tables, and incremental models -- Sources with inline expression data (no external setup) -- Seeds, SQL functions, and a custom materialization -- Built-in and custom audits -- SQL unit tests and multi-model tests -- E2E scenario tests -- Python macros -- AI agent skill files (auto-installed for OpenCode, Claude Code, and other agents) - -The `dagster` template adds: - -- `dagster/definitions.py` - Dagster definitions with `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource` -- `dagster/README.md` - Setup instructions - -The `python_nodes` template instead creates a focused Python-nodes project: - -- `tasks/orders.py`, `loaders/orders.py`, `assets/orders_export.py`, `checks/orders_export.py` -- A `fact_orders` SQL model over a managed `raw_orders` source -- Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check - -### Examples - -```bash -# Default waffle shop -sqb playground waffle-shop -cd waffle-shop -sqb build - -# With Dagster integration -sqb playground --template dagster -cd sqlbuild-playground -uv pip install 'sqlbuild[dagster]' -dagster dev -f dagster/definitions.py - -# With Rivers integration -sqb playground --template rivers -cd sqlbuild-playground -uv pip install 'sqlbuild[rivers]' -uv run rivers dev rivers_pipeline.definitions - -# With virtual environments -sqb playground --template virtual -cd sqlbuild-playground -sqb state init -sqb build -sqb build --virtual-env pr -sqb promote --from pr --to dev - -# With Python nodes -sqb playground --template python_nodes -cd sqlbuild-playground -sqb build --select +fact_orders --select +orders_export -sqb check --select +check_orders_export -``` - -### Notes - -- The target directory must not already exist -- DuckDB is included as a core dependency - no extra installation needed -- The local DuckDB database file is created on the first build -- The Dagster template uses `prepare_if_dev()` to auto-generate the DAG artifact when Dagster starts in dev mode - -## skills - -Source: `cli/skills.mdx` - -Install SQLBuild skill files for AI coding agents. - -## sqb skills - -Install or update SQLBuild skill files so AI coding agents (Claude Code, OpenCode, Cursor, etc.) understand your project's framework, syntax, and conventions. - -Write the packaged SQLBuild skill file to agent-specific locations in your project. - -```bash -sqb skills [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--target` | Specify agent targets to install for. Can be passed multiple times. | -| `--global` | Install to global agent config directories instead of project-local | -| `--force` | Overwrite existing skill files even if they were not generated by SQLBuild | - -### Targets - -Three agent targets are supported: - -| Target | Local path | Global path | -|--------|-----------|-------------| -| `opencode` | `.opencode/skills/sqlbuild/SKILL.md` | `~/.config/opencode/skills/sqlbuild/SKILL.md` | -| `claude` | `.claude/skills/sqlbuild/SKILL.md` | `~/.claude/skills/sqlbuild/SKILL.md` | -| `agents` | `.agents/skills/sqlbuild/SKILL.md` | `~/.agents/skills/sqlbuild/SKILL.md` | - -By default, SQLBuild installs `agents` and `claude`. OpenCode consumes the shared `.agents` -skill, while Claude requires its `.claude` copy. The `opencode` target remains available as an -explicit override for installations that require `.opencode`: - -```bash -# Install the agents and claude targets (default) -sqb skills - -# Install for Claude Code only -sqb skills --target claude - -# Install for OpenCode and Claude -sqb skills --target opencode --target claude - -# Install globally -sqb skills --global -``` - -### Overwrite behavior - -Generated skill files include an ownership marker. `sqb skills` will: - -- Overwrite files it previously generated (safe to rerun) -- Refuse to overwrite files that were manually created or edited (no marker) -- Overwrite any file when `--force` is passed - -Normal project commands append a non-blocking notice when an installed, SQLBuild-owned generated -skill is stale. The command still runs normally. - -### Configuration - -Default targets can be set in `sqlbuild_project.toml` so the team shares the same agent config: - -```toml -[skills] -targets = ["agents", "claude"] -auto_update = true -``` - -CLI `--target` flags override the TOML config. - -Set `auto_update = true` to refresh stale generated skills during normal project commands. Auto -update reads the skill bundled with the installed SQLBuild package and rewrites only files marked -as SQLBuild-owned. It never overwrites a custom file or an unowned path collision. - -### Playground - -The playground command automatically runs `sqb skills` after creating the project, so AI agents are ready to use immediately: - -```bash -sqb playground waffle-shop -cd waffle-shop -# Agent skill files are already installed -``` - -### Project Rules skills - -`sqb skills` installs general SQLBuild framework guidance. `sqb rules skills` generates -project-specific guidance from the active Rules, options, thresholds, naming vocabulary, and scoped -deviations: - -```bash -sqb rules skills -sqb rules skills --check -``` - -Use `--check` in CI to detect missing or stale Rules guidance without rewriting files. Rules use -the separate `sqlbuild-rules` skill path and refuse to overwrite divergent or unowned content. See -[Compiler-integrated Rules](/concepts/rules) and the [Rules CLI reference](/cli/rules). - -## compile - -Source: `cli/compile.mdx` - -Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - -## sqb compile - -Discovers the complete project, resolves references, expands macros, validates SQL, checks column contracts, computes column lineage, and writes compiled artifacts to `target/`. The compile command is fully offline—it does not connect to the warehouse. - -### Usage - -```bash -sqb --project-dir compile [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | -| `--defer-to` | Resolve unselected model references against another target | -| `--json` | Output the full compile report as JSON | -| `--manifest` | Generate `target/manifest.json` with project metadata | -| `--lineage-mode` | Column lineage mode: `fast` (default), `rich` (slower, more detail), or `none` | -| `--select`, `-s` | Analyze and report selected resources | -| `--select-file` | Read selectors from a file, one selector per line | -| `--exclude` | Remove resources from the selected scope | - -### What compile does - -1. **Discovery** - finds `sqlbuild_project.toml`, scans for models, sources, seeds, functions, audits, tests, and macros -2. **Graph resolution** - resolves `ref()` and `source()` calls, expands macros, orders models by dependency -3. **SQL validation** - validates SQL syntax (when SQL analysis is enabled) -4. **Column lineage** - analyzes column-level dependencies across models (fast mode by default) -5. **Contract validation** - checks declared column contracts against inferred query output -6. **Rules** - evaluates selected native built-ins, then selected custom Python Rules -7. **Artifact write** - writes compiled SQL to `target/compiled/` when Rules pass - -### Focused compilation - -Use normal selectors to limit expensive analysis and reporting while retaining complete project -discovery and reference integrity: - -```bash -sqb compile --select fact_orders daily_revenue -sqb compile --select-file selected-models.txt -sqb compile --exclude deprecated_model -``` - -SQLBuild deeply analyzes the selected resources and the upstream closure needed to understand them. -The text report contains only the selected scope. Project totals remain available in JSON, together -with selected model, seed, and function counts. - -Manifest and DAG outputs remain full-project artifacts. Focused compilation changes analysis and -reporting scope; it does not produce a partial project graph. - -### Static analysis - -When SQL analysis is enabled (default), compile performs static analysis on your models without connecting to the warehouse: - -- **Column inference**: Infers output columns from each model's SQL, including through CTEs, subqueries, and JOINs -- **Column contract validation**: Under the default `settings.column_contract_mode = "implicit"`, a model with declared columns and no model-level `contract` declaration checks that every declared column exists in the statically inferred query output. `explicit` mode requires `contract enforced` to activate shape checks. Explicit type enforcement remains independent and verifies inferred types when possible -- **Column lineage**: Traces which source columns flow into each output column, including transform classification. See [Column Lineage](/concepts/column-lineage) for details - -`sqb compile` is authoritative for mandatory correctness, contracts, lineage, and configured Rules. -Rules findings block artifact completion. Use [`sqb rules run`](/cli/rules) to focus on one exact -code or family without changing project configuration. - -#### Contract diagnostics - -When a contract violation is found, compile reports it with source-annotated diagnostics: - -``` -error[K001]: declared column 'total_cents' was not found in statically inferred output for model 'fact_orders' - model: fact_orders - --> models/marts/fact_orders.sql:6:5 - 6 | total_cents (), - | ^^^^^^^^^^^ - = help: add total_cents to the SELECT list or correct/remove MODEL(columns (...)); MODEL(columns (...)) is validated using static SQL analysis because settings.column_contract_mode is "implicit" (the default). If this project intentionally uses columns only for metadata and audits, set [settings] column_contract_mode = "explicit"; models with contract enforced remain validated -``` - -The configuration guidance is an intentional project-policy choice, not a general error suppression. Fix the query or declaration when the model is intended to have a column contract. Diagnostics for `contract enforced` models do not recommend changing the project mode because explicit model contracts remain authoritative. - -Diagnostic codes: - -| Code | Meaning | -|------|---------| -| `K001` | A declared column is missing from the model's query output | -| `K002` | A column's inferred type does not match the declared type | -| `K003` | A column's type could not be proven (with `type_enforcement` enabled) | - -Compile returns exit code `1` when any error-severity diagnostic is found, making it suitable for CI checks. - -### Output - -#### Text output (default) - -```bash -sqb compile -``` - -``` -Compile ready (12 models) - - stg_customers OK 3 columns - stg_orders OK 5 columns - stg_payments OK 4 columns - fact_orders OK 6 columns - dim_customers OK 4 columns - daily_revenue OK 3 columns - ... - - Compiled: 12 models, 1 seed, 5 functions, 0 errors, 0 warnings - Wrote: target/compiled/ -``` - -Each model shows its name, status (OK or FAIL), and column count. Models with contract errors are marked FAIL. - -#### JSON output - -```bash -sqb compile --json -``` - -Returns a structured report including: - -- `summary` - model, seed, function, audit, test, error, and warning counts -- `resources` - per-model details including column count, dependencies, lineage summary, and compiled SQL -- `diagnostics` - all contract violations with source locations -- `compile_timings` - timing breakdown for discovery, graph, lineage, contracts, and write phases -- `lineage_mode` - which lineage mode was used -- `artifacts` - paths to written files - -### Column lineage modes - -The `--lineage-mode` flag controls how column lineage is computed during compile: - -| Mode | Description | -|------|-------------| -| `fast` | Default. Lightweight column extraction using SQL model metadata. | -| `rich` | Full SQL analysis with transform classification and deeper tracing. Slower on large projects. | -| `none` | Skip column lineage entirely. | - -The JSON compile report includes a lineage summary for each model, not the full column graph. Use `sqb lineage [.]` to inspect lineage as a tree, edge list, or JSON. See [Column Lineage](/concepts/column-lineage) for details on analysis modes and transform types. - -### Examples - -```bash -# Basic compile -sqb compile - -# Compile with full JSON report -sqb compile --json - -# Compile with rich column lineage -sqb compile --lineage-mode rich - -# Skip column lineage -sqb compile --lineage-mode none - -# Generate manifest -sqb compile --manifest - -# Compile only selected models -sqb compile --select fact_orders daily_revenue - -# Disable SQL analysis -sqb compile --no-sql-analysis -``` - -## format - -Source: `cli/format.mdx` - -Apply canonical SQLBuild SQL formatting. - -## sqb format - -Formats SQL files using SQLBuild's canonical, dialect-aware representation. - -```bash -sqb format [flags] -``` - -| Flag | Description | -|------|-------------| -| `--check` | Fail when formatting changes are needed without writing | -| `--diff` | Print the proposed diff without writing | -| `--json` | Print structured results | -| `--select`, `-s` | Format selected models | -| `--select-file` | Read selectors from a file | -| `--exclude` | Exclude models from the selected scope | - -```bash -sqb format -sqb format --check -sqb format --diff --select tag:marts -``` - -Formatting owns presentation and source rewriting. Compiler-integrated diagnostics belong to -configured [Rules](/concepts/rules), which never mutate source. - -## contract - -Source: `cli/contract.mdx` - -Compare or generate repository contracts from physical warehouse schemas. - -## sqb contract - -Existing-schema comparison is explicitly online and read-only with respect to the warehouse. - -### diff - -```bash -sqb contract diff --from [flags] -``` - -Reports declaration differences. Exit code `0` means agreement and `1` means differences exist. - -### generate - -```bash -sqb contract generate --from [flags] -``` - -Prints proposed declaration updates. Add `--write` to change repository files and `--overwrite` to -replace conflicts rather than preserving authored declarations. - -### Flags - -| Flag | Commands | Description | -|------|----------|-------------| -| `--from` | both | Target namespace to inspect (required) | -| `--json` | both | Print structured findings | -| `--select`, `-s` | both | Inspect selected models or sources | -| `--select-file` | both | Read selectors from a file | -| `--exclude` | both | Exclude resources from the selected scope | -| `--vars` | both | Override project variables | -| `--write` | generate | Apply generated repository changes | -| `--overwrite` | generate | Replace conflicting declarations; requires `--write` | - -See [Contracts](/concepts/models/contracts#adopting-an-existing-schema) for ownership and safety -guarantees. - -## scope - -Source: `cli/scope.mdx` - -Inspect declaration visibility, usage, placement, and move impact offline. - -`sqb scope` explains the lexical environment around native SQLBuild macros, enums, and constants. Use it to see what a resource can access, what it actually uses, why a declaration is visible or inaccessible, and whether a move would cross a scope boundary. - -The command is read-only and offline. It does not connect to a warehouse, require warehouse credentials, or inspect relations. It reads the compiler-owned declaration-scope index and reuses a deterministic cache when the relevant source and configuration fingerprint has not changed. Building a cold cache uses the normal compiler expansion path, which loads the configured adapter and may execute authored Python macros; a warm cache hit reconstructs scope facts without importing macro modules. - -Text output is deterministic, bounded, and requires no interactive pager. - -### Usage - -```bash -# Inspect an existing resource, declaration, or exact resource path -sqb scope TARGET [flags] - -# Inspect the scope at a prospective file or directory -sqb scope --at PATH [flags] - -# Preview moving an existing resource to a new path -sqb scope TARGET --as-path PATH [flags] -``` - -`TARGET` is either a kind-qualified identity or an exact project-relative resource path: - -```bash -sqb scope model:stg_orders -sqb scope test:stg_orders__excludes_cancelled -sqb scope scenario:daily_revenue -sqb scope hook:grant_select -sqb scope function:normalize_status -sqb scope audit:positive_order_value -sqb scope source:raw__orders - -sqb scope macro:normalize_order_status -sqb scope enum:order_status -sqb scope constant:minimum_order_value - -sqb scope models/staging/orders/stg_orders.sql -``` - -Bare names such as `stg_orders` and `order_status` are rejected even if they are currently unique. Qualification keeps commands stable when names collide across resource or declaration kinds. Public declaration identities are `macro:`, `enum:`, and `constant:`. - -Model-private constants and enums include their owner in the introspection identity: - -```text -enum:model:stg_orders._state -constant:model:stg_orders._minimum_value -``` - -These identities are for inspection. SQL authored inside the owning model continues to use `@enum("_state")` and `@const("_minimum_value")`. - -### Flags - -| Flag | Description | -|------|-------------| -| `--at PATH` | Inspect path-derived visibility for a prospective file or directory. A trailing `/` denotes a directory whose direct children are being considered. | -| `--as-path PATH` | Preview moving the existing target resource to `PATH`. No file or configuration is changed. | -| `--browse PATH` | Show direct child declaration folders and recursive counts without listing declarations. | -| `--list PATH` | Recursively list declarations under one declaration folder. | -| `--defined-under PATH` | Keep declarations whose definitions are under the project-relative path. | -| `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | -| `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | -| `--used-only` | Keep declarations used by the target. | -| `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | -| `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | -| `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | -| `--explain QUALIFIED_NAME` | Explain one qualified declaration in detail at the target. | -| `--globals POLICY` | Control global declarations in the main report: `summary` (default), `used`, or `all`. | -| `--page-size N` | Return at most `N` declarations per paged section (default: `100`). | -| `--after CURSOR` | Continue after a declaration identity returned as the previous section's next cursor. | -| `--paths MODE` | Render paths as `relative`, `compact`, or `none`. | -| `--json` | Write the canonical versioned JSON report instead of the text tree. | - -Filters combine in a fixed order: definition path, kind, glob, then usage. `--dependency-depth` -expands the filtered used declarations afterward. Parent-folder visibility is never depth-limited: -every unprefixed declaration role above the resource contributes to its scope. - -### Reading A Report - -The default text report keeps distinct facts in distinct sections: - -- **Available** contains declarations visible through the resource's own lexical path. -- **Used** contains declarations consumed by the resource, including tracked declaration dependencies. -- **Relationship grants** contains eligible file-based enums and constants made available through expected-output sections in a test or scenario. -- **Nearby unavailable** is opt-in and explains close declarations that are outside the target's scope. -- **Scope chain** starts with the resource's owner folder, then shows parent folders and the project - declaration roles. Its compact labels identify the access rule checked at each path. - -Ordinary text rows show each declaration's qualified identity, definition location, visibility, -reason for appearing, declaration-role root, and safe type or signature metadata where useful. -`--explain` additionally shows its owning path, narrowest required placement, current consumers, -dependencies, and the consumers affected by a placement mismatch. JSON includes the complete -structured declaration report. - -#### See the whole scope at once - -Suppose orders and returns models use a macro published from their shared `commerce` folder, and -that macro composes two project-wide macros: - -```text -project/ -├── macros/ -│ └── currency.py add_tax, round_money -└── models/ - └── commerce/ - ├── macros/ - │ └── orders.py formatted_order_total - ├── orders.sql - └── returns/ - └── returns.sql -``` - -```python -# models/commerce/macros/orders.py -from macros.currency import add_tax, round_money - -def formatted_order_total(expression: str) -> str: - return round_money(add_tax(expression)) -``` - -```sql --- models/commerce/orders.sql -MODEL(); - -SELECT @formatted_order_total("subtotal") AS total -``` - -`returns/returns.sql` calls the same macro, so the declaration must remain available throughout -`models/commerce/` and its descendants. - -The default report puts actual usage next to the complete directory-derived scope. Unused globals -stay collapsed, so the useful facts remain visible even in a large project: - -```console -$ sqb scope model:orders -Scope - Target: model:orders - Resource: model:orders - Path: models/commerce/orders.sql - -Used (1) - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 - -Scope chain - ├─ exact-owner-private models/commerce (1) - ├─ descendant-public models (0) - └─ project global (2) - -Available (1 of 3, 2 collapsed) - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 - … 2 globals collapsed; run sqb scope model:orders --globals all - -Relationship grants (0 of 0) - (none) - -Nearby unavailable (0 of 0) - (none) - -Diagnostics (0) - (none) -Completeness: complete -``` - -This answers several questions without opening declaration files: what the model actually uses, -where that declaration came from, which owner and parent folders SQLBuild checks, how many -project-wide declarations are available, and whether the result is complete. The scope-chain count -at `models/commerce` includes declarations owned there; the `exact-owner-private` chain label does -not mean every declaration in that count is private. - -#### Follow composed macro dependencies - -`Used` is direct by default. Add `--dependency-depth` to follow declarations used by those -declarations. In this example, the model calls `formatted_order_total`, whose Python body calls -`add_tax` and `round_money`: - -```console -$ sqb scope model:orders --used-only --dependency-depth 1 -Scope - Target: model:orders - Resource: model:orders - Path: models/commerce/orders.sql - -Used (3) - ├─ ● macro:add_tax [macro; project; dependency; params 1; role macros] macros/currency.py:1:1 - ├─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 - └─ ● macro:round_money [macro; project; dependency; params 1; role macros] macros/currency.py:5:1 -``` - -The remaining report sections follow the `Used` section as usual. The dependency graph comes from -actual Python calls, including nested calls and calls reached through private helpers. Merely -importing a macro does not make it used. Increase the depth to follow longer chains. - -Explain the composed macro to inspect its direct graph and placement facts: - -```console -$ sqb scope model:orders --explain macro:formatted_order_total -Explanation - └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 - Owner: (none) - Owning path: models/commerce - Consumers: model:orders, model:returns - Dependencies: macro:add_tax, macro:round_money - Grants: (none) - Required scope: descendant-public - Required path: models/commerce - Promotion impact: (none) -``` - -The complete command also prints the ordinary report sections above the `Explanation` section. -Use `--json` when another tool needs the same graph and placement facts as structured data. - -Global declarations are an intentional project-wide API and can be numerous. The default `--globals summary` always retains globals used by the target but collapses the unused global inventory with exact counts. Use `--globals used` for only used globals or `--globals all` when a bounded full list is appropriate. - -```bash -# Include every global declaration in the paged report -sqb scope model:stg_orders --globals all - -# Focus on constants and enums defined in finance -sqb scope model:stg_orders \ - --kind constant \ - --kind enum \ - --defined-under models/finance - -# Find visible or used settlement declarations -sqb scope model:stg_orders --match '*settlement*' -sqb scope model:stg_orders --used-only -``` - -### Expected-output access - -Tests and scenarios have two independent ways to reach declarations. Their own folder paths provide -ordinary visibility. For tests, each explicit `__expected__` section adds the eligible -file-based macros, enums, and constants available to that model. Scenarios receive the model's -file-based enums and constants but not its macros. These relationships include eligible -exact-owner-private declarations. - -`sqb scope` reports this additional access separately under **Relationship grants** and names the -model that provides it. With multiple expected models, SQLBuild makes the deterministic union of -their eligible declarations available while compiling the whole test or scenario. It never includes -declarations defined inside a model's `MODEL()` header. A test filename, mirrored path, or mock does -not provide this access by itself. - -For macro-mode tests, **Relationship grants** also reports -`tested_macro through macro:`. This grants the tested macro and the scoped file-based -declarations available from its production owner, as inferred from calls in `__macro_actual__`. - -```bash -sqb scope test:orders__completed_only -sqb scope scenario:daily_revenue --used-only -``` - -### Nearby And Explain - -Nearby discovery is deliberately opt-in and bounded. It considers relevant declarations in the -same authored resource tree, parent folders, close neighboring branches and child folders, and -folders connected through expected-output relationships. It does not dump every private -declaration in the project. - -```bash -# Find declarations just outside the model's effective scope -sqb scope model:stg_orders --include-nearby - -# Include descendants and siblings two directory levels away -sqb scope model:stg_orders --include-nearby --nearby-depth 2 - -# Explain one declaration's visibility and placement at this model -sqb scope model:stg_orders --explain enum:customer_status -``` - -An explanation distinguishes a known but unavailable declaration from an unknown name. It reports -where the declaration is defined, why it is or is not available, which files use it, access through -expected output, required placement, and consumers affected by a placement mismatch. It does not -move or rewrite the declaration. - -`--dependency-depth` is separate from `--nearby-depth`: it follows tracked declaration dependencies from the used section rather than filesystem proximity. - -```bash -sqb scope model:stg_orders --used-only --dependency-depth 2 -``` - -### Prospective Paths - -Use `--at` before creating a resource. A prospective file receives the declarations implied by that exact authored path. A prospective directory describes what a direct child resource would receive. - -```bash -sqb scope --at models/staging/orders/new_model.sql -sqb scope --at tests/unit/staging/orders/ -``` - -The path must be project-relative, below a configured authored resource root, and use the appropriate resource suffix (`.sql`, or `.yml`/`.yaml` for sources). Paths outside those roots produce a diagnostic rather than borrowing scope from a nearby directory. - -Prospective reports are intentionally partial: static path visibility is available, but runtime -usage and expected-output relationships do not exist yet. The report marks those sections -incomplete and exits nonzero while preserving the useful static result. - -### Move Preview - -`--as-path` calculates the visibility delta for moving one existing resource. It reports retained, -gained, and lost declarations; direct usages that the move would invalidate; the new resource-tree -root; declarations private to the resource's owner folder; and expected-output access retained -independently of folder visibility. - -```bash -sqb scope model:stg_orders \ - --as-path models/marts/orders/stg_orders.sql -``` - -The destination must be a valid project-relative file path for that resource kind. This is a pure preview: `sqb scope` never moves files, edits declarations, or changes configuration. - -### Folder Browsing - -Browse and list are separate so a project with 10,000 or more declarations remains safe to explore. `--browse` returns only direct child declaration folders. Each folder has exact recursive declaration, usage, and kind counts plus its direct child-folder count; no arbitrary alphabetical prefix of declarations is printed. - -Global roots appear in the browse namespace as `global/macros`, `global/constants`, and `global/enums`, regardless of how files are organized recursively beneath the top-level declaration roots. - -```bash -# Start with folder summaries only -sqb scope model:stg_orders --browse . - -# Walk down without listing declarations -sqb scope model:stg_orders --browse global -sqb scope model:stg_orders --browse global/macros/finance - -# List declarations only after choosing a bounded domain -sqb scope model:stg_orders --list global/macros/finance/payments -``` - -`--list` is recursive and supports `--kind`, `--match`, `--defined-under`, `--used-only`, `--page-size`, and `--after`. Browse output itself stays folder-first. - -### Pagination - -Flat lists and report sections use qualified declaration identities as stable lexical cursors, not page numbers. In canonical JSON, every paged section reports its total, returned count, completeness, truncation state, and `next_cursor`. Repeat the same command and filters with that identity as `--after`: - -```bash -# First page -sqb scope model:stg_orders \ - --list global/macros/legacy \ - --page-size 50 - -# Continue with the next_cursor from the first result -sqb scope model:stg_orders \ - --list global/macros/legacy \ - --page-size 50 \ - --after macro:legacy_batch_0049 -``` - -Keep all semantic filters unchanged while continuing. Qualified cursors avoid cross-kind ambiguity. A bare cursor is accepted only when it resolves uniquely; invalid or ambiguous cursors produce a diagnostic and nonzero status rather than silently selecting a different page. - -For automation, read each section's `next_cursor` from JSON and continue until it is `null`: - -```bash -sqb scope model:stg_orders \ - --list global/macros/legacy \ - --page-size 50 \ - --json > scope-page.json -``` - -### Paths And JSON - -`--paths relative` shows normalized project-relative paths. `compact` shortens repeated path context -in text output. `none` omits declaration definition locations and replaces structural and header -paths with `(hidden)`. Paths in stable machine output never expose an absolute workspace root. - -`--json` emits the canonical schema rather than serializing the visual tree. The top-level `schema_version` is currently `1`. Reports include the target, scope chain, declaration sections, applied filters, section totals, collapsed and truncated flags, cursors, diagnostics, and aggregate and section-level completeness. Move previews and explanations appear when requested. - -JSON is deterministically ordered, ASCII, newline-terminated, and byte-stable for identical inputs. It contains no ANSI formatting. Consumers should check `schema_version` before relying on fields. - -### Partial Projects - -Scope inspection remains useful while a project is broken. SQLBuild retains valid facts and marks -affected report sections complete or incomplete. Text and JSON never present a partial section as -complete, and diagnostics identify faults that prevented the missing facts from being indexed. - -Diagnostics are stable and include project-relative source locations when available. A partial result, invalid target or cursor, or any error diagnostic produces a nonzero exit status after the available report is written. This lets an editor or agent consume path visibility while still treating incomplete analysis as a failed check. - -### Output Safety - -Scope reports describe declarations without exposing authored values or runtime secrets: - -- Constants show logical type, nullability, collection kind, item count, and rendering mode, not values. -- Enums show scalar type, member count, and a bounded preview of member names, not member values. -- Macros show parameters and tracked declaration dependencies, not source bodies, callables, or source digests. -- Credentials, connection fields, environment variables, warehouse data, and absolute machine paths are not inspected or emitted. - -There is no `--show-values` option. - -### Native SQLBuild Only - -`sqb scope` inspects native SQLBuild authored resources and declarations. It does not discover or emulate dbt models, dbt or Jinja macros, package dispatch, dbt manifests, dbt selectors, dbt tests, or dbt schema YAML visibility. An external dbt graph dependency does not contribute declarations or lexical scope. - -For declaration directory rules, placement checks, test access through expected output, and scoped -macro imports, see [Declarations and Scopes](/concepts/declaration-scopes). - -## rules - -Source: `cli/rules.mdx` - -List, inspect, or run compiler-integrated Rules and generate project guidance. - -## sqb rules - -Configured Rules run automatically through `sqb compile` and build planning. The `rules` command -provides focused catalogue, inspection, execution, and guidance operations. - -### Usage - -```bash -sqb rules list -sqb rules show -sqb rules run [--select ] [--exclude ] -sqb rules skills [--check] -``` - -Add `--json` after `rules` for machine-readable list or run output: - -```bash -sqb rules --json list -sqb rules --json run SQBRSQL -``` - -Lifecycle progress is written to stderr so stdout remains valid JSON. - -### Commands - -- `list`: print built-in and repository-defined Rules. -- `show`: print metadata for one exact code. -- `run`: compile and evaluate only one exact code or family prefix. -- `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. - -`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. - -### Exit codes - -| Command | Code | Meaning | -|---|---:|---| -| `rules list` | 0 | Catalogue printed | -| `rules show` | 0 | Exact code found | -| `rules show` | 2 | Unknown code | -| `rules run` | 0 | No findings | -| `rules run` | 1 | Findings reported or compilation failed | -| `rules run` | 2 | Unknown code or family | -| `rules skills --check` | 0 | Guidance is fresh | -| `rules skills --check` | 1 | Guidance is missing or stale | - -## plan - -Source: `cli/plan.mdx` - -Preview what SQLBuild will do before executing. - -## sqb plan - -Shows the execution plan without making any changes. Useful for inspecting change detection, backfill policies, and selector scope before building. - -### Usage - -```bash -sqb --project-dir plan [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | -| `--changes-only` | Virtual mode only: show work not already bound to its expected VDE version | -| `--no-python` | Exclude read-side Python tasks and assets from the plan | -| `--defer-to` | Resolve unselected model references against another target | -| `--json` | Output the plan as JSON | -| `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | -| `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | -| `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | -| `--start-cursor-int` | Override start cursor for integer incremental models | -| `--end-cursor-int` | Override end cursor for integer incremental models | -| `--select`, `-s` | Select specific models | -| `--exclude` | Exclude specific models | - -### Output - -First run: - -``` -Plan ready (13 selected) - -First run (12) - stg_customers view - stg_payments view - stg_orders view - daily_order_partitioned partition_tracked (custom) - daily_revenue table - dim_customers table - fact_orders table - customer_status_snapshot merge (timestamp) - hourly_order_activity delete_insert (timestamp, microbatch) - daily_activity_rollup delete_insert (timestamp, microbatch) - hourly_activity_with_daily_context delete_insert (timestamp, microbatch) - order_status_index delete_insert (integer) - -Seeds (1) - waffle_types -``` - -Steady state: - -``` -Plan ready (13 selected) - -Normal (12) - 3 view - 3 table - 3 delete_insert (timestamp, microbatch) - 1 partition_tracked (custom) - 1 merge (timestamp) - 1 delete_insert (integer) - -Seeds (1) - waffle_types -``` - -When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. - -### Missing upstream dependencies - -Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: - -``` -error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) -``` - -Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. - -## build - -Source: `cli/build.mdx` - -Compile, plan, and execute the selected build lifecycle. - -## sqb build - -Compiles, plans, and executes the selected build lifecycle. Direct mode runs the full selected -scope. [Virtual environments](/concepts/virtual-environments) can additionally use -`--changes-only` to build work not already represented by the target VDE's bound versions. Use -`--no-tests` and `--no-audits` to skip validation for fast iteration. - -### Usage - -```bash -sqb --project-dir build [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--target` | Build against a configured target instead of the active/default target | -| `--changes-only` | Virtual mode only: build work not already bound to its expected VDE version | -| `--no-tests` | Skip SQL unit tests | -| `--no-audits` | Skip audits | -| `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | -| `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | -| `--defer-to` | Resolve unselected model references against another target | -| `--defer-sources-to` | Read managed source data from another target | -| `--fail-fast` | Stop on first failure and skip remaining nodes | -| `--concurrency` | Number of worker connections (default: 1) | -| `--verbose`, `-v` | Show lifecycle SQL inline after each model | -| `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | -| `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | -| `--start-cursor-int` | Override start cursor for integer incremental models | -| `--end-cursor-int` | Override end cursor for integer incremental models | -| `--load` | Explicitly load managed sources before building | -| `--no-load` | Skip automatic source loading | -| `--reload` | Reload managed sources (passes `is_reload=True` to loaders) | -| `--include-stale-upstreams` | Expand selection to include stale upstream models needed for coherence | -| `--manifest` | Generate `target/manifest.json` with plan-aware project metadata | -| `--select`, `-s` | Select specific models | -| `--exclude` | Exclude specific models | - -### Fast iteration - -Use `--no-tests` and `--no-audits` to skip validation when you only want to materialize models: - -```bash -sqb build --no-tests --no-audits -``` - -This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. - -### Execution order - -1. Managed sources are loaded (unless `--no-load`) -2. Seeds are loaded (if changed) -3. Source audits run before their dependent models (unless `--no-audits`) -4. SQL unit tests run before their target model (unless `--no-tests`) -5. Models are materialized in DAG topological order (virtual `--changes-only` limits this to stale work) -6. Error-severity audits run against the staging table before promotion to the target (unless `--no-audits`) - -### Output - -``` -Execution sqb build (concurrency: 1) - - 1/13 seed waffle_types OK 0.09s - 2/13 view stg_customers OK 0.05s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - 3/13 view stg_orders OK 0.03s - test test_stg_orders PASS - audit not_null (order_id) PASS - audit unique (order_id) PASS - 10/13 table hourly_order_activity (delete_insert) OK 0.16s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS - audit (f) not_null (activity_hour) PASS - -Completed successfully. -PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) -``` - -### Deferred builds - -Use `--defer-to` to resolve unselected model references against another target. This lets you build a subset of models in dev while referencing production tables for everything else: - -```bash -sqb build --select fact_orders --defer-to prod -``` - -No `manifest.json` is required. `--defer-to` selects only the namespace used for unselected -references. SQLBuild resolves those relations through the active target's sole physical -connection and never opens the deferred target's connection. The deferred namespace must be -visible and readable through the active connection; deferral is not a cross-account, -cross-server, or cross-file transfer mechanism. - -### Failure behavior - -When a model fails: -- Downstream models are automatically blocked and skipped -- Staging/delta tables are retained for inspection -- Failure details show the model name, failed phase, and error message - -### Fingerprints - -After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. See [Planning and Change Detection](/concepts/planning) for details. - -### Runtime artifacts - -Build writes executed lifecycle SQL to `target/run/models/`. These files contain the actual SQL that was executed, including resolved cursor bounds and runtime substitutions. - -## load - -Source: `cli/load.mdx` - -Load managed sources into the warehouse. - -## sqb load - -Runs source loader functions and writes data into their target tables using the configured write strategy. - -### Usage - -```bash -sqb --project-dir load [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--select`, `-s` | Select specific sources or loaders by name | -| `--exclude` | Exclude specific sources or loaders | -| `--reload` | Pass `is_reload=True` to loader functions | -| `--concurrency` | Number of worker connections (default: from settings or 1) | -| `--start-cursor-ts` | Override start cursor for timestamp-based loaders (ISO format) | -| `--end-cursor-ts` | Override end cursor for timestamp-based loaders (ISO format) | -| `--start-cursor-int` | Override start cursor for integer-based loaders | -| `--end-cursor-int` | Override end cursor for integer-based loaders | -| `--json` | Output results as JSON | -| `--json-output` | Write JSON results to a file path | -| `--var` | Set project variables (`--var key=value`) | - -### Examples - -```bash -# Load all managed sources -sqb load - -# Load a specific source -sqb load --select raw_customers - -# Load with concurrency -sqb load --concurrency 4 - -# Reload all sources (full refresh behavior) -sqb load --reload - -# Override cursor bounds -sqb load --start-cursor-ts "2026-05-01T00:00:00" -``` - -### Execution order - -Loaders are executed in DAG topological order based on `depends_on` declarations. Independent loaders run concurrently when `--concurrency` is greater than 1. - -Intermediate loaders (those referenced only via `depends_on` without a direct source binding) run first, followed by the source-bound loaders that depend on them. - -### Output - -``` -Load ready (3 selected) - -Sources (3) - raw_customers - raw_orders - raw_payments - -Execution sqb load (concurrency: 1) - - 1/3 source raw_customers OK 0.05s rows=5 - 2/3 source raw_orders OK 0.03s rows=10 - 3/3 source raw_payments OK 0.02s rows=8 - -Completed successfully. -PASS=3 WARN=0 FAIL=0 SKIP=0 TOTAL=3 (0.12s) -``` - -### Auto-load during builds - -`sqb build` automatically loads managed sources before building dependent models. This is controlled by the `auto_load_sources` setting (default: `true`) and the `--load` / `--no-load` / `--reload` flags: - -```bash -# Default: auto-load is on -sqb build - -# Explicitly skip loading -sqb build --no-load - -# Force reload -sqb build --reload -``` - -See [Loaders](/concepts/python-nodes/loaders) for full documentation on write strategies, the loader context API, and source deferral. - -## seed - -Source: `cli/seed.mdx` - -Load seed CSV files into the warehouse. - -## sqb seed - -Loads seed CSV files into the warehouse as tables. Seeds are fully replaced on every run. - -### Usage - -```bash -sqb --project-dir seed [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--select`, `-s` | Select specific seeds by name | -| `--exclude` | Exclude specific seeds | - -### Examples - -```bash -# Load all seeds -sqb seed - -# Load a specific seed -sqb seed --select seed:waffle_types -``` - -## test - -Source: `cli/test.mdx` - -Run SQL unit tests and multi-model tests in isolation. - -## sqb test - -Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without -building models. Useful for validating test logic independently. - -### Usage - -```bash -sqb --project-dir test [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--select`, `-s` | Select tests targeting specific models | -| `--exclude` | Exclude tests targeting specific models | - -Parameterized cases use parent-level selection. Selecting a target model includes every case in -each matching `TEST` template; case names are not model selectors. - -Text output identifies each case as ` []` and includes its source path and safe typed -parameters. Structured JSON emits one check per case with stable source/block/case identity, -declared parameter types and nullability, typed values, and a content fingerprint. Exact decimals -are strings in JSON so their scale is preserved. - -### Examples - -```bash -# Run all tests -sqb test - -# Run tests for a specific model -sqb test --select stg_orders -``` - -## scenario - -Source: `cli/scenario.mdx` - -Run end-to-end scenario tests against the warehouse or locally with DuckDB. - -## sqb scenario - -Run end-to-end scenario tests. Scenarios materialize fixture inputs as physical relations, build the real project graph against them, and evaluate expected outputs and assertions. See [Scenarios](/concepts/scenarios) for concepts and authoring details. - -### sqb scenario test - -Run scenario tests against the warehouse. - -```bash -sqb scenario test [flags] -``` - -#### Flags - -| Flag | Description | -|------|-------------| -| `--select`, `-s` | Select scenarios to run | -| `--exclude` | Exclude scenarios from the selection | -| `--retain` | Keep scenario-owned warehouse artifacts for inspection | -| `--local` | Run locally against DuckDB using captured JSONL snapshots | -| `--strict` | Treat missing/stale local snapshots as errors instead of skips | -| `--sync-snapshots` | Capture missing/stale snapshots before local run (requires `--local`) | -| `--refresh` | Recapture all selected snapshots before local run (requires `--local`) | -| `--force` | Bypass snapshot capture safety limits | -| `--max-snapshot-rows` | Override per-relation row limit for capture | -| `--max-snapshot-total-rows` | Override total row limit for capture | -| `--max-snapshot-bytes` | Override per-relation byte limit for capture | -| `--max-snapshot-total-bytes` | Override total byte limit for capture | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | - -#### Selectors - -Scenarios are selected with `--select`. `--exclude` removes scenarios from the selection. Without any selectors, all discovered scenarios run. - -| Selector | Example | -|----------|---------| -| Scenario name | `sqb scenario test --select daily_revenue_minimal` | -| Multiple names | `sqb scenario test --select daily_revenue_minimal --select daily_revenue_multi_order` | -| `.sql` file path | `sqb scenario test --select tests/scenarios/revenue/daily_revenue_minimal.sql` | -| Folder | `sqb scenario test --select tests/scenarios/revenue` | -| Scenario-root-relative folder | `sqb scenario test --select revenue` | -| Exclude | `sqb scenario test --select revenue --exclude daily_revenue_multi_order` | - -Mixed selector types are supported and the result is de-duplicated by scenario name. - -#### Remote examples - -```bash -# Run all scenarios -sqb scenario test - -# Run one scenario -sqb scenario test --select daily_revenue_minimal - -# Run and retain warehouse artifacts -sqb scenario test --select daily_revenue_minimal --retain - -# Run all scenarios in a folder -sqb scenario test --select revenue -``` - -#### Local examples - -```bash -# Run locally (requires prior capture) -sqb scenario test --local - -# Run locally, capture missing/stale snapshots first -sqb scenario test --local --sync-snapshots - -# Run locally, recapture everything first -sqb scenario test --local --refresh - -# Fail on missing/stale snapshots instead of skipping -sqb scenario test --local --strict -``` - -#### Output - -Remote scenarios report per-scenario PASS/FAIL with nested check rows: - -``` -daily_revenue_minimal PASS - check expected daily_revenue PASS - check assertion no_negative_revenue PASS - -PASS=1 FAIL=0 TOTAL=1 -``` - -Local scenarios add ERROR and SKIP statuses: - -``` -PASS=2 FAIL=0 ERROR=0 SKIP=1 TOTAL=3 -``` - -Failed scenarios suggest rerunning with `--retain` for inspection. Local runs always keep the DuckDB file at `target/run/scenarios//local.duckdb`. - -### sqb scenario capture - -Capture scenario input fixtures from the warehouse as JSONL snapshots for local replay. - -```bash -sqb scenario capture [flags] -``` - -#### Flags - -| Flag | Description | -|------|-------------| -| `--select`, `-s` | Select scenarios to capture | -| `--exclude` | Exclude scenarios from the selection | -| `--retain` | Keep warehouse fixture artifacts after capture | -| `--force` | Bypass snapshot capture safety limits | -| `--max-snapshot-rows` | Override per-relation row limit | -| `--max-snapshot-total-rows` | Override total row limit | -| `--max-snapshot-bytes` | Override per-relation byte limit | -| `--max-snapshot-total-bytes` | Override total byte limit | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | - -#### Examples - -```bash -# Capture all scenarios -sqb scenario capture - -# Capture one scenario -sqb scenario capture --select daily_revenue_minimal - -# Capture and retain warehouse artifacts -sqb scenario capture --select daily_revenue_minimal --retain -``` - -Snapshots are written to `tests/_scenario_snapshots//` with a `scenario.json` manifest and JSONL files for each fixture relation. These files can be committed to version control. - -### Runtime artifacts - -Both remote and local scenario runs write runtime artifacts to `target/run/scenarios//`: - -``` -target/run/scenarios/daily_revenue_minimal/ - cleanup/ - prepare.sql - final.sql - fixtures/ - ref__stg_orders.sql - ref__stg_payments.sql - models/ - marts/daily_revenue.sql - checks/ - expected__daily_revenue.sql - assertion__no_negative_revenue.sql -``` - -Local runs additionally write to a `local/` subdirectory and create `local.duckdb`. - -## audit - -Source: `cli/audit.mdx` - -Run data quality audits in isolation. - -## sqb audit - -Runs all attached audits without rebuilding models. Useful for verifying data quality on existing warehouse state. - -### Usage - -```bash -sqb --project-dir audit [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--defer-to` | Resolve model references against another target | -| `--select`, `-s` | Select audits attached to specific models | -| `--select-file` | Read selectors from a file | -| `--exclude` | Exclude audits attached to specific models | -| `--concurrency` | Maximum number of audits to run concurrently (default: `1`) | -| `--json` | Print structured audit results | -| `--json-output` | Write structured audit results to a file | - -### Concurrency - -Standalone audits run serially by default. Use `--concurrency` to opt into parallel warehouse -queries: - -```bash -sqb audit --concurrency 8 -``` - -SQLBuild resolves the limit in this order: an explicit `--concurrency` value, the -`SQLBUILD_CONCURRENCY` environment variable, the effective project `settings.concurrency`, then -the product default of `1`. Values must be at least `1`. - -The physical worker count is bounded by the number of selected audits. Each active worker opens -and exclusively uses one warehouse connection, so higher concurrency can increase warehouse load -and cost. Results and final JSON remain in plan order even when queries finish in a different -order. - -If an audit query raises a runtime, adapter, or framework error, SQLBuild records that failure and -continues running the remaining independent audits. After all selected audits have been attempted, -it reports every query failure in plan order and exits nonzero. Failed queries do not produce -fabricated audit results. - -Interruption and cancellation are different: SQLBuild stops scheduling new audits, drains queries -that have already started, and then closes every worker connection. Generic adapters cannot -guarantee immediate cancellation of a query already running in the warehouse. - -### Examples - -```bash -# Run all audits -sqb audit - -# Run audits for marts only -sqb audit --select path:models/marts - -# Run up to 8 audit queries concurrently -sqb audit --concurrency 8 -``` - -## freshness - -Source: `cli/freshness.mdx` - -Observe source freshness without writing state. - -## sqb freshness - -Observes the current data version of each source and reports whether the data has changed since the last build. Does not write any state or trigger builds. - -### Usage - -```bash -sqb --project-dir freshness [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--state` | Compare observations against stored freshness state from the last build | -| `--fail-on-error` | Exit with code 1 if any source observation fails or is unknown | -| `--fail-on-stale` | Exit with code 1 if any source has changed, is unknown, or errored (requires `--state`) | -| `--virtual-env` | Read previous state from the specified virtual environment instead of direct state | -| `--json` | Output as JSON instead of human-readable text | -| `--json-output` | Write JSON output to a file path (also prints text to stdout unless `--json` is set) | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--select`, `-s` | Select specific sources or models (sources upstream of selected models are included) | -| `--exclude` | Exclude specific sources or models | - -### Source selection - -Without `--select`, all sources in the project are observed. When `--select` is provided, selectors work like other commands: you can select sources by name, or select models and their upstream sources are automatically included: - -```bash -# Observe all sources -sqb freshness - -# Observe a specific source -sqb freshness --select raw_orders - -# Observe sources upstream of a model -sqb freshness --select fact_orders -``` - -### Observation without state - -By default, `sqb freshness` observes the current data version of each source and reports what it found. No comparison is made against previous observations: - -```bash -sqb freshness -``` - -``` -Source freshness - -Observed (3) - raw_customers timestamp 2026-06-05T14:30:00 adapter - raw_orders timestamp 2026-06-05T15:45:00 column tolerance 15m - raw_payments integer 42871 column - -Summary: observed=3 changed=0 unchanged=0 tolerated=0 unknown=0 errors=0 -``` - -Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata. Sources that can't be observed (expression sources, managed sources without freshness config on unsupported adapters) show as `unknown`. - -### Comparing against state - -Use `--state` to compare current observations against the freshness state stored from the last successful build: - -```bash -sqb freshness --state -``` - -``` -Source freshness - -Changed (1) - raw_orders previous 2026-06-05T12:00:00 current 2026-06-05T15:45:00 tolerance 15m - -Unchanged (1) - raw_customers previous 2026-06-05T14:30:00 current 2026-06-05T14:30:00 - -Tolerated (1) - raw_payments previous 2026-06-05T14:28:00 current 2026-06-05T14:30:00 tolerance 15m - -Summary: observed=0 changed=1 unchanged=1 tolerated=1 unknown=0 errors=0 -``` - -#### Statuses - -| Status | Meaning | -|--------|---------| -| `observed` | Successfully observed (no state comparison) | -| `changed` | Data version differs from the stored state | -| `unchanged` | Data version matches the stored state exactly | -| `tolerated` | Data version differs but is within the `lag_tolerance` threshold | -| `unknown` | No freshness config and adapter metadata unavailable, or no previous state to compare against | -| `error` | Observation failed (e.g. source table does not exist, query error) | - -#### Virtual environment state - -To compare against state stored in a virtual environment instead of direct mode state: - -```bash -sqb freshness --state --virtual-env pr_123 -``` - -### CI integration - -Use `--fail-on-error` to fail the pipeline if any source can't be observed: - -```bash -sqb freshness --fail-on-error -``` - -Use `--fail-on-stale` with `--state` to fail if any source has new data that hasn't been built yet: - -```bash -sqb freshness --state --fail-on-stale -``` - -This is useful for CI gates that should block if source data has changed but the pipeline hasn't run yet. - -### JSON output - -```bash -sqb freshness --json -``` - -```json -{ - "sources": [ - { - "name": "raw_orders", - "status": "observed", - "strategy": "column", - "value_kind": "timestamp", - "current_data_version": "2026-06-05T15:45:00", - "previous_data_version": null, - "lag_tolerance": "15m", - "target": { - "database": null, - "schema": "raw", - "name": "orders" - }, - "message": null - } - ], - "summary": { - "observed": 1, - "changed": 0, - "unchanged": 0, - "tolerated": 0, - "unknown": 0, - "errors": 0 - } -} -``` - -### See also - -- [Sources: Source freshness](/concepts/sources#source-freshness) for freshness configuration -- [Planning and Change Detection](/concepts/planning) for how freshness feeds into change-aware builds - -## check - -Source: `cli/check.mdx` - -Run Python checks against tasks, assets, and loaders. - -## sqb check - -Runs Python [checks](/concepts/python-nodes/checks) in isolation. Checks validate the output of tasks, assets, and loaders. For SQL relation validation, use [`sqb audit`](/cli/audit) instead. - -### Usage - -```bash -sqb --project-dir check [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--select`, `-s` | Select checks to run (by name, `check:`, `tag:`, or graph expansion) | -| `--exclude` | Exclude checks from the selection | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--json` | Print check results as JSON | -| `--json-output` | Write check results JSON to a file path | -| `--vars` | Override project variables | - -Selecting a non-check node is rejected; use [`sqb build`](/cli/build) to run tasks and assets. - -### Examples - -```bash -# Run all checks -sqb check - -# Run one check and its required dependencies -sqb check --select +check_orders_exported - -# Run checks by tag -sqb check --select tag:exports - -# Print results as JSON -sqb check --json -``` - -### Output - -Checks report PASS, FAIL, or WARN per check. A failing `error`-severity check exits non-zero; `warn`-severity failures are reported without failing the command. - -Results are written to `target/run/checks/python_checks.json`. - -### Relationship to build - -`sqb build` runs relevant Python checks by default after the tasks, assets, and loaders they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. - -## clone - -Source: `cli/clone.mdx` - -Copy model relations between configured targets. - -## sqb clone - -Copies selected relations from one target to another. It uses adapter-native cloning where supported and physical copies where required; `--hard-copy` forces a physical copy on adapters that support both. - -No `manifest.json` generation or artifact management is required. Clone works directly against live targets. - -When `--to` is omitted, the destination is the active target selected by `--target`, -`sqlbuild_local.toml`, or `default_target`, in that order. - -`--from` selects only the origin namespace and its clone-origin policy. `--to`, or the active -target when it is omitted, selects the sole physical connection used by the operation. The -origin namespace must therefore be visible and readable through the destination connection. -SQLBuild never resolves or opens the origin target's connection, including for direct, -deferred, and virtual clone. Clone cannot perform a cross-account, cross-server, or cross-file -transfer by opening a second connection. - -### Usage - -```bash -sqb --project-dir clone --from [--to ] [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--from` | Source target (required) | -| `--to` | Destination target; defaults to the active target | -| `--hard-copy` | Force physical table copies instead of zero-copy cloning | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--select`, `-s` | Select specific models to clone | -| `--exclude` | Exclude specific models from cloning | - -### Examples - -```bash -# Clone all models from prod to dev -sqb clone --from prod --to dev - -# Equivalent when dev is the active target -sqb clone --from prod - -# Clone only marts to dev -sqb clone --from prod --to dev --select path:models/marts - -# Force physical copies -sqb clone --from prod --to dev --hard-copy -``` - -### Managed sources - -Clone includes selected managed physical sources when the destination target reads its own -loader namespace. Sources are copied before seeds and models so views can be recreated in a -destination that has never been built. - -Source locations respect target configuration: - -- The origin relation comes from the target that the origin uses for managed source reads. -- The destination relation uses the destination target's `loader_schema`, falling back to - its model `schema`. -- If the destination defers source reads to another target, clone reuses that target's source - relation and does not copy or overwrite it. -- Expression sources and unmanaged external sources are not cloned. - -`--hard-copy` controls how included relations are copied. It does not change which managed -sources are selected. - -### Clone policies - -Targets deny cloning by default. Enable the origin and destination explicitly in `sqlbuild_project.toml`: - -```toml -[targets.prod.clone] -allow_as_clone_origin = true - -[targets.dev.clone] -allow_as_clone_destination = true -``` - -The origin target must already contain the built relations being cloned. The destination -connection's credentials must be able to read the origin namespace and create relations in the -destination. Managed physical sources may bootstrap a destination that has not been built. See -[Project Configuration](/concepts/project-configuration) for details. - -## diff - -Source: `cli/diff.mdx` - -Compare schemas and data between targets, virtual environments, or two literal read-only queries. - -Compares schemas and optionally row-level data between two build contexts, or materialized results -from two literal read-only SQL inputs. See [Data Diffs](/concepts/diff) for detailed usage. - -### Usage - -```bash -sqb diff : [flags] -sqb diff (--left-query SQL | --left-query-file PATH) \ - (--right-query SQL | --right-query-file PATH) (--key COLUMN... | --unkeyed) [flags] -``` - -Model mode uses a positional `FROM:TO` range and requires exactly one of `--full`, `--schema-only`, -or `--bounded `. Raw-query mode does not accept `FROM:TO`; it defaults to full comparison, -also supports `--schema-only`, and does not support `--bounded`. - -In direct mode, `FROM` and `TO` are configured target names. Their database/schema namespaces remain -authoritative, while the `TO` target's named connection executes the complete comparison and must -be able to read both namespaces. - -Full and bounded keyed comparisons use the model's `unique_key` unless repeated `--key` arguments -override it. `--unkeyed` opts into exact multiset comparison. Bounded mode uses the model's cursor -and falls back to a full row comparison when no cursor is configured. - -### Flags - -| Flag | Description | -|------|-------------| -| `--full` | Compare both schema and all row data | -| `--schema-only` | Compare column names and types only | -| `--bounded` | Compare row data within a recent window (e.g. `14d`, `6h`) | -| `--verbose`, `-v` | Show more example rows (default: 3, verbose: 10) | -| `--max-column-examples` | Override maximum examples per changed column | -| `--max-row-only-examples` | Override maximum examples for side-only rows | -| `--sample-rows` | Override the deterministic unique-key sample size | -| `--sample-seed` | Override the deterministic sample seed | -| `--exhaustive` | Disable inherited sampling for this invocation | -| `--json-output` | Write structured comparison scope, coverage, and results to a JSON file | -| `--json` | Write one structured JSON document to stdout; progress remains on stderr | -| `--max-models` | Fail when the selected scope contains more models than this limit | -| `--max-columns` | Fail when either side of a model has more columns than this limit | -| `--left-query` | Literal read-only SQL for the left side | -| `--left-query-file` | UTF-8 SQL file for the left side | -| `--right-query` | Literal read-only SQL for the right side | -| `--right-query-file` | UTF-8 SQL file for the right side | -| `--left-label` | Display and structured-output label for the left query | -| `--right-label` | Display and structured-output label for the right query | -| `--target` | Target connection for raw-query mode; defaults to the active target | -| `--key` | Row identity column; repeat for a composite key or to override model configuration | -| `--unkeyed` | Compare exact full-row multiplicities without a key | -| `--exclude-column` | Column to omit from row comparison; repeat as needed | -| `--tolerance` | Per-column numeric rule such as `amount:absolute=0.01`; repeat as needed | -| `--max-value-length` | Maximum rendered characters per example value (default: 160) | -| `--no-example-values` | Retain example keys and counts while suppressing values | -| `--full-example-values` | Explicitly opt in to complete example values | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--select`, `-s` | Select specific models to diff (required in v1) | -| `--exclude` | Exclude specific models from diffing | - -### Examples - -```bash -# Full diff of a specific model -sqb diff prod:dev --full --select customer_status_snapshot - -# Schema-only diff of all marts -sqb diff prod:dev --schema-only --select path:models/marts - -# Bounded diff of last 14 days -sqb diff prod:dev --bounded 14d --select hourly_order_activity - -# Deterministic bounded sample -sqb diff prod:dev --bounded 14d --sample-rows 50000 --sample-seed 7 --select order_lines - -# Force exhaustive comparison despite inherited sampling defaults -sqb diff prod:dev --bounded 14d --exhaustive --select order_lines - -# Compare two raw queries with a composite key -sqb diff \ - --left-query-file queries/orders_before.sql \ - --right-query-file queries/orders_after.sql \ - --left-label before \ - --right-label after \ - --key account_id \ - --key order_id -``` - -### Exit codes - -Returns `0` when no differences are found. Returns `1` when any input has schema or row differences -or the comparison cannot execute successfully. - -## lineage - -Source: `cli/lineage.mdx` - -Explore model and column-level dependency graphs from the command line. - -Inspect upstream and downstream dependencies for any model, source, seed, or function in your project. Supports both model-level lineage (dependency graph) and column-level lineage (tracing individual columns through transformations). Outputs as a tree, edge list, or structured JSON. - -`sqb compile` computes lineage for validation and includes per-model summaries in its JSON report. Use this command when you need to inspect or export the actual lineage graph. - -### Usage - -```bash -# Model lineage: dependency graph around a single resource -sqb lineage [flags] - -# Column lineage: trace a specific column -sqb lineage . [flags] - -# Selector mode: lineage for a selected scope -sqb lineage --select [flags] -``` - -Exactly one of a positional target or `--select` is required. - -### Flags - -| Flag | Description | -|------|-------------| -| `--direction` | `upstream` (default), `downstream`, or `both`. `both` is only available for model lineage. | -| `--depth` | How many hops to traverse. An integer or `all` (default: `all`). | -| `--format` | Output format: `tree` (default), `list`, or `json`. | -| `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | -| `--select`, `-s` | Select resources using standard selector syntax. | -| `--exclude` | Exclude resources from the selection. | - -### Model lineage - -When the target is a plain resource name, lineage shows the model-level dependency graph. - -#### Tree - -The default. Shows an indented dependency tree with resource types and file paths: - -```bash -sqb lineage fact_orders --direction both -``` - -``` -Lineage model fact_orders models/marts/fact_orders.sql both -upstream -├── model stg_orders models/staging/stg_orders.sql -│ └── source raw__orders sources/raw.yml -├── model stg_payments models/staging/stg_payments.sql -│ └── source raw__payments sources/raw.yml -├── seed waffle_types seeds/waffle_types.csv -└── function udf__is_completed_order functions/sql/udf__is_completed_order.sql -downstream -├── model customer_status_snapshot models/intermediate/customer_status_snapshot.sql -├── model hourly_order_activity models/marts/hourly_order_activity.sql -│ ├── model daily_activity_rollup models/marts/daily_activity_rollup.sql -│ │ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql -│ │ └── model hourly_order_activity (already shown) -│ └── model hourly_activity_with_daily_context (already shown) -└── model order_status_index models/intermediate/order_status_index.sql -``` - -Cycles and repeated nodes are annotated with "(already shown)" to avoid infinite recursion. - -#### List - -An edge list showing each dependency as a directed pair: - -```bash -sqb lineage fact_orders --format list -``` - -``` -source:raw__orders -> model:stg_orders -source:raw__payments -> model:stg_payments -model:stg_orders -> model:fact_orders -model:stg_payments -> model:fact_orders -seed:waffle_types -> model:fact_orders -``` - -#### JSON - -Structured output with nodes, edges, and metadata: - -```bash -sqb lineage fact_orders --format json -``` - -```json -{ - "nodes": [ - { - "id": "model:fact_orders", - "name": "fact_orders", - "resource_type": "model", - "relative_path": "models/marts/fact_orders.sql", - "qualified_name": "dev.fact_orders" - }, - { - "id": "source:raw__orders", - "name": "raw__orders", - "resource_type": "source", - "relative_path": "sources/raw.yml" - } - ], - "edges": [ - {"from": "source:raw__orders", "to": "model:stg_orders"}, - {"from": "model:stg_orders", "to": "model:fact_orders"} - ], - "focus": ["model:fact_orders"], - "direction": "upstream" -} -``` - -### Column lineage - -When the target uses `model.column` syntax, lineage traces the specific column through upstream or downstream transformations. Each edge is annotated with a transform type (`direct`, `expression`, `aggregation`, `cast`, `star`, `constant`) and a confidence level. See [Column Lineage](/concepts/column-lineage) for a full explanation of transform types, confidence levels, and analysis modes. - -```bash -# Where does fact_orders.total_cents come from? -sqb lineage fact_orders.total_cents - -# What consumes fact_orders.order_id downstream? -sqb lineage fact_orders.order_id --direction downstream -``` - -Column lineage supports `upstream` and `downstream` directions (not `both`). The `--mode` flag selects the analysis mode: `rich` (default, full SQL analysis) or `fast` (lightweight, faster on large projects). - -#### Tree - -```bash -sqb lineage fact_orders.total_cents -``` - -``` -Column trace fact_orders.total_cents upstream - - <- stg_payments.amount_cents (expression) - <- raw__payments.amount_cents (direct) -``` - -#### List - -```bash -sqb lineage fact_orders.total_cents --format list -``` - -``` -Column dependencies - -stg_payments.amount_cents -> fact_orders.total_cents expression -raw__payments.amount_cents -> stg_payments.amount_cents direct -``` - -#### JSON - -```bash -sqb lineage fact_orders.total_cents --format json -``` - -```json -{ - "target": { - "resource_type": "model", - "resource_name": "fact_orders", - "column_name": "total_cents" - }, - "direction": "upstream", - "metadata": { - "mode": "rich", - "max_depth": null, - "analyzed_models": 5, - "truncated": false - }, - "trace": [ - { - "source": { - "resource_type": "model", - "resource_name": "stg_payments", - "column_name": "amount_cents" - }, - "target": { - "resource_type": "model", - "resource_name": "fact_orders", - "column_name": "total_cents" - }, - "transform": "expression", - "confidence": "high" - } - ] -} -``` - -### Examples - -```bash -# Upstream dependencies of a model (default) -sqb lineage fact_orders - -# Downstream dependents -sqb lineage fact_orders --direction downstream - -# Both directions, limited to 1 hop -sqb lineage fact_orders --direction both --depth 1 - -# Column lineage: trace a specific column upstream -sqb lineage fact_orders.total_cents - -# Column lineage: trace downstream consumers -sqb lineage fact_orders.order_id --direction downstream - -# Column lineage with depth limit -sqb lineage fact_orders.total_cents --depth 1 - -# Lineage for all models in a path -sqb lineage --select path:models/marts - -# Lineage between two models (path-between) -sqb lineage --select "stg_orders~daily_activity_rollup" - -# Upstream expansion from a model -sqb lineage --select "+fact_orders" - -# JSON output for programmatic consumption -sqb lineage fact_orders --format json --direction upstream -``` - -### Depth limiting - -`--depth` controls how many hops from the focus node(s) to include. In selector mode, `--depth` requires name, source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors. - -```bash -# Only immediate parents -sqb lineage fact_orders --depth 1 - -# Parents and grandparents -sqb lineage fact_orders --depth 2 - -# Only direct column dependencies -sqb lineage fact_orders.total_cents --depth 1 -``` - -## dag - -Source: `cli/dag.mdx` - -Generate the static DAG artifact for Dagster and other integrations. - -## sqb dag - -Compiles the project and outputs the static DAG artifact. The artifact contains every node (source, seed, model, function), dependency edge, and check (test, audit, scenario) in your project as structured JSON. It is the bridge between SQLBuild and external orchestrators like Dagster. - -This artifact describes resource dependencies, not column-level lineage. Use [`sqb lineage`](/cli/lineage) to inspect or export column traces. - -### Usage - -```bash -sqb --project-dir dag [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--json` | Print the full DAG artifact as JSON to stdout | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | -| `--vars` | JSON object of project variable overrides | - -Without `--json`, the command prints a summary: - -``` -DAG ready (24 nodes, 18 edges, 15 checks) -``` - -### Generating via compile - -You can also generate the DAG artifact as part of a compile: - -```bash -# Write to default location (target/sqlbuild_dag.json) -sqb compile --dag - -# Write to a specific path -sqb compile --dag target/my_dag.json -``` - -This is useful when you want to compile and generate the DAG in one step. The `SqlBuildProject.prepare()` method in the Dagster integration uses this path. - -### Output format - -The JSON artifact has this structure: - -```json -{ - "version": 1, - "project_name": "waffle_shop", - "nodes": [...], - "edges": [...], - "checks": [...] -} -``` - -#### Nodes - -Each node represents a source, seed, model, or function: - -```json -{ - "id": "model:fact_orders", - "kind": "model", - "name": "fact_orders", - "asset_key": ["dev", "fact_orders"], - "target": { - "database": null, - "schema": "dev", - "name": "fact_orders", - "qualified_name": "dev.fact_orders" - }, - "path": "models/marts/fact_orders.sql", - "description": "Order fact table with waffle and payment details.", - "tags": ["marts"], - "columns": [ - {"name": "order_id", "type": "INTEGER"}, - {"name": "customer_id"} - ], - "materialization_type": "table" -} -``` - -| Field | Description | -|-------|-------------| -| `id` | Unique identifier (`{kind}:{name}`) | -| `kind` | `source`, `seed`, `model`, or `function` | -| `name` | Resource name | -| `asset_key` | Tuple used as the Dagster asset key (typically `[schema, name]` or `[database, schema, name]`) | -| `target` | Warehouse identity (database, schema, name, qualified_name) | -| `path` | Relative file path in the project | -| `description` | Model or source description, if declared | -| `tags` | Model tags | -| `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | -| `language` | For functions: `sql` or `python` | -| `return_kind` | For functions: `scalar` or `table` | -| `arguments` | For functions: argument name and type pairs | - -#### Edges - -Each edge is a dependency between two nodes: - -```json -{ - "from_id": "model:stg_orders", - "to_id": "model:fact_orders" -} -``` - -#### Checks - -Each check represents a test, audit, or scenario: - -```json -{ - "id": "audit:not_null:model:fact_orders:order_id", - "kind": "audit", - "name": "not_null", - "checked_asset_ids": ["model:fact_orders"], - "path": "audits/generic/not_null.sql", - "severity": "error", - "attached_target_name": "fact_orders", - "attached_column_name": "order_id" -} -``` - -| Field | Description | -|-------|-------------| -| `id` | Unique check identifier | -| `kind` | `sql_test`, `audit`, or `scenario` | -| `name` | Check name | -| `checked_asset_ids` | Node IDs this check is attached to | -| `path` | Relative file path | -| `severity` | For audits: `error` or `warn` | -| `mode` | For tests: `model`, `macro`, `udf`, or `table_fn` | -| `assertion_names` | For scenarios: names of `__assert__` CTEs | -| `expected_model_names` | For scenarios: names of `__expected__` models | -| `fixture_refs` | For scenarios: names of fixture sources, refs, and seeds | - -### Examples - -```bash -# Summary output -sqb dag - -# Full JSON to stdout -sqb dag --json - -# Generate as part of compile -sqb compile --dag - -# Generate to a specific path -sqb compile --dag target/sqlbuild_dag.json - -# Pipe to jq for inspection -sqb dag --json | jq '.nodes | length' -``` - -## query - -Source: `cli/query.mdx` - -Run ad hoc SQL queries against the project database. - -## sqb query - -Execute ad hoc SQL against the active project connection. Useful for inspecting data, debugging models, or running one-off queries without leaving the SQLBuild CLI. - -### Usage - -```bash -sqb --project-dir query "SELECT * FROM dev.fact_orders LIMIT 5" -``` - -Or from a file: - -```bash -sqb --project-dir query --file my_query.sql -``` - -File paths are resolved from the current working directory. - -### Flags - -| Flag | Description | -|------|-------------| -| `sql` | SQL to execute (positional argument) | -| `--file` | Read SQL from a file instead of the command line | -| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | -| `--limit` | Maximum rows to return (default: 20) | -| `--no-limit` | Disable the row limit | - -### Output formats - -#### long (default) - -Vertical record format, one field per line: - -``` --[ RECORD 1 ]---------------------------+ -order_id | 1 -customer_id | 100 -status | completed - --[ RECORD 2 ]---------------------------+ -order_id | 2 -customer_id | 200 -status | completed - -2 rows -``` - -#### table - -Horizontal table format: - -``` -order_id | customer_id | status --------- | ----------- | --------- -1 | 100 | completed -2 | 200 | completed - -2 rows -``` - -#### json - -JSON array of objects: - -```json -[{"order_id": 1, "customer_id": 100, "status": "completed"}] -``` - -#### csv - -Standard CSV with headers: - -```csv -order_id,customer_id,status -1,100,completed -2,200,completed -``` - -### Examples - -```bash -# Quick inspection with default format -sqb query "SELECT * FROM dev.fact_orders" - -# Table format with higher limit -sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 - -# Export to JSON -sqb query "SELECT * FROM dev.daily_revenue" --format json --no-limit - -# Run SQL from a file -sqb query --file debug_query.sql -``` - -## debug - -Source: `cli/debug.mdx` - -Validate project configuration and test the warehouse connection. - -Checks that your project config is valid, the adapter is resolvable, and the warehouse connection works. Useful for diagnosing setup issues. - -### Usage - -```bash -sqb debug [flags] -``` - -### What it checks - -The command runs three groups of checks: - -**Runtime** - SQLBuild version, Python version, Python path, OS info. - -**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter and active target. - -**Connection** - Displays connection settings (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. - -### Flags - -| Flag | Description | -|------|-------------| -| `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | -| `--json` | Output results as JSON instead of formatted text. | - -### Example - -```bash -sqb debug -``` - -``` -SQLBuild Diagnostics - -Runtime: - sqlbuild version: 0.2.1 - python version: 3.14.3 - python path: /home/user/.venv/bin/python3 - os info: Linux-6.6.87 - -Configuration: - project file: /home/user/waffle_shop/sqlbuild_project.toml [OK found and valid] - local config: /home/user/waffle_shop/sqlbuild_local.toml [OK found] - project: waffle_shop [OK loaded] - adapter: snowflake [OK found] - target: dev [OK resolved] - -Connection: - account: FJMQFQV-OJ66172 - authenticator: programmatic_access_token - database: SQB_DB - role: role_sqb_test - schema: TEST - token: **** - user: svc_sqb_test - warehouse: SQB_WH - connection test: [OK connected] - query test: [OK SELECT 1] -``` - -### Exit codes - -Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). - -## janitor - -Source: `cli/janitor.mdx` - -Clean up stale warehouse relations. - -## sqb janitor - -Identifies and removes stale warehouse relations that are no longer part of the project. Requires `janitor.enabled = true` in `sqlbuild_project.toml`. - -### Usage - -```bash -sqb --project-dir janitor [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--auto-approve` | Skip the confirmation prompt and delete immediately | -| `--retention-days` | Override the configured retention period (days) | - -### Configuration - -Configure janitor behavior in `sqlbuild_project.toml`: - -```toml -[janitor] -enabled = true -retention_days = 30 -delete_tracked_only = true -exclude_patterns = ["audit_*", "tmp_*"] -``` - -See [Project Configuration](/concepts/project-configuration) for details on janitor settings. - -### Examples - -```bash -# Interactive mode (prompts for confirmation) -sqb janitor - -# Auto-approve deletion -sqb janitor --auto-approve - -# Override retention to 7 days -sqb janitor --retention-days 7 -``` - -### State history pruning - -In addition to cleaning up stale relations, the janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` tables, retaining only the latest record per identity. This keeps state tables compact without affecting change detection. - -### Safety - -Janitor prompts for confirmation before deleting. The confirmation requires typing an exact string to prevent accidental deletion. Use `--auto-approve` only in CI or when you're certain. - -## clean - -Source: `cli/clean.mdx` - -Remove compiled artifacts from the target directory. - -## sqb clean - -Removes the `target/` directory containing compiled artifacts, runtime SQL recordings, and other build outputs. - -### Usage - -```bash -sqb --project-dir clean -``` - -No flags. This command has no confirmation prompt since it only removes local build artifacts, not warehouse data. - -## dbt - -Source: `cli/dbt.mdx` - -Coordinate dbt and SQLBuild projects. - -## sqb dbt - -Orchestrate dbt and SQLBuild together. Each subcommand runs dbt first, then SQLBuild, with selection logic across both project graphs. dbt remains responsible for dbt-owned models; SQLBuild validates and executes SQLBuild-owned models downstream. See [Using SQLBuild with dbt](/concepts/dbt-compatibility/overview) for scope and selection behavior. - -### sqb dbt plan - -Preview combined dbt and SQLBuild work without executing. - -```bash -sqb dbt plan [--select ...] [--exclude ...] [--json] [--verbose] -``` - -Shows which dbt models will run, which SQLBuild models will run, and the dbt/SQLBuild commands that would be executed. - -### sqb dbt run - -Run dbt models first, then SQLBuild models. - -```bash -sqb dbt run [--select ...] [--exclude ...] -``` - -### sqb dbt build - -Build dbt models first (including dbt tests), then SQLBuild models with audits. - -```bash -sqb dbt build [--select ...] [--exclude ...] -``` - -### sqb dbt debug - -Run dbt diagnostics followed by SQLBuild diagnostics. - -```bash -sqb dbt debug [--no-connection] -``` - -### Selectors - -All `sqb dbt` commands accept `--select` and `--exclude`. Selectors work across both dbt and SQLBuild: - -```bash -# SQLBuild model with dbt dependencies -sqb dbt build --select downstream_orders - -# Full upstream chain including dbt -sqb dbt build --select +downstream_orders - -# Downstream of modified dbt models -sqb dbt build --select state:modified+ - -# SQLBuild models by tag -sqb dbt build --select tag:nightly - -# SQLBuild models by path -sqb dbt build --select path:models/marts - -# Exclude by tag -sqb dbt build --select fact_orders+ --exclude tag:nightly -``` - -See [Selection](/concepts/dbt-compatibility/selection) for full details on how selectors route work between dbt and SQLBuild. - -### Configuration - -Configure the dbt project location in `sqlbuild_project.toml`: - -```toml -[dbt] -project_dir = "../dbt_project" -profiles_dir = "../profiles" -target_path = "../dbt_project/target" -``` - -See [Project Configuration](/concepts/project-configuration#dbt) for all fields. - -## state - -Source: `cli/state.mdx` - -Manage the virtual mode state store. - -## sqb state - -Manages the virtual mode state store lifecycle, locks, and checkpoints. Requires `virtual_environments = true`. - -### Usage - -```bash -sqb state [flags] -``` - -### Subcommands - -#### init - -Create state tables in the configured schema. - -```bash -sqb state init -``` - -#### migrate - -Back up the current state schema and re-initialize state tables. Creates a backup schema (e.g. `sqlbuild_state__backup_`) before re-initializing. - -```bash -sqb state migrate -``` - -#### rollback - -Restore state from a backup. - -```bash -# Restore from the latest backup -sqb state rollback - -# Restore from a specific backup -sqb state rollback --backup-id -``` - -#### reset - -Drop all state tables. Requires `allow_reset = true` in the target state config and `--auto-approve` on the command line. - -```bash -sqb state reset --auto-approve -``` - -#### adopt - -Convert existing stateless warehouse relations into versioned physical relations with VDE views. Interactive only - requires typed confirmation. - -```bash -sqb state adopt [--allow-copy] -``` - -| Flag | Description | -|------|-------------| -| `--allow-copy` | Allow CTAS copy fallback when native rename is not available | - -See [Adopt and Detach](/concepts/virtual-environments/adopt-detach) for details. - -#### detach - -Collapse a VDE back into stateless warehouse relations. Interactive only - requires typed confirmation. The VDE must be finalized. - -```bash -sqb state detach [--allow-copy] -``` - -| Flag | Description | -|------|-------------| -| `--allow-copy` | Allow CTAS copy fallback when native rename is not available | - -#### locks - -List active locks in the state store. - -```bash -sqb state locks -``` - -#### locks clear - -Clear a stuck lock. - -```bash -sqb state locks clear --force -``` - -Only clear locks when you are certain the holding operation is no longer running. - -#### checkpoints list - -List finalized checkpoints for a VDE. - -```bash -sqb state checkpoints list [--virtual-env ] -``` - -#### checkpoints show - -Show the model refs stored in a checkpoint. - -```bash -sqb state checkpoints show [--virtual-env ] -``` - -#### checkpoints diff - -Diff current VDE refs against a checkpoint. - -```bash -sqb state checkpoints diff [--virtual-env ] -``` - -## promote - -Source: `cli/promote.mdx` - -Promote VDE refs from one virtual environment to another. - -## sqb promote - -Promotes model version refs from a source VDE to a target VDE. No models are rebuilt - promotion swaps pointers and refreshes logical views. Virtual mode only. - -### Usage - -```bash -sqb promote --from --to [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--from` | Source VDE name (required) | -| `--to` | Target VDE name (required) | -| `--select`, `-s` | Select specific models to promote | -| `--exclude` | Exclude specific models | -| `--allow-partial-promotion` | Accept a working target VDE when partial promotion leaves downstream stale | -| `--include-stale-upstreams` | Add required stale upstream refs to the promotion scope | -| `--verbose`, `-v` | Show uncapped model sets | -| `--var` | Set project variables | - -### Examples - -```bash -# Promote all models from pr_123 to dev -sqb promote --from pr_123 --to dev - -# Promote specific models -sqb promote --from pr_123 --to dev --select fact_orders - -# Partial promotion with working target accepted -sqb promote --from pr_123 --to dev --select fact_orders --allow-partial-promotion - -# Include stale upstreams for coherent partial promotion -sqb promote --from pr_123 --to dev --select fact_orders --include-stale-upstreams -``` - -See [Promotion](/concepts/virtual-environments/promotion) for details on promotion behavior, source requirements, and guards. - -## rollback - -Source: `cli/rollback.mdx` - -Roll back a VDE to a prior finalized checkpoint. - -## sqb rollback - -Restores a virtual environment to a prior finalized checkpoint by rebinding its refs and refreshing logical views. Virtual mode only. - -### Usage - -```bash -sqb rollback [flags] -``` - -### Flags - -| Flag | Description | -|------|-------------| -| `--virtual-env` | Target VDE name (defaults to active target name) | -| `--checkpoint-id` | Restore a specific checkpoint instead of the previous one | -| `--select`, `-s` | Roll back specific models only | -| `--exclude` | Exclude specific models | -| `--allow-partial-rollback` | Accept a working VDE when partial rollback leaves downstream stale | -| `--include-stale-upstreams` | Add required upstream refs from the checkpoint | -| `--verbose`, `-v` | Show uncapped model sets | -| `--var` | Set project variables | - -### Examples - -```bash -# Roll back to the previous finalized checkpoint -sqb rollback - -# Roll back a specific VDE -sqb rollback --virtual-env pr_123 - -# Roll back to a specific checkpoint -sqb rollback --checkpoint-id - -# Partial rollback -sqb rollback --select fact_orders --allow-partial-rollback -``` - -See [Rollback](/concepts/virtual-environments/rollback) for details on checkpoints, rollback behavior, and guards. - -## reconcile - -Source: `cli/reconcile.mdx` - -Diagnose and repair drift between virtual state and warehouse. - -## sqb reconcile - -Detects and repairs inconsistencies between the virtual state store and the warehouse. Virtual mode only. - -### Usage - -```bash -sqb reconcile [--virtual-env ] [--model ] -sqb reconcile repair-view --virtual-env --model -sqb reconcile attach --virtual-env --model --physical-relation -``` - -### Subcommands - -#### (default) - -Run a diagnostic report without changing anything: - -```bash -sqb reconcile --virtual-env dev -``` - -#### repair-view - -Recreate a logical VDE view from trusted state refs: - -```bash -sqb reconcile repair-view --virtual-env dev --model fact_orders -``` - -No confirmation needed. Idempotent (`CREATE OR REPLACE VIEW`). - -#### attach - -Rebind a VDE model ref to a different tracked physical relation: - -```bash -sqb reconcile attach --virtual-env dev --model fact_orders \ - --physical-relation dev__sqb_physical.fact_orders__v_8f3a9c12 -``` - -Requires confirmation by default. - -### Examples - -```bash -# Diagnose issues -sqb reconcile --virtual-env dev - -# Fix a missing view -sqb reconcile repair-view --virtual-env dev --model fact_orders - -# Rebind a model to a different physical version -sqb reconcile attach --virtual-env dev --model fact_orders \ - --physical-relation dev__sqb_physical.fact_orders__v_71d0e4ab -``` - -See [Reconcile](/concepts/virtual-environments/reconcile) for details on guards and when to use each subcommand. - -## Enums and Constants Have Moved - -Source: `concepts/enums-and-constants.mdx` - -Find the new focused guides for enums, constants, collections, contracts, and private values. - -Enums and constants now have separate guides so each feature is easier to find and learn. This page -remains available for existing links and bookmarks but is not part of the main navigation. - - Define fixed string or integer domains and reference validated members. - Define reusable scalar values and reference them safely from SQL. - Work with lists, sets, objects, value lists, and native arrays. - Keep enums and constants inside one model. - -### Public declarations - -See [Enums](/concepts/enums) and [Constants](/concepts/constants) for project-wide declarations. - -### Constant values - -See [Constants](/concepts/constants#scalar-values) for scalars and -[Collections and Rendering](/concepts/constants/collections-and-rendering) for collections. - -### References and rendering - -See [Collections and Rendering](/concepts/constants/collections-and-rendering). - -### Adapter matrix - -See the [native-array rendering matrix](/concepts/constants/collections-and-rendering#native-array-rendering). - -### Rendering configuration - -See [Project default](/concepts/constants/collections-and-rendering#project-default). - -### Model-local declarations - -See [Model-Private Values](/concepts/model-private-values). - -### Model-private declarations - -See [Model-Private Values](/concepts/model-private-values). - -### Enum validation - -See [Enum validation rules](/concepts/enums#validation-rules). - -### Enum-typed contracts +## Selectors -See [Enum Model Contracts](/concepts/enums/model-contracts). +`--select a b` unions, `--exclude` subtracts. `+model` includes upstream, `model+` downstream. +`tag:`, `path:models/marts` (or any value containing `/`), `seed:`, `source:`, +and quoted name globs such as `"stg_*"`. They work with plan, build, test, audit, seed, clone and +diff. + +## Exact syntax and everything else + +- Task guides, one level deep: + - [references/verifying-changes.md](references/verifying-changes.md): data diff and query diff. + - [references/exploring-a-project.md](references/exploring-a-project.md): scope, lineage, dag, + query, plan and compile output. + - [references/incremental-models.md](references/incremental-models.md): strategies, cursors, + microbatch, watermarks, limits, replay. + - [references/testing.md](references/testing.md): unit tests, fixtures, scenarios, audits, + Rules. + - [references/macros-and-declarations.md](references/macros-and-declarations.md): macros, + enums, constants, placement and scopes. +- Full documentation for this SQLBuild version: [references/docs/INDEX.md](references/docs/INDEX.md) + lists every page; open only the page you need, for example `references/docs/cli/build.md` or + `references/docs/concepts/snapshots.md`. The same pages are online at + https://docs.sqlbuild.com if you need a newer version. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md new file mode 100644 index 0000000000..6cdf75a7e2 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md @@ -0,0 +1,150 @@ + + +# SQLBuild documentation index + +Bundled copies of every page on the SQLBuild documentation site, matching the installed SQLBuild version. Open only the page you need. + +## Getting Started + +- [Introduction](index.md) (`index`) - Verify early, test properly, and deploy reversibly. SQL pipelines with the rigor of real software - free and open source. +- [Quickstart](quickstart.md) (`quickstart`) - Get a SQLBuild project running locally with DuckDB in under a minute. +- [Feature Comparison](feature-comparison.md) (`feature-comparison`) - Feature comparison between SQLBuild, dbt, and SQLMesh. +- [Benchmarks](benchmarks.md) (`benchmarks`) - Complete compiler performance on projects with thousands of models, SQL tests, and audit attachments. + +## dbt Compatibility + +- [Using SQLBuild with dbt](concepts/dbt-compatibility/overview.md) (`concepts/dbt-compatibility/overview`) - Run dbt and SQLBuild side by side with coordinated selection and SQLBuild models downstream. +- [Selection](concepts/dbt-compatibility/selection.md) (`concepts/dbt-compatibility/selection`) - How select and exclude flags route work across the dbt and SQLBuild graphs. +- [Adding SQLBuild models](concepts/dbt-compatibility/adding-sqlbuild-models.md) (`concepts/dbt-compatibility/adding-sqlbuild-models`) - Grow into SQLBuild's own models, tests, audits, and scenarios downstream of your dbt project. + +## Concepts + +- [Project Configuration](concepts/project-configuration.md) (`concepts/project-configuration`) - Configure your SQLBuild project with sqlbuild_project.toml and sqlbuild_local.toml. +- [Resource Identities](concepts/resource-identities.md) (`concepts/resource-identities`) - Canonical names for SQLBuild resources, selectors, state, and integrations. +- [Overview](concepts/adapters.md) (`concepts/adapters`) - Supported database engines and their connection configuration. +- [DuckDB](concepts/adapters/duckdb.md) (`concepts/adapters/duckdb`) - DuckDB adapter configuration for SQLBuild. +- [MotherDuck](concepts/adapters/motherduck.md) (`concepts/adapters/motherduck`) - MotherDuck adapter configuration for SQLBuild. +- [Snowflake](concepts/adapters/snowflake.md) (`concepts/adapters/snowflake`) - Snowflake adapter configuration for SQLBuild. +- [BigQuery](concepts/adapters/bigquery.md) (`concepts/adapters/bigquery`) - BigQuery adapter configuration for SQLBuild. +- [Databricks](concepts/adapters/databricks.md) (`concepts/adapters/databricks`) - Databricks adapter configuration for SQLBuild. +- [PostgreSQL](concepts/adapters/postgres.md) (`concepts/adapters/postgres`) - PostgreSQL adapter configuration for SQLBuild. +- [SQL Server](concepts/adapters/sqlserver.md) (`concepts/adapters/sqlserver`) - Microsoft SQL Server adapter configuration for SQLBuild. +- [Sources](concepts/sources.md) (`concepts/sources`) - Declare external data inputs for your pipeline. +- [Seeds](concepts/seeds.md) (`concepts/seeds`) - Load static CSV data into your pipeline as tables. +- [Overview](concepts/models.md) (`concepts/models`) - SQL model anatomy, references, dependencies, and the model documentation guide. +- [Materializations](concepts/models/materializations.md) (`concepts/models/materializations`) - Choose how SQLBuild persists model output. +- [Schemas](concepts/models/schemas.md) (`concepts/models/schemas`) - Declare model columns inline or reuse canonical inherited schemas. +- [Type Enforcement](concepts/models/type-enforcement.md) (`concepts/models/type-enforcement`) - Understand declared model types, static checks, and runtime casting. +- [Contracts](concepts/models/contracts.md) (`concepts/models/contracts`) - Validate required or exact model output schemas. +- [Hooks](concepts/models/hooks.md) (`concepts/models/hooks`) - Run SQL or Python lifecycle hooks around model materialization. +- [SQL Hooks](concepts/models/hooks/sql.md) (`concepts/models/hooks/sql`) - Define, parameterize, compile, and invoke reusable or inline SQL lifecycle hooks. +- [Python Hooks](concepts/models/hooks/python.md) (`concepts/models/hooks/python`) - Define Python lifecycle hooks with runtime context, providers, SQL access, and skips. +- [Configuration](concepts/models/configuration.md) (`concepts/models/configuration`) - MODEL() header fields and SQL-analysis controls. +- [Enums](concepts/enums.md) (`concepts/enums`) - Define a fixed set of named string or integer values and use them safely in SQL. +- [Constants](concepts/constants.md) (`concepts/constants`) - Define reusable compiler-validated values and reference them safely from SQL. +- [Collections and Rendering](concepts/constants/collections-and-rendering.md) (`concepts/constants/collections-and-rendering`) - Define list, set, and object constants and control their adapter-specific SQL rendering. +- [Enum Model Contracts](concepts/enums/model-contracts.md) (`concepts/enums/model-contracts`) - Use an enum as a portable model-column domain with generated accepted-value validation. +- [Model-Private Values](concepts/model-private-values.md) (`concepts/model-private-values`) - Keep enums and constants inside one model when no other resource should use them. +- [Writing Macros](concepts/macros.md) (`concepts/macros`) - Write Python functions that generate reusable SQL fragments at compile time. +- [Composition and Context](concepts/macros/composition-and-context.md) (`concepts/macros/composition-and-context`) - Compose macros through Python, use compile context, and understand scoped imports. +- [Interpolation](concepts/interpolation.md) (`concepts/interpolation`) - How SQLBuild processes variables, context, and dynamic content in SQL and config. +- [Functions](concepts/functions.md) (`concepts/functions`) - User-defined functions and table functions managed as part of your project. +- [Incremental Models](concepts/incremental.md) (`concepts/incremental`) - Cursor-based incremental strategies, microbatch execution, and backfill policies. +- [Planning and change detection](concepts/planning.md) (`concepts/planning`) - How SQLBuild explains build work, detects changes, and chooses safe model actions. +- [Cascade propagation](concepts/planning/cascade-propagation.md) (`concepts/planning/cascade-propagation`) - How a change signal propagates downstream through the DAG, and how each materialization type responds. +- [Source freshness](concepts/planning/source-freshness.md) (`concepts/planning/source-freshness`) - Observe external source changes and propagate them through planning. +- [Selection and staleness](concepts/planning/selection-and-staleness.md) (`concepts/planning/selection-and-staleness`) - How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. +- [Snapshots (SCD Type 2)](concepts/snapshots.md) (`concepts/snapshots`) - Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. +- [Audits](concepts/audits.md) (`concepts/audits`) - Violation and measurement checks that gate data and record quality outcomes. +- [Overview](concepts/rules.md) (`concepts/rules`) - Turn repeated SQL and project review decisions into compiler-enforced requirements. +- [Configuration and selection](concepts/rules/configuration-and-selection.md) (`concepts/rules/configuration-and-selection`) - Select built-in and custom Rules by exact code or family prefix. +- [Findings and exceptions](concepts/rules/findings-and-exceptions.md) (`concepts/rules/findings-and-exceptions`) - Understand Rule diagnostics and record intentional exceptions safely. +- [Execution and caching](concepts/rules/execution-and-caching.md) (`concepts/rules/execution-and-caching`) - Understand Rule enforcement order, focused runs, and dependency-aware reuse. +- [Testing](concepts/testing.md) (`concepts/testing`) - SQL unit tests and multi-model tests with macro support, assertions, and model chaining. +- [Scenarios](concepts/scenarios.md) (`concepts/scenarios`) - End-to-end tests that build real project graphs against coherent fixture data. +- [Selectors](concepts/selectors.md) (`concepts/selectors`) - Target specific models, paths, tags, or DAG subsets with select and exclude flags. +- [Column Lineage](concepts/column-lineage.md) (`concepts/column-lineage`) - Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. +- [Data Diffs](concepts/diff.md) (`concepts/diff`) - Compare schemas and data between targets or virtual environments to validate changes before promotion. + +## Advanced Concepts + +- [Execution Observability](concepts/observability.md) (`concepts/observability`) - Choose authoritative lifecycle facts, readable logs, or command-output records. +- [Typed Sinks](concepts/observability/sinks.md) (`concepts/observability/sinks`) - Export lifecycle facts and command output through project-owned providers. +- [Overview](concepts/declaration-scopes.md) (`concepts/declaration-scopes`) - Limit enums, constants, and macros to the parts of a project that use them. +- [How Visibility Works](concepts/declaration-scopes/visibility.md) (`concepts/declaration-scopes/visibility`) - See which enums, constants, and macros are available to each SQL file. +- [Where to Put Declarations](concepts/declaration-scopes/placement.md) (`concepts/declaration-scopes/placement`) - Choose the narrowest folder that contains every real use. +- [Scope Explorer](concepts/declaration-scopes/explorer.md) (`concepts/declaration-scopes/explorer`) - Inspect visibility, explain resolution, browse declarations, and preview moves offline. +- [Overview](concepts/rules/custom-rules/overview.md) (`concepts/rules/custom-rules/overview`) - Define repository-owned checks with one typed Python API. +- [RuleContext and compiler facts](concepts/rules/custom-rules/rule-context.md) (`concepts/rules/custom-rules/rule-context`) - Inspect stable compiler-owned facts and opt into lazy SQL AST access. +- [Testing and deterministic Rules](concepts/rules/custom-rules/testing-and-determinism.md) (`concepts/rules/custom-rules/testing-and-determinism`) - Test custom findings and keep implementations safe for dependency-aware caching. + +## Python Nodes + +- [Overview](concepts/python-nodes/overview.md) (`concepts/python-nodes/overview`) - Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph. +- [Loaders](concepts/python-nodes/loaders.md) (`concepts/python-nodes/loaders`) - Load external data into source tables with Python functions. +- [Tasks](concepts/python-nodes/tasks.md) (`concepts/python-nodes/tasks`) - Run Python computation and side effects as nodes in the SQLBuild graph. +- [Assets](concepts/python-nodes/assets.md) (`concepts/python-nodes/assets`) - Produce or observe external artifacts as nodes in the SQLBuild graph. +- [Checks](concepts/python-nodes/checks.md) (`concepts/python-nodes/checks`) - Validate tasks, assets, and loaders with Python checks. +- [Factories](concepts/python-nodes/factories.md) (`concepts/python-nodes/factories`) - Generate Python nodes programmatically with @factory. +- [Providers](concepts/python-nodes/providers.md) (`concepts/python-nodes/providers`) - Shared runtime services for Python nodes and hooks. +- [SQL References](concepts/python-nodes/sql-references.md) (`concepts/python-nodes/sql-references`) - Read SQL models and sources from Python nodes without creating SQL dependencies. + +## Virtual Environments (Alpha) + +- [Overview](concepts/virtual-environments.md) (`concepts/virtual-environments`) - Version-controlled SQL pipeline environments with instant promotion and rollback. +- [Setup](concepts/virtual-environments/setup.md) (`concepts/virtual-environments/setup`) - Configure virtual environments and initialize the state store. +- [Building](concepts/virtual-environments/building.md) (`concepts/virtual-environments/building`) - Virtual builds, VDE creation, partial builds, and seeded incrementals. +- [Promotion](concepts/virtual-environments/promotion.md) (`concepts/virtual-environments/promotion`) - Promote VDE refs and diff virtual environments. +- [Rollback](concepts/virtual-environments/rollback.md) (`concepts/virtual-environments/rollback`) - Checkpoints and rollback for virtual environments. +- [Adopt and Detach](concepts/virtual-environments/adopt-detach.md) (`concepts/virtual-environments/adopt-detach`) - Migrate existing projects to and from virtual mode. +- [Clone](concepts/virtual-environments/clone.md) (`concepts/virtual-environments/clone`) - Hydrate physical versions from a source warehouse. +- [Diff](concepts/virtual-environments/diff.md) (`concepts/virtual-environments/diff`) - Compare virtual data environments. +- [Reconcile](concepts/virtual-environments/reconcile.md) (`concepts/virtual-environments/reconcile`) - Diagnose and repair drift between state and warehouse. +- [Locks](concepts/virtual-environments/locks.md) (`concepts/virtual-environments/locks`) - Advisory locks for concurrent access control. +- [Janitor](concepts/virtual-environments/janitor.md) (`concepts/virtual-environments/janitor`) - Cleanup of virtual environment artifacts and retention policies. +- [Recovery](concepts/virtual-environments/recovery.md) (`concepts/virtual-environments/recovery`) - Diagnosing and recovering from failures in virtual mode. + +## Integrations + +- [Overview](integrations/dagster.md) (`integrations/dagster`) - Orchestrate SQLBuild pipelines with Dagster scheduling, retries, and asset UI. +- [API Reference](integrations/dagster-reference.md) (`integrations/dagster-reference`) - Dagster integration classes, decorators, and translator hooks. +- [Rivers](integrations/rivers.md) (`integrations/rivers`) - Orchestrate SQLBuild pipelines with Rivers scheduling, jobs, and asset tracking. +- [dlt](integrations/dlt.md) (`integrations/dlt`) - Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders. +- [ingestr](integrations/ingestr.md) (`integrations/ingestr`) - Declarative data ingestion from 50+ sources using ingestr. + +## CLI Reference + +- [init](cli/init.md) (`cli/init`) - Scaffold a new SQLBuild project. +- [playground](cli/playground.md) (`cli/playground`) - Create a self-contained SQLBuild project to explore locally. +- [skills](cli/skills.md) (`cli/skills`) - Install SQLBuild skill files for AI coding agents. +- [compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. +- [format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. +- [contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. +- [scope](cli/scope.md) (`cli/scope`) - Inspect declaration visibility, usage, placement, and move impact offline. +- [rules](cli/rules.md) (`cli/rules`) - List, inspect, or run compiler-integrated Rules and generate project guidance. +- [plan](cli/plan.md) (`cli/plan`) - Preview what SQLBuild will do before executing. +- [build](cli/build.md) (`cli/build`) - Compile, plan, and execute the selected build lifecycle. +- [load](cli/load.md) (`cli/load`) - Load managed sources into the warehouse. +- [seed](cli/seed.md) (`cli/seed`) - Load seed CSV files into the warehouse. +- [test](cli/test.md) (`cli/test`) - Run SQL unit tests and multi-model tests in isolation. +- [scenario](cli/scenario.md) (`cli/scenario`) - Run end-to-end scenario tests against the warehouse or locally with DuckDB. +- [audit](cli/audit.md) (`cli/audit`) - Run data quality audits in isolation. +- [freshness](cli/freshness.md) (`cli/freshness`) - Observe source freshness without writing state. +- [check](cli/check.md) (`cli/check`) - Run Python checks against tasks, assets, and loaders. +- [clone](cli/clone.md) (`cli/clone`) - Copy model relations between configured targets. +- [diff](cli/diff.md) (`cli/diff`) - Compare schemas and data between targets or virtual environments. +- [lineage](cli/lineage.md) (`cli/lineage`) - Explore model and column-level dependency graphs from the command line. +- [dag](cli/dag.md) (`cli/dag`) - Generate the static DAG artifact for Dagster and other integrations. +- [query](cli/query.md) (`cli/query`) - Run ad hoc SQL queries against the project database. +- [debug](cli/debug.md) (`cli/debug`) - Validate project configuration and test the warehouse connection. +- [janitor](cli/janitor.md) (`cli/janitor`) - Clean up stale warehouse relations. +- [clean](cli/clean.md) (`cli/clean`) - Remove compiled artifacts from the target directory. +- [dbt](cli/dbt.md) (`cli/dbt`) - Coordinate dbt and SQLBuild projects. +- [state](cli/state.md) (`cli/state`) - Manage the virtual mode state store. +- [promote](cli/promote.md) (`cli/promote`) - Promote VDE refs from one virtual environment to another. +- [rollback](cli/rollback.md) (`cli/rollback`) - Roll back a VDE to a prior finalized checkpoint. +- [reconcile](cli/reconcile.md) (`cli/reconcile`) - Diagnose and repair drift between virtual state and warehouse. + +## Other pages + +- [Enums and Constants Have Moved](concepts/enums-and-constants.md) (`concepts/enums-and-constants`) - Find the new focused guides for enums, constants, collections, contracts, and private values. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md new file mode 100644 index 0000000000..44fa20d9ee --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md @@ -0,0 +1,125 @@ + + +# Benchmarks + +> Complete compiler performance on projects with thousands of models, SQL tests, and audit attachments. + +Online: https://docs.sqlbuild.com/benchmarks + +## Contents + +- Compiler performance +- Rules performance +- Methodology +- Reproduce the benchmarks + +SQLBuild compiles a 5,000-model project with nearly 5,000 SQL tests and more than 8,000 audit +attachments from scratch in **19 seconds** in our CI benchmark. An unchanged recompile takes +**13 seconds**. + +These measurements run the complete `sqb compile` command, including project discovery, dependency +resolution, SQL expansion and analysis, contract validation, test and audit preparation, configured +Rules, cache publication, and artifact generation. + +## Compiler performance + +| Compile time | **3k models**
2.9k tests · 5.1k audit attachments | **5k models**
4.9k tests · 8.4k audit attachments | +|---|---:|---:| +| First compile / cache fill | **12.48s** | **19.01s** | +| Unchanged recompile | **8.79s** | **12.76s** | + +Timings include test and audit preparation, not warehouse execution. + +The 5,000-model compiler process used approximately **1.3 GiB peak resident memory** on its first +compile and **0.7 GiB** on its unchanged recompile. + +### Workload details + +The deterministic generated projects exercise compiler work at scale, with synthetic SQL, +contracts, macros, functions, tests, and audits. + +| Resource | 3k-model project | 5k-model project | +|---|---:|---:| +| Models | 3,000 | 5,000 | +| Sources | 713 | 1,189 | +| Seeds | 141 | 236 | +| SQL functions | 71 | 118 | +| Python macros | 37 | 61 | +| SQL test cases | 2,945 | 4,908 | +| Audit attachments | 5,056 | 8,427 | + +The projects use repeated generated structures. Individual project timings depend on SQL +complexity, macros, configuration, and hardware as well as model count. + +### Incremental compilation + +SQLBuild reuses cached analysis for unaffected models. In the 5,000-model benchmark, a single-model +file edit reused **4,999 model analyses**. A scoped Python-macro edit required fresh analysis for +**7 models**. Both scenarios completed in approximately **9.8 seconds**. + +The single-model scenario appends a SQL comment to exercise file-change invalidation. Separate +regression coverage checks expression-changing edits against uncached compilation results. +Scenarios run in a fixed sequence and use different cache-loading paths; an edit timing below the +initial unchanged timing does not mean editing inherently requires less work. + +## Rules performance + +Rules participate in ordinary compilation. Model Rules are cached per model, and SQLBuild tracks +their dependencies so unaffected findings can be reused after edits. Project Rules evaluate the +complete project view and have broader invalidation. + +A separate custom-Rules benchmark exercises 20- and 100-Rule packs over 5,000 models. It measures +first compilation, unchanged compilation, model edits, and Rule implementation changes. This is a +different workload from the compiler table above; its results should be compared within that profile. + +See [Execution and caching](concepts/rules/execution-and-caching.md) for the Rules lifecycle, or run +the publication profile below to collect current custom-Rules measurements. + +## Methodology + +The compiler results above are individual scenario measurements from the +[passing CI run on September 22, 2026](https://github.com/chio-labs/sqlbuild/actions/runs/35764207007), +using the compiler code released in **SQLBuild 0.113.3** and Python 3.12 on Linux. They are not +multi-run medians. + +Each scenario launches a fresh process. The first compile starts with an empty project cache; +subsequent scenarios reuse that cache. The CI job enforces a **4 GiB combined memory limit**. +Reported peak resident memory is for the compiler process, not the entire CI job. + +CI checks cache-hit and miss expectations and pinned semantic fingerprints alongside timing and +memory budgets. The varied 1,000-model guard additionally compares SQL-expression and macro edits +with uncached analysis and Rules results. + +These measurements cover compilation and static analysis. SQL tests and audits are prepared, not +executed against a warehouse. + +## Reproduce the benchmarks + +Run the fresh-process compiler cache guards from the +[SQLBuild repository](https://github.com/chio-labs/sqlbuild): + +```bash +make test-e2e-cache-compile-performance SQLBUILD_BENCHMARK_MODELS=3000 +make test-e2e-cache-compile-performance SQLBUILD_BENCHMARK_MODELS=5000 +``` + +Run the five-iteration custom-Rules publication profile: + +```bash +uv run python -m scripts.benchmark_rules \ + --models 5000 \ + --rule-counts 20 100 \ + --iterations 5 \ + --output rules-5000.json +``` + +Run the bounded Rules profile used by required CI: + +```bash +uv run python -m scripts.benchmark_rules_ci \ + --output rules-performance-ci.json \ + --summary-output rules-performance-ci.md +``` + +The benchmark generators, guard thresholds, and CI workflow are maintained in the public +[SQLBuild repository](https://github.com/chio-labs/sqlbuild). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/audit.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/audit.md new file mode 100644 index 0000000000..dcc7418bfa --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/audit.md @@ -0,0 +1,70 @@ + + +# audit + +> Run data quality audits in isolation. + +Online: https://docs.sqlbuild.com/cli/audit + +# sqb audit + +Runs all attached audits without rebuilding models. Useful for verifying data quality on existing warehouse state. + +## Usage + +```bash +sqb --project-dir audit [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--defer-to` | Resolve model references against another target | +| `--select`, `-s` | Select audits attached to specific models | +| `--select-file` | Read selectors from a file | +| `--exclude` | Exclude audits attached to specific models | +| `--concurrency` | Maximum number of audits to run concurrently (default: `1`) | +| `--json` | Print structured audit results | +| `--json-output` | Write structured audit results to a file | + +## Concurrency + +Standalone audits run serially by default. Use `--concurrency` to opt into parallel warehouse +queries: + +```bash +sqb audit --concurrency 8 +``` + +SQLBuild resolves the limit in this order: an explicit `--concurrency` value, the +`SQLBUILD_CONCURRENCY` environment variable, the effective project `settings.concurrency`, then +the product default of `1`. Values must be at least `1`. + +The physical worker count is bounded by the number of selected audits. Each active worker opens +and exclusively uses one warehouse connection, so higher concurrency can increase warehouse load +and cost. Results and final JSON remain in plan order even when queries finish in a different +order. + +If an audit query raises a runtime, adapter, or framework error, SQLBuild records that failure and +continues running the remaining independent audits. After all selected audits have been attempted, +it reports every query failure in plan order and exits nonzero. Failed queries do not produce +fabricated audit results. + +Interruption and cancellation are different: SQLBuild stops scheduling new audits, drains queries +that have already started, and then closes every worker connection. Generic adapters cannot +guarantee immediate cancellation of a query already running in the warehouse. + +## Examples + +```bash +# Run all audits +sqb audit + +# Run audits for marts only +sqb audit --select path:models/marts + +# Run up to 8 audit queries concurrently +sqb audit --concurrency 8 +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md new file mode 100644 index 0000000000..68a929e6e7 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -0,0 +1,132 @@ + + +# build + +> Compile, plan, and execute the selected build lifecycle. + +Online: https://docs.sqlbuild.com/cli/build + +## Contents + +- Usage +- Flags +- Fast iteration +- Execution order +- Output +- Deferred builds +- Failure behavior +- Fingerprints +- Runtime artifacts + +# sqb build + +Compiles, plans, and executes the selected build lifecycle. Direct mode runs the full selected +scope. [Virtual environments](../concepts/virtual-environments.md) can additionally use +`--changes-only` to build work not already represented by the target VDE's bound versions. Use +`--no-tests` and `--no-audits` to skip validation for fast iteration. + +## Usage + +```bash +sqb --project-dir build [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--target` | Build against a configured target instead of the active/default target | +| `--changes-only` | Virtual mode only: build work not already bound to its expected VDE version | +| `--no-tests` | Skip SQL unit tests | +| `--no-audits` | Skip audits | +| `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--defer-to` | Resolve unselected model references against another target | +| `--defer-sources-to` | Read managed source data from another target | +| `--fail-fast` | Stop on first failure and skip remaining nodes | +| `--concurrency` | Number of worker connections (default: 1) | +| `--verbose`, `-v` | Show lifecycle SQL inline after each model | +| `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | +| `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | +| `--start-cursor-int` | Override start cursor for integer incremental models | +| `--end-cursor-int` | Override end cursor for integer incremental models | +| `--load` | Explicitly load managed sources before building | +| `--no-load` | Skip automatic source loading | +| `--reload` | Reload managed sources (passes `is_reload=True` to loaders) | +| `--include-stale-upstreams` | Expand selection to include stale upstream models needed for coherence | +| `--manifest` | Generate `target/manifest.json` with plan-aware project metadata | +| `--select`, `-s` | Select specific models | +| `--exclude` | Exclude specific models | + +## Fast iteration + +Use `--no-tests` and `--no-audits` to skip validation when you only want to materialize models: + +```bash +sqb build --no-tests --no-audits +``` + +This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. + +## Execution order + +1. Managed sources are loaded (unless `--no-load`) +2. Seeds are loaded (if changed) +3. Source audits run before their dependent models (unless `--no-audits`) +4. SQL unit tests run before their target model (unless `--no-tests`) +5. Models are materialized in DAG topological order (virtual `--changes-only` limits this to stale work) +6. Error-severity audits run against the staging table before promotion to the target (unless `--no-audits`) + +## Output + +``` +Execution sqb build (concurrency: 1) + + 1/13 seed waffle_types OK 0.09s + 2/13 view stg_customers OK 0.05s + audit not_null (customer_id) PASS + audit unique (customer_id) PASS + 3/13 view stg_orders OK 0.03s + test test_stg_orders PASS + audit not_null (order_id) PASS + audit unique (order_id) PASS + 10/13 table hourly_order_activity (delete_insert) OK 0.16s + audit (d) expression_is_true PASS 4/4 + audit (d) not_null (activity_hour) PASS 4/4 + audit (f) expression_is_true PASS + audit (f) not_null (activity_hour) PASS + +Completed successfully. +PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) +``` + +## Deferred builds + +Use `--defer-to` to resolve unselected model references against another target. This lets you build a subset of models in dev while referencing production tables for everything else: + +```bash +sqb build --select fact_orders --defer-to prod +``` + +No `manifest.json` is required. `--defer-to` selects only the namespace used for unselected +references. SQLBuild resolves those relations through the active target's sole physical +connection and never opens the deferred target's connection. The deferred namespace must be +visible and readable through the active connection; deferral is not a cross-account, +cross-server, or cross-file transfer mechanism. + +## Failure behavior + +When a model fails: +- Downstream models are automatically blocked and skipped +- Staging/delta tables are retained for inspection +- Failure details show the model name, failed phase, and error message + +## Fingerprints + +After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. See [Planning and Change Detection](../concepts/planning.md) for details. + +## Runtime artifacts + +Build writes executed lifecycle SQL to `target/run/models/`. These files contain the actual SQL that was executed, including resolved cursor bounds and runtime substitutions. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md new file mode 100644 index 0000000000..e80ed16ea7 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md @@ -0,0 +1,56 @@ + + +# check + +> Run Python checks against tasks, assets, and loaders. + +Online: https://docs.sqlbuild.com/cli/check + +# sqb check + +Runs Python [checks](../concepts/python-nodes/checks.md) in isolation. Checks validate the output of tasks, assets, and loaders. For SQL relation validation, use [`sqb audit`](audit.md) instead. + +## Usage + +```bash +sqb --project-dir check [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--select`, `-s` | Select checks to run (by name, `check:`, `tag:`, or graph expansion) | +| `--exclude` | Exclude checks from the selection | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--json` | Print check results as JSON | +| `--json-output` | Write check results JSON to a file path | +| `--vars` | Override project variables | + +Selecting a non-check node is rejected; use [`sqb build`](build.md) to run tasks and assets. + +## Examples + +```bash +# Run all checks +sqb check + +# Run one check and its required dependencies +sqb check --select +check_orders_exported + +# Run checks by tag +sqb check --select tag:exports + +# Print results as JSON +sqb check --json +``` + +## Output + +Checks report PASS, FAIL, or WARN per check. A failing `error`-severity check exits non-zero; `warn`-severity failures are reported without failing the command. + +Results are written to `target/run/checks/python_checks.json`. + +## Relationship to build + +`sqb build` runs relevant Python checks by default after the tasks, assets, and loaders they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clean.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clean.md new file mode 100644 index 0000000000..de8fdc9c2b --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clean.md @@ -0,0 +1,19 @@ + + +# clean + +> Remove compiled artifacts from the target directory. + +Online: https://docs.sqlbuild.com/cli/clean + +# sqb clean + +Removes the `target/` directory containing compiled artifacts, runtime SQL recordings, and other build outputs. + +## Usage + +```bash +sqb --project-dir clean +``` + +No flags. This command has no confirmation prompt since it only removes local build artifacts, not warehouse data. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md new file mode 100644 index 0000000000..bb830b801d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md @@ -0,0 +1,91 @@ + + +# clone + +> Copy model relations between configured targets. + +Online: https://docs.sqlbuild.com/cli/clone + +# sqb clone + +Copies selected relations from one target to another. It uses adapter-native cloning where supported and physical copies where required; `--hard-copy` forces a physical copy on adapters that support both. + +No `manifest.json` generation or artifact management is required. Clone works directly against live targets. + +When `--to` is omitted, the destination is the active target selected by `--target`, +`sqlbuild_local.toml`, or `default_target`, in that order. + +`--from` selects only the origin namespace and its clone-origin policy. `--to`, or the active +target when it is omitted, selects the sole physical connection used by the operation. The +origin namespace must therefore be visible and readable through the destination connection. +SQLBuild never resolves or opens the origin target's connection, including for direct, +deferred, and virtual clone. Clone cannot perform a cross-account, cross-server, or cross-file +transfer by opening a second connection. + +## Usage + +```bash +sqb --project-dir clone --from [--to ] [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--from` | Source target (required) | +| `--to` | Destination target; defaults to the active target | +| `--hard-copy` | Force physical table copies instead of zero-copy cloning | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--select`, `-s` | Select specific models to clone | +| `--exclude` | Exclude specific models from cloning | + +## Examples + +```bash +# Clone all models from prod to dev +sqb clone --from prod --to dev + +# Equivalent when dev is the active target +sqb clone --from prod + +# Clone only marts to dev +sqb clone --from prod --to dev --select path:models/marts + +# Force physical copies +sqb clone --from prod --to dev --hard-copy +``` + +## Managed sources + +Clone includes selected managed physical sources when the destination target reads its own +loader namespace. Sources are copied before seeds and models so views can be recreated in a +destination that has never been built. + +Source locations respect target configuration: + +- The origin relation comes from the target that the origin uses for managed source reads. +- The destination relation uses the destination target's `loader_schema`, falling back to + its model `schema`. +- If the destination defers source reads to another target, clone reuses that target's source + relation and does not copy or overwrite it. +- Expression sources and unmanaged external sources are not cloned. + +`--hard-copy` controls how included relations are copied. It does not change which managed +sources are selected. + +## Clone policies + +Targets deny cloning by default. Enable the origin and destination explicitly in `sqlbuild_project.toml`: + +```toml +[targets.prod.clone] +allow_as_clone_origin = true + +[targets.dev.clone] +allow_as_clone_destination = true +``` + +The origin target must already contain the built relations being cloned. The destination +connection's credentials must be able to read the origin namespace and create relations in the +destination. Managed physical sources may bootstrap a destination that has not been built. See +[Project Configuration](../concepts/project-configuration.md) for details. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md new file mode 100644 index 0000000000..1653e621bb --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -0,0 +1,184 @@ + + +# compile + +> Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. + +Online: https://docs.sqlbuild.com/cli/compile + +## Contents + +- Usage +- Flags +- What compile does +- Focused compilation +- Static analysis +- Output +- Column lineage modes +- Examples + +# sqb compile + +Discovers the complete project, resolves references, expands macros, validates SQL, checks column contracts, computes column lineage, and writes compiled artifacts to `target/`. The compile command is fully offline—it does not connect to the warehouse. + +## Usage + +```bash +sqb --project-dir compile [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | +| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--defer-to` | Resolve unselected model references against another target | +| `--json` | Output the full compile report as JSON | +| `--manifest` | Generate `target/manifest.json` with project metadata | +| `--lineage-mode` | Column lineage mode: `fast` (default), `rich` (slower, more detail), or `none` | +| `--select`, `-s` | Analyze and report selected resources | +| `--select-file` | Read selectors from a file, one selector per line | +| `--exclude` | Remove resources from the selected scope | + +## What compile does + +1. **Discovery** - finds `sqlbuild_project.toml`, scans for models, sources, seeds, functions, audits, tests, and macros +2. **Graph resolution** - resolves `ref()` and `source()` calls, expands macros, orders models by dependency +3. **SQL validation** - validates SQL syntax (when SQL analysis is enabled) +4. **Column lineage** - analyzes column-level dependencies across models (fast mode by default) +5. **Contract validation** - checks declared column contracts against inferred query output +6. **Rules** - evaluates selected native built-ins, then selected custom Python Rules +7. **Artifact write** - writes compiled SQL to `target/compiled/` when Rules pass + +## Focused compilation + +Use normal selectors to limit expensive analysis and reporting while retaining complete project +discovery and reference integrity: + +```bash +sqb compile --select fact_orders daily_revenue +sqb compile --select-file selected-models.txt +sqb compile --exclude deprecated_model +``` + +SQLBuild deeply analyzes the selected resources and the upstream closure needed to understand them. +The text report contains only the selected scope. Project totals remain available in JSON, together +with selected model, seed, and function counts. + +Manifest and DAG outputs remain full-project artifacts. Focused compilation changes analysis and +reporting scope; it does not produce a partial project graph. + +## Static analysis + +When SQL analysis is enabled (default), compile performs static analysis on your models without connecting to the warehouse: + +- **Column inference**: Infers output columns from each model's SQL, including through CTEs, subqueries, and JOINs +- **Column contract validation**: Under the default `settings.column_contract_mode = "implicit"`, a model with declared columns and no model-level `contract` declaration checks that every declared column exists in the statically inferred query output. `explicit` mode requires `contract enforced` to activate shape checks. Explicit type enforcement remains independent and verifies inferred types when possible +- **Column lineage**: Traces which source columns flow into each output column, including transform classification. See [Column Lineage](../concepts/column-lineage.md) for details + +`sqb compile` is authoritative for mandatory correctness, contracts, lineage, and configured Rules. +Rules findings block artifact completion. Use [`sqb rules run`](rules.md) to focus on one exact +code or family without changing project configuration. + +### Contract diagnostics + +When a contract violation is found, compile reports it with source-annotated diagnostics: + +``` +error[K001]: declared column 'total_cents' was not found in statically inferred output for model 'fact_orders' + model: fact_orders + --> models/marts/fact_orders.sql:6:5 + 6 | total_cents (), + | ^^^^^^^^^^^ + = help: add total_cents to the SELECT list or correct/remove MODEL(columns (...)); MODEL(columns (...)) is validated using static SQL analysis because settings.column_contract_mode is "implicit" (the default). If this project intentionally uses columns only for metadata and audits, set [settings] column_contract_mode = "explicit"; models with contract enforced remain validated +``` + +The configuration guidance is an intentional project-policy choice, not a general error suppression. Fix the query or declaration when the model is intended to have a column contract. Diagnostics for `contract enforced` models do not recommend changing the project mode because explicit model contracts remain authoritative. + +Diagnostic codes: + +| Code | Meaning | +|------|---------| +| `K001` | A declared column is missing from the model's query output | +| `K002` | A column's inferred type does not match the declared type | +| `K003` | A column's type could not be proven (with `type_enforcement` enabled) | + +Compile returns exit code `1` when any error-severity diagnostic is found, making it suitable for CI checks. + +## Output + +### Text output (default) + +```bash +sqb compile +``` + +``` +Compile ready (12 models) + + stg_customers OK 3 columns + stg_orders OK 5 columns + stg_payments OK 4 columns + fact_orders OK 6 columns + dim_customers OK 4 columns + daily_revenue OK 3 columns + ... + + Compiled: 12 models, 1 seed, 5 functions, 0 errors, 0 warnings + Wrote: target/compiled/ +``` + +Each model shows its name, status (OK or FAIL), and column count. Models with contract errors are marked FAIL. + +### JSON output + +```bash +sqb compile --json +``` + +Returns a structured report including: + +- `summary` - model, seed, function, audit, test, error, and warning counts +- `resources` - per-model details including column count, dependencies, lineage summary, and compiled SQL +- `diagnostics` - all contract violations with source locations +- `compile_timings` - timing breakdown for discovery, graph, lineage, contracts, and write phases +- `lineage_mode` - which lineage mode was used +- `artifacts` - paths to written files + +## Column lineage modes + +The `--lineage-mode` flag controls how column lineage is computed during compile: + +| Mode | Description | +|------|-------------| +| `fast` | Default. Lightweight column extraction using SQL model metadata. | +| `rich` | Full SQL analysis with transform classification and deeper tracing. Slower on large projects. | +| `none` | Skip column lineage entirely. | + +The JSON compile report includes a lineage summary for each model, not the full column graph. Use `sqb lineage [.]` to inspect lineage as a tree, edge list, or JSON. See [Column Lineage](../concepts/column-lineage.md) for details on analysis modes and transform types. + +## Examples + +```bash +# Basic compile +sqb compile + +# Compile with full JSON report +sqb compile --json + +# Compile with rich column lineage +sqb compile --lineage-mode rich + +# Skip column lineage +sqb compile --lineage-mode none + +# Generate manifest +sqb compile --manifest + +# Compile only selected models +sqb compile --select fact_orders daily_revenue + +# Disable SQL analysis +sqb compile --no-sql-analysis +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/contract.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/contract.md new file mode 100644 index 0000000000..161074b015 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/contract.md @@ -0,0 +1,44 @@ + + +# contract + +> Compare or generate repository contracts from physical warehouse schemas. + +Online: https://docs.sqlbuild.com/cli/contract + +# sqb contract + +Existing-schema comparison is explicitly online and read-only with respect to the warehouse. + +## diff + +```bash +sqb contract diff --from [flags] +``` + +Reports declaration differences. Exit code `0` means agreement and `1` means differences exist. + +## generate + +```bash +sqb contract generate --from [flags] +``` + +Prints proposed declaration updates. Add `--write` to change repository files and `--overwrite` to +replace conflicts rather than preserving authored declarations. + +## Flags + +| Flag | Commands | Description | +|------|----------|-------------| +| `--from` | both | Target namespace to inspect (required) | +| `--json` | both | Print structured findings | +| `--select`, `-s` | both | Inspect selected models or sources | +| `--select-file` | both | Read selectors from a file | +| `--exclude` | both | Exclude resources from the selected scope | +| `--vars` | both | Override project variables | +| `--write` | generate | Apply generated repository changes | +| `--overwrite` | generate | Replace conflicting declarations; requires `--write` | + +See [Contracts](../concepts/models/contracts.md#adopting-an-existing-schema) for ownership and safety +guarantees. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md new file mode 100644 index 0000000000..51ded90d7c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -0,0 +1,172 @@ + + +# dag + +> Generate the static DAG artifact for Dagster and other integrations. + +Online: https://docs.sqlbuild.com/cli/dag + +## Contents + +- Usage +- Flags +- Generating via compile +- Output format +- Examples + +# sqb dag + +Compiles the project and outputs the static DAG artifact. The artifact contains every node (source, seed, model, function), dependency edge, and check (test, audit, scenario) in your project as structured JSON. It is the bridge between SQLBuild and external orchestrators like Dagster. + +This artifact describes resource dependencies, not column-level lineage. Use [`sqb lineage`](lineage.md) to inspect or export column traces. + +## Usage + +```bash +sqb --project-dir dag [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--json` | Print the full DAG artifact as JSON to stdout | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--vars` | JSON object of project variable overrides | + +Without `--json`, the command prints a summary: + +``` +DAG ready (24 nodes, 18 edges, 15 checks) +``` + +## Generating via compile + +You can also generate the DAG artifact as part of a compile: + +```bash +# Write to default location (target/sqlbuild_dag.json) +sqb compile --dag + +# Write to a specific path +sqb compile --dag target/my_dag.json +``` + +This is useful when you want to compile and generate the DAG in one step. The `SqlBuildProject.prepare()` method in the Dagster integration uses this path. + +## Output format + +The JSON artifact has this structure: + +```json +{ + "version": 1, + "project_name": "waffle_shop", + "nodes": [...], + "edges": [...], + "checks": [...] +} +``` + +### Nodes + +Each node represents a source, seed, model, or function: + +```json +{ + "id": "model:fact_orders", + "kind": "model", + "name": "fact_orders", + "asset_key": ["dev", "fact_orders"], + "target": { + "database": null, + "schema": "dev", + "name": "fact_orders", + "qualified_name": "dev.fact_orders" + }, + "path": "models/marts/fact_orders.sql", + "description": "Order fact table with waffle and payment details.", + "tags": ["marts"], + "columns": [ + {"name": "order_id", "type": "INTEGER"}, + {"name": "customer_id"} + ], + "materialization_type": "table" +} +``` + +| Field | Description | +|-------|-------------| +| `id` | Unique identifier (`{kind}:{name}`) | +| `kind` | `source`, `seed`, `model`, or `function` | +| `name` | Resource name | +| `asset_key` | Tuple used as the Dagster asset key (typically `[schema, name]` or `[database, schema, name]`) | +| `target` | Warehouse identity (database, schema, name, qualified_name) | +| `path` | Relative file path in the project | +| `description` | Model or source description, if declared | +| `tags` | Model tags | +| `columns` | Column metadata (name, type, nullable, description) | +| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | +| `language` | For functions: `sql` or `python` | +| `return_kind` | For functions: `scalar` or `table` | +| `arguments` | For functions: argument name and type pairs | + +### Edges + +Each edge is a dependency between two nodes: + +```json +{ + "from_id": "model:stg_orders", + "to_id": "model:fact_orders" +} +``` + +### Checks + +Each check represents a test, audit, or scenario: + +```json +{ + "id": "audit:not_null:model:fact_orders:order_id", + "kind": "audit", + "name": "not_null", + "checked_asset_ids": ["model:fact_orders"], + "path": "audits/generic/not_null.sql", + "severity": "error", + "attached_target_name": "fact_orders", + "attached_column_name": "order_id" +} +``` + +| Field | Description | +|-------|-------------| +| `id` | Unique check identifier | +| `kind` | `sql_test`, `audit`, or `scenario` | +| `name` | Check name | +| `checked_asset_ids` | Node IDs this check is attached to | +| `path` | Relative file path | +| `severity` | For audits: `error` or `warn` | +| `mode` | For tests: `model`, `macro`, `udf`, or `table_fn` | +| `assertion_names` | For scenarios: names of `__assert__` CTEs | +| `expected_model_names` | For scenarios: names of `__expected__` models | +| `fixture_refs` | For scenarios: names of fixture sources, refs, and seeds | + +## Examples + +```bash +# Summary output +sqb dag + +# Full JSON to stdout +sqb dag --json + +# Generate as part of compile +sqb compile --dag + +# Generate to a specific path +sqb compile --dag target/sqlbuild_dag.json + +# Pipe to jq for inspection +sqb dag --json | jq '.nodes | length' +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md new file mode 100644 index 0000000000..26be5dcf32 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md @@ -0,0 +1,84 @@ + + +# dbt + +> Coordinate dbt and SQLBuild projects. + +Online: https://docs.sqlbuild.com/cli/dbt + +# sqb dbt + +Orchestrate dbt and SQLBuild together. Each subcommand runs dbt first, then SQLBuild, with selection logic across both project graphs. dbt remains responsible for dbt-owned models; SQLBuild validates and executes SQLBuild-owned models downstream. See [Using SQLBuild with dbt](../concepts/dbt-compatibility/overview.md) for scope and selection behavior. + +## sqb dbt plan + +Preview combined dbt and SQLBuild work without executing. + +```bash +sqb dbt plan [--select ...] [--exclude ...] [--json] [--verbose] +``` + +Shows which dbt models will run, which SQLBuild models will run, and the dbt/SQLBuild commands that would be executed. + +## sqb dbt run + +Run dbt models first, then SQLBuild models. + +```bash +sqb dbt run [--select ...] [--exclude ...] +``` + +## sqb dbt build + +Build dbt models first (including dbt tests), then SQLBuild models with audits. + +```bash +sqb dbt build [--select ...] [--exclude ...] +``` + +## sqb dbt debug + +Run dbt diagnostics followed by SQLBuild diagnostics. + +```bash +sqb dbt debug [--no-connection] +``` + +## Selectors + +All `sqb dbt` commands accept `--select` and `--exclude`. Selectors work across both dbt and SQLBuild: + +```bash +# SQLBuild model with dbt dependencies +sqb dbt build --select downstream_orders + +# Full upstream chain including dbt +sqb dbt build --select +downstream_orders + +# Downstream of modified dbt models +sqb dbt build --select state:modified+ + +# SQLBuild models by tag +sqb dbt build --select tag:nightly + +# SQLBuild models by path +sqb dbt build --select path:models/marts + +# Exclude by tag +sqb dbt build --select fact_orders+ --exclude tag:nightly +``` + +See [Selection](../concepts/dbt-compatibility/selection.md) for full details on how selectors route work between dbt and SQLBuild. + +## Configuration + +Configure the dbt project location in `sqlbuild_project.toml`: + +```toml +[dbt] +project_dir = "../dbt_project" +profiles_dir = "../profiles" +target_path = "../dbt_project/target" +``` + +See [Project Configuration](../concepts/project-configuration.md#dbt) for all fields. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md new file mode 100644 index 0000000000..4b70bcd03f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -0,0 +1,71 @@ + + +# debug + +> Validate project configuration and test the warehouse connection. + +Online: https://docs.sqlbuild.com/cli/debug + +Checks that your project config is valid, the adapter is resolvable, and the warehouse connection works. Useful for diagnosing setup issues. + +## Usage + +```bash +sqb debug [flags] +``` + +## What it checks + +The command runs three groups of checks: + +**Runtime** - SQLBuild version, Python version, Python path, OS info. + +**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter and active target. + +**Connection** - Displays connection settings (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. + +## Flags + +| Flag | Description | +|------|-------------| +| `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | +| `--json` | Output results as JSON instead of formatted text. | + +## Example + +```bash +sqb debug +``` + +``` +SQLBuild Diagnostics + +Runtime: + sqlbuild version: 0.2.1 + python version: 3.14.3 + python path: /home/user/.venv/bin/python3 + os info: Linux-6.6.87 + +Configuration: + project file: /home/user/waffle_shop/sqlbuild_project.toml [OK found and valid] + local config: /home/user/waffle_shop/sqlbuild_local.toml [OK found] + project: waffle_shop [OK loaded] + adapter: snowflake [OK found] + target: dev [OK resolved] + +Connection: + account: FJMQFQV-OJ66172 + authenticator: programmatic_access_token + database: SQB_DB + role: role_sqb_test + schema: TEST + token: **** + user: svc_sqb_test + warehouse: SQB_WH + connection test: [OK connected] + query test: [OK SELECT 1] +``` + +## Exit codes + +Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/diff.md new file mode 100644 index 0000000000..d849384294 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/diff.md @@ -0,0 +1,66 @@ + + +# diff + +> Compare schemas and data between targets or virtual environments. + +Online: https://docs.sqlbuild.com/cli/diff + +Compares schemas and optionally row-level data between two build contexts: two targets (e.g. `prod:dev`) in direct mode, or two virtual environments when virtual mode is enabled. See [Data Diffs](../concepts/diff.md) for detailed usage. + +## Usage + +```bash +sqb diff : [flags] +``` + +The first argument is a positional `FROM:TO` range. Exactly one mode is required: `--full`, `--schema-only`, or `--bounded `. + +In direct mode, `FROM` and `TO` are configured target names. Their database/schema namespaces remain +authoritative, while the `TO` target's named connection executes the complete comparison and must +be able to read both namespaces. + +Full and bounded row comparisons require the model to define `unique_key`. Bounded mode uses the model's cursor and falls back to a full row comparison when no cursor is configured. + +## Flags + +| Flag | Description | +|------|-------------| +| `--full` | Compare both schema and all row data | +| `--schema-only` | Compare column names and types only | +| `--bounded` | Compare row data within a recent window (e.g. `14d`, `6h`) | +| `--verbose`, `-v` | Show more example rows (default: 3, verbose: 10) | +| `--max-column-examples` | Override maximum examples per changed column | +| `--max-row-only-examples` | Override maximum examples for side-only rows | +| `--sample-rows` | Override the deterministic unique-key sample size | +| `--sample-seed` | Override the deterministic sample seed | +| `--exhaustive` | Disable inherited sampling for this invocation | +| `--json-output` | Write structured comparison scope, coverage, and results to a JSON file | +| `--max-models` | Fail when the selected scope contains more models than this limit | +| `--max-columns` | Fail when either side of a model has more columns than this limit | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--select`, `-s` | Select specific models to diff (required in v1) | +| `--exclude` | Exclude specific models from diffing | + +## Examples + +```bash +# Full diff of a specific model +sqb diff prod:dev --full --select customer_status_snapshot + +# Schema-only diff of all marts +sqb diff prod:dev --schema-only --select path:models/marts + +# Bounded diff of last 14 days +sqb diff prod:dev --bounded 14d --select hourly_order_activity + +# Deterministic bounded sample +sqb diff prod:dev --bounded 14d --sample-rows 50000 --sample-seed 7 --select order_lines + +# Force exhaustive comparison despite inherited sampling defaults +sqb diff prod:dev --bounded 14d --exhaustive --select order_lines +``` + +## Exit codes + +Returns `0` when all selected models have no differences, `1` when any model has schema or row differences. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md new file mode 100644 index 0000000000..49650950e5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -0,0 +1,33 @@ + + +# format + +> Apply canonical SQLBuild SQL formatting. + +Online: https://docs.sqlbuild.com/cli/format + +# sqb format + +Formats SQL files using SQLBuild's canonical, dialect-aware representation. + +```bash +sqb format [flags] +``` + +| Flag | Description | +|------|-------------| +| `--check` | Fail when formatting changes are needed without writing | +| `--diff` | Print the proposed diff without writing | +| `--json` | Print structured results | +| `--select`, `-s` | Format selected models | +| `--select-file` | Read selectors from a file | +| `--exclude` | Exclude models from the selected scope | + +```bash +sqb format +sqb format --check +sqb format --diff --select tag:marts +``` + +Formatting owns presentation and source rewriting. Compiler-integrated diagnostics belong to +configured [Rules](../concepts/rules.md), which never mutate source. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md new file mode 100644 index 0000000000..ebe5c6e8cf --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md @@ -0,0 +1,177 @@ + + +# freshness + +> Observe source freshness without writing state. + +Online: https://docs.sqlbuild.com/cli/freshness + +## Contents + +- Usage +- Flags +- Source selection +- Observation without state +- Comparing against state +- CI integration +- JSON output +- See also + +# sqb freshness + +Observes the current data version of each source and reports whether the data has changed since the last build. Does not write any state or trigger builds. + +## Usage + +```bash +sqb --project-dir freshness [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--state` | Compare observations against stored freshness state from the last build | +| `--fail-on-error` | Exit with code 1 if any source observation fails or is unknown | +| `--fail-on-stale` | Exit with code 1 if any source has changed, is unknown, or errored (requires `--state`) | +| `--virtual-env` | Read previous state from the specified virtual environment instead of direct state | +| `--json` | Output as JSON instead of human-readable text | +| `--json-output` | Write JSON output to a file path (also prints text to stdout unless `--json` is set) | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--select`, `-s` | Select specific sources or models (sources upstream of selected models are included) | +| `--exclude` | Exclude specific sources or models | + +## Source selection + +Without `--select`, all sources in the project are observed. When `--select` is provided, selectors work like other commands: you can select sources by name, or select models and their upstream sources are automatically included: + +```bash +# Observe all sources +sqb freshness + +# Observe a specific source +sqb freshness --select raw_orders + +# Observe sources upstream of a model +sqb freshness --select fact_orders +``` + +## Observation without state + +By default, `sqb freshness` observes the current data version of each source and reports what it found. No comparison is made against previous observations: + +```bash +sqb freshness +``` + +``` +Source freshness + +Observed (3) + raw_customers timestamp 2026-06-05T14:30:00 adapter + raw_orders timestamp 2026-06-05T15:45:00 column tolerance 15m + raw_payments integer 42871 column + +Summary: observed=3 changed=0 unchanged=0 tolerated=0 unknown=0 errors=0 +``` + +Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata. Sources that can't be observed (expression sources, managed sources without freshness config on unsupported adapters) show as `unknown`. + +## Comparing against state + +Use `--state` to compare current observations against the freshness state stored from the last successful build: + +```bash +sqb freshness --state +``` + +``` +Source freshness + +Changed (1) + raw_orders previous 2026-06-05T12:00:00 current 2026-06-05T15:45:00 tolerance 15m + +Unchanged (1) + raw_customers previous 2026-06-05T14:30:00 current 2026-06-05T14:30:00 + +Tolerated (1) + raw_payments previous 2026-06-05T14:28:00 current 2026-06-05T14:30:00 tolerance 15m + +Summary: observed=0 changed=1 unchanged=1 tolerated=1 unknown=0 errors=0 +``` + +### Statuses + +| Status | Meaning | +|--------|---------| +| `observed` | Successfully observed (no state comparison) | +| `changed` | Data version differs from the stored state | +| `unchanged` | Data version matches the stored state exactly | +| `tolerated` | Data version differs but is within the `lag_tolerance` threshold | +| `unknown` | No freshness config and adapter metadata unavailable, or no previous state to compare against | +| `error` | Observation failed (e.g. source table does not exist, query error) | + +### Virtual environment state + +To compare against state stored in a virtual environment instead of direct mode state: + +```bash +sqb freshness --state --virtual-env pr_123 +``` + +## CI integration + +Use `--fail-on-error` to fail the pipeline if any source can't be observed: + +```bash +sqb freshness --fail-on-error +``` + +Use `--fail-on-stale` with `--state` to fail if any source has new data that hasn't been built yet: + +```bash +sqb freshness --state --fail-on-stale +``` + +This is useful for CI gates that should block if source data has changed but the pipeline hasn't run yet. + +## JSON output + +```bash +sqb freshness --json +``` + +```json +{ + "sources": [ + { + "name": "raw_orders", + "status": "observed", + "strategy": "column", + "value_kind": "timestamp", + "current_data_version": "2026-06-05T15:45:00", + "previous_data_version": null, + "lag_tolerance": "15m", + "target": { + "database": null, + "schema": "raw", + "name": "orders" + }, + "message": null + } + ], + "summary": { + "observed": 1, + "changed": 0, + "unchanged": 0, + "tolerated": 0, + "unknown": 0, + "errors": 0 + } +} +``` + +## See also + +- [Sources: Source freshness](../concepts/sources.md#source-freshness) for freshness configuration +- [Planning and Change Detection](../concepts/planning.md) for how freshness feeds into change-aware builds diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md new file mode 100644 index 0000000000..dca0fc030f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md @@ -0,0 +1,79 @@ + + +# init + +> Scaffold a new SQLBuild project. + +Online: https://docs.sqlbuild.com/cli/init + +# sqb init + +Creates a new SQLBuild project with a minimal directory structure and configuration files. + +## Usage + +```bash +sqb init +``` + +No flags. Run in the directory where you want to create the project. + +## Project layout + +`sqb init` creates the configuration, linter settings, and empty resource directories needed for a standalone project: + +```text +my-project/ + sqlbuild_project.toml + .sqruff + models/ + staging/ + marts/ + schemas/ + sources/ + seeds/ + loaders/ + tasks/ + assets/ + checks/ + hooks/ + sql/ + python/ + tests/ + unit/ + scenarios/ + functions/ + sql/ + python/ + macros/ + audits/ + generic/ + singular/ +``` + +Empty directories contain `.gitkeep` files so the scaffold can be committed. Reusable attached +audit definitions belong in `audits/generic/`; standalone audits belong in `audits/singular/`. +Add reusable SQL lifecycle hooks to `hooks/sql/` and decorated Python lifecycle hooks to +`hooks/python/`; see [Hooks](../concepts/models/hooks.md). + +The generated project uses DuckDB, creates a named `local` connection shared by `dev` and +`prod`, and defaults models to table materialization. Its configuration follows this shape: + +```toml +adapter = "duckdb" +default_target = "dev" + +[connections.local] +database = "my_project.duckdb" + +[targets.dev] +connection = "local" +schema = "dev" + +[targets.prod] +connection = "local" +schema = "prod" +``` + +The project name is derived from the current directory name, with hyphens converted to +underscores. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md new file mode 100644 index 0000000000..60356ca55b --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -0,0 +1,59 @@ + + +# janitor + +> Clean up stale warehouse relations. + +Online: https://docs.sqlbuild.com/cli/janitor + +# sqb janitor + +Identifies and removes stale warehouse relations that are no longer part of the project. Requires `janitor.enabled = true` in `sqlbuild_project.toml`. + +## Usage + +```bash +sqb --project-dir janitor [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--auto-approve` | Skip the confirmation prompt and delete immediately | +| `--retention-days` | Override the configured retention period (days) | + +## Configuration + +Configure janitor behavior in `sqlbuild_project.toml`: + +```toml +[janitor] +enabled = true +retention_days = 30 +delete_tracked_only = true +exclude_patterns = ["audit_*", "tmp_*"] +``` + +See [Project Configuration](../concepts/project-configuration.md) for details on janitor settings. + +## Examples + +```bash +# Interactive mode (prompts for confirmation) +sqb janitor + +# Auto-approve deletion +sqb janitor --auto-approve + +# Override retention to 7 days +sqb janitor --retention-days 7 +``` + +## State history pruning + +In addition to cleaning up stale relations, the janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` tables, retaining only the latest record per identity. This keeps state tables compact without affecting change detection. + +## Safety + +Janitor prompts for confirmation before deleting. The confirmation requires typing an exact string to prevent accidental deletion. Use `--auto-approve` only in CI or when you're certain. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md new file mode 100644 index 0000000000..4ec2ed8766 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md @@ -0,0 +1,258 @@ + + +# lineage + +> Explore model and column-level dependency graphs from the command line. + +Online: https://docs.sqlbuild.com/cli/lineage + +## Contents + +- Usage +- Flags +- Model lineage +- Column lineage +- Examples +- Depth limiting + +Inspect upstream and downstream dependencies for any model, source, seed, or function in your project. Supports both model-level lineage (dependency graph) and column-level lineage (tracing individual columns through transformations). Outputs as a tree, edge list, or structured JSON. + +`sqb compile` computes lineage for validation and includes per-model summaries in its JSON report. Use this command when you need to inspect or export the actual lineage graph. + +## Usage + +```bash +# Model lineage: dependency graph around a single resource +sqb lineage [flags] + +# Column lineage: trace a specific column +sqb lineage . [flags] + +# Selector mode: lineage for a selected scope +sqb lineage --select [flags] +``` + +Exactly one of a positional target or `--select` is required. + +## Flags + +| Flag | Description | +|------|-------------| +| `--direction` | `upstream` (default), `downstream`, or `both`. `both` is only available for model lineage. | +| `--depth` | How many hops to traverse. An integer or `all` (default: `all`). | +| `--format` | Output format: `tree` (default), `list`, or `json`. | +| `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | +| `--select`, `-s` | Select resources using standard selector syntax. | +| `--exclude` | Exclude resources from the selection. | + +## Model lineage + +When the target is a plain resource name, lineage shows the model-level dependency graph. + +### Tree + +The default. Shows an indented dependency tree with resource types and file paths: + +```bash +sqb lineage fact_orders --direction both +``` + +``` +Lineage model fact_orders models/marts/fact_orders.sql both +upstream +├── model stg_orders models/staging/stg_orders.sql +│ └── source raw__orders sources/raw.yml +├── model stg_payments models/staging/stg_payments.sql +│ └── source raw__payments sources/raw.yml +├── seed waffle_types seeds/waffle_types.csv +└── function udf__is_completed_order functions/sql/udf__is_completed_order.sql +downstream +├── model customer_status_snapshot models/intermediate/customer_status_snapshot.sql +├── model hourly_order_activity models/marts/hourly_order_activity.sql +│ ├── model daily_activity_rollup models/marts/daily_activity_rollup.sql +│ │ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql +│ │ └── model hourly_order_activity (already shown) +│ └── model hourly_activity_with_daily_context (already shown) +└── model order_status_index models/intermediate/order_status_index.sql +``` + +Cycles and repeated nodes are annotated with "(already shown)" to avoid infinite recursion. + +### List + +An edge list showing each dependency as a directed pair: + +```bash +sqb lineage fact_orders --format list +``` + +``` +source:raw__orders -> model:stg_orders +source:raw__payments -> model:stg_payments +model:stg_orders -> model:fact_orders +model:stg_payments -> model:fact_orders +seed:waffle_types -> model:fact_orders +``` + +### JSON + +Structured output with nodes, edges, and metadata: + +```bash +sqb lineage fact_orders --format json +``` + +```json +{ + "nodes": [ + { + "id": "model:fact_orders", + "name": "fact_orders", + "resource_type": "model", + "relative_path": "models/marts/fact_orders.sql", + "qualified_name": "dev.fact_orders" + }, + { + "id": "source:raw__orders", + "name": "raw__orders", + "resource_type": "source", + "relative_path": "sources/raw.yml" + } + ], + "edges": [ + {"from": "source:raw__orders", "to": "model:stg_orders"}, + {"from": "model:stg_orders", "to": "model:fact_orders"} + ], + "focus": ["model:fact_orders"], + "direction": "upstream" +} +``` + +## Column lineage + +When the target uses `model.column` syntax, lineage traces the specific column through upstream or downstream transformations. Each edge is annotated with a transform type (`direct`, `expression`, `aggregation`, `cast`, `star`, `constant`) and a confidence level. See [Column Lineage](../concepts/column-lineage.md) for a full explanation of transform types, confidence levels, and analysis modes. + +```bash +# Where does fact_orders.total_cents come from? +sqb lineage fact_orders.total_cents + +# What consumes fact_orders.order_id downstream? +sqb lineage fact_orders.order_id --direction downstream +``` + +Column lineage supports `upstream` and `downstream` directions (not `both`). The `--mode` flag selects the analysis mode: `rich` (default, full SQL analysis) or `fast` (lightweight, faster on large projects). + +### Tree + +```bash +sqb lineage fact_orders.total_cents +``` + +``` +Column trace fact_orders.total_cents upstream + + <- stg_payments.amount_cents (expression) + <- raw__payments.amount_cents (direct) +``` + +### List + +```bash +sqb lineage fact_orders.total_cents --format list +``` + +``` +Column dependencies + +stg_payments.amount_cents -> fact_orders.total_cents expression +raw__payments.amount_cents -> stg_payments.amount_cents direct +``` + +### JSON + +```bash +sqb lineage fact_orders.total_cents --format json +``` + +```json +{ + "target": { + "resource_type": "model", + "resource_name": "fact_orders", + "column_name": "total_cents" + }, + "direction": "upstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 5, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_payments", + "column_name": "amount_cents" + }, + "target": { + "resource_type": "model", + "resource_name": "fact_orders", + "column_name": "total_cents" + }, + "transform": "expression", + "confidence": "high" + } + ] +} +``` + +## Examples + +```bash +# Upstream dependencies of a model (default) +sqb lineage fact_orders + +# Downstream dependents +sqb lineage fact_orders --direction downstream + +# Both directions, limited to 1 hop +sqb lineage fact_orders --direction both --depth 1 + +# Column lineage: trace a specific column upstream +sqb lineage fact_orders.total_cents + +# Column lineage: trace downstream consumers +sqb lineage fact_orders.order_id --direction downstream + +# Column lineage with depth limit +sqb lineage fact_orders.total_cents --depth 1 + +# Lineage for all models in a path +sqb lineage --select path:models/marts + +# Lineage between two models (path-between) +sqb lineage --select "stg_orders~daily_activity_rollup" + +# Upstream expansion from a model +sqb lineage --select "+fact_orders" + +# JSON output for programmatic consumption +sqb lineage fact_orders --format json --direction upstream +``` + +## Depth limiting + +`--depth` controls how many hops from the focus node(s) to include. In selector mode, `--depth` requires name, source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors. + +```bash +# Only immediate parents +sqb lineage fact_orders --depth 1 + +# Parents and grandparents +sqb lineage fact_orders --depth 2 + +# Only direct column dependencies +sqb lineage fact_orders.total_cents --depth 1 +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md new file mode 100644 index 0000000000..99cc03766e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md @@ -0,0 +1,95 @@ + + +# load + +> Load managed sources into the warehouse. + +Online: https://docs.sqlbuild.com/cli/load + +# sqb load + +Runs source loader functions and writes data into their target tables using the configured write strategy. + +## Usage + +```bash +sqb --project-dir load [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--select`, `-s` | Select specific sources or loaders by name | +| `--exclude` | Exclude specific sources or loaders | +| `--reload` | Pass `is_reload=True` to loader functions | +| `--concurrency` | Number of worker connections (default: from settings or 1) | +| `--start-cursor-ts` | Override start cursor for timestamp-based loaders (ISO format) | +| `--end-cursor-ts` | Override end cursor for timestamp-based loaders (ISO format) | +| `--start-cursor-int` | Override start cursor for integer-based loaders | +| `--end-cursor-int` | Override end cursor for integer-based loaders | +| `--json` | Output results as JSON | +| `--json-output` | Write JSON results to a file path | +| `--var` | Set project variables (`--var key=value`) | + +## Examples + +```bash +# Load all managed sources +sqb load + +# Load a specific source +sqb load --select raw_customers + +# Load with concurrency +sqb load --concurrency 4 + +# Reload all sources (full refresh behavior) +sqb load --reload + +# Override cursor bounds +sqb load --start-cursor-ts "2026-05-01T00:00:00" +``` + +## Execution order + +Loaders are executed in DAG topological order based on `depends_on` declarations. Independent loaders run concurrently when `--concurrency` is greater than 1. + +Intermediate loaders (those referenced only via `depends_on` without a direct source binding) run first, followed by the source-bound loaders that depend on them. + +## Output + +``` +Load ready (3 selected) + +Sources (3) + raw_customers + raw_orders + raw_payments + +Execution sqb load (concurrency: 1) + + 1/3 source raw_customers OK 0.05s rows=5 + 2/3 source raw_orders OK 0.03s rows=10 + 3/3 source raw_payments OK 0.02s rows=8 + +Completed successfully. +PASS=3 WARN=0 FAIL=0 SKIP=0 TOTAL=3 (0.12s) +``` + +## Auto-load during builds + +`sqb build` automatically loads managed sources before building dependent models. This is controlled by the `auto_load_sources` setting (default: `true`) and the `--load` / `--no-load` / `--reload` flags: + +```bash +# Default: auto-load is on +sqb build + +# Explicitly skip loading +sqb build --no-load + +# Force reload +sqb build --reload +``` + +See [Loaders](../concepts/python-nodes/loaders.md) for full documentation on write strategies, the loader context API, and source deferral. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md new file mode 100644 index 0000000000..8614a3b64a --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -0,0 +1,89 @@ + + +# plan + +> Preview what SQLBuild will do before executing. + +Online: https://docs.sqlbuild.com/cli/plan + +# sqb plan + +Shows the execution plan without making any changes. Useful for inspecting change detection, backfill policies, and selector scope before building. + +## Usage + +```bash +sqb --project-dir plan [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--changes-only` | Virtual mode only: show work not already bound to its expected VDE version | +| `--no-python` | Exclude read-side Python tasks and assets from the plan | +| `--defer-to` | Resolve unselected model references against another target | +| `--json` | Output the plan as JSON | +| `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | +| `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | +| `--start-cursor-int` | Override start cursor for integer incremental models | +| `--end-cursor-int` | Override end cursor for integer incremental models | +| `--select`, `-s` | Select specific models | +| `--exclude` | Exclude specific models | + +## Output + +First run: + +``` +Plan ready (13 selected) + +First run (12) + stg_customers view + stg_payments view + stg_orders view + daily_order_partitioned partition_tracked (custom) + daily_revenue table + dim_customers table + fact_orders table + customer_status_snapshot merge (timestamp) + hourly_order_activity delete_insert (timestamp, microbatch) + daily_activity_rollup delete_insert (timestamp, microbatch) + hourly_activity_with_daily_context delete_insert (timestamp, microbatch) + order_status_index delete_insert (integer) + +Seeds (1) + waffle_types +``` + +Steady state: + +``` +Plan ready (13 selected) + +Normal (12) + 3 view + 3 table + 3 delete_insert (timestamp, microbatch) + 1 partition_tracked (custom) + 1 merge (timestamp) + 1 delete_insert (integer) + +Seeds (1) + waffle_types +``` + +When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. + +## Missing upstream dependencies + +Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: + +``` +error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) +``` + +Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md new file mode 100644 index 0000000000..5cce645ea1 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md @@ -0,0 +1,95 @@ + + +# playground + +> Create a self-contained SQLBuild project to explore locally. + +Online: https://docs.sqlbuild.com/cli/playground + +# sqb playground + +Creates a self-contained waffle shop project with DuckDB. No warehouse credentials, no git clone, no external data - just a working project you can compile, build, test, and explore immediately. + +## Usage + +```bash +sqb playground [name] +``` + +The positional argument is the directory to create (default `sqlbuild-playground`). The template is chosen with `--template` (default `waffle_shop`). + +## Templates + +| Template | Description | +|----------|-------------| +| `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | +| `dagster` | Waffle shop project plus a `dagster/` directory with a ready-to-run `definitions.py`. | +| `rivers` | Waffle shop project plus a `rivers_pipeline/` directory with a Rivers repository definition. | +| `virtual` | Waffle shop with virtual environments enabled, a local DuckDB state store, loaders, and the full virtual lifecycle (build, promote, rollback). | +| `python_nodes` | A small DuckDB project demonstrating [Python nodes](../concepts/python-nodes/overview.md): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | + +## What it creates + +A complete DuckDB-backed project with: + +- Staging views, fact/dimension tables, and incremental models +- Sources with inline expression data (no external setup) +- Seeds, SQL functions, and a custom materialization +- Built-in and custom audits +- SQL unit tests and multi-model tests +- E2E scenario tests +- Python macros +- AI agent skill files (auto-installed for OpenCode, Claude Code, and other agents) + +The `dagster` template adds: + +- `dagster/definitions.py` - Dagster definitions with `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource` +- `dagster/README.md` - Setup instructions + +The `python_nodes` template instead creates a focused Python-nodes project: + +- `tasks/orders.py`, `loaders/orders.py`, `assets/orders_export.py`, `checks/orders_export.py` +- A `fact_orders` SQL model over a managed `raw_orders` source +- Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check + +## Examples + +```bash +# Default waffle shop +sqb playground waffle-shop +cd waffle-shop +sqb build + +# With Dagster integration +sqb playground --template dagster +cd sqlbuild-playground +uv pip install 'sqlbuild[dagster]' +dagster dev -f dagster/definitions.py + +# With Rivers integration +sqb playground --template rivers +cd sqlbuild-playground +uv pip install 'sqlbuild[rivers]' +uv run rivers dev rivers_pipeline.definitions + +# With virtual environments +sqb playground --template virtual +cd sqlbuild-playground +sqb state init +sqb build +sqb build --virtual-env pr +sqb promote --from pr --to dev + +# With Python nodes +sqb playground --template python_nodes +cd sqlbuild-playground +sqb build --select +fact_orders --select +orders_export +sqb check --select +check_orders_export +``` + +## Notes + +- The target directory must not already exist +- DuckDB is included as a core dependency - no extra installation needed +- The local DuckDB database file is created on the first build +- The Dagster template uses `prepare_if_dev()` to auto-generate the DAG artifact when Dagster starts in dev mode diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/promote.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/promote.md new file mode 100644 index 0000000000..67ee037a54 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/promote.md @@ -0,0 +1,48 @@ + + +# promote + +> Promote VDE refs from one virtual environment to another. + +Online: https://docs.sqlbuild.com/cli/promote + +# sqb promote + +Promotes model version refs from a source VDE to a target VDE. No models are rebuilt - promotion swaps pointers and refreshes logical views. Virtual mode only. + +## Usage + +```bash +sqb promote --from --to [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--from` | Source VDE name (required) | +| `--to` | Target VDE name (required) | +| `--select`, `-s` | Select specific models to promote | +| `--exclude` | Exclude specific models | +| `--allow-partial-promotion` | Accept a working target VDE when partial promotion leaves downstream stale | +| `--include-stale-upstreams` | Add required stale upstream refs to the promotion scope | +| `--verbose`, `-v` | Show uncapped model sets | +| `--var` | Set project variables | + +## Examples + +```bash +# Promote all models from pr_123 to dev +sqb promote --from pr_123 --to dev + +# Promote specific models +sqb promote --from pr_123 --to dev --select fact_orders + +# Partial promotion with working target accepted +sqb promote --from pr_123 --to dev --select fact_orders --allow-partial-promotion + +# Include stale upstreams for coherent partial promotion +sqb promote --from pr_123 --to dev --select fact_orders --include-stale-upstreams +``` + +See [Promotion](../concepts/virtual-environments/promotion.md) for details on promotion behavior, source requirements, and guards. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md new file mode 100644 index 0000000000..4ecac29e9f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -0,0 +1,102 @@ + + +# query + +> Run ad hoc SQL queries against the project database. + +Online: https://docs.sqlbuild.com/cli/query + +# sqb query + +Execute ad hoc SQL against the active project connection. Useful for inspecting data, debugging models, or running one-off queries without leaving the SQLBuild CLI. + +## Usage + +```bash +sqb --project-dir query "SELECT * FROM dev.fact_orders LIMIT 5" +``` + +Or from a file: + +```bash +sqb --project-dir query --file my_query.sql +``` + +File paths are resolved from the current working directory. + +## Flags + +| Flag | Description | +|------|-------------| +| `sql` | SQL to execute (positional argument) | +| `--file` | Read SQL from a file instead of the command line | +| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | +| `--limit` | Maximum rows to return (default: 20) | +| `--no-limit` | Disable the row limit | + +## Output formats + +### long (default) + +Vertical record format, one field per line: + +``` +-[ RECORD 1 ]---------------------------+ +order_id | 1 +customer_id | 100 +status | completed + +-[ RECORD 2 ]---------------------------+ +order_id | 2 +customer_id | 200 +status | completed + +2 rows +``` + +### table + +Horizontal table format: + +``` +order_id | customer_id | status +-------- | ----------- | --------- +1 | 100 | completed +2 | 200 | completed + +2 rows +``` + +### json + +JSON array of objects: + +```json +[{"order_id": 1, "customer_id": 100, "status": "completed"}] +``` + +### csv + +Standard CSV with headers: + +```csv +order_id,customer_id,status +1,100,completed +2,200,completed +``` + +## Examples + +```bash +# Quick inspection with default format +sqb query "SELECT * FROM dev.fact_orders" + +# Table format with higher limit +sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 + +# Export to JSON +sqb query "SELECT * FROM dev.daily_revenue" --format json --no-limit + +# Run SQL from a file +sqb query --file debug_query.sql +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/reconcile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/reconcile.md new file mode 100644 index 0000000000..e059f710e7 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/reconcile.md @@ -0,0 +1,66 @@ + + +# reconcile + +> Diagnose and repair drift between virtual state and warehouse. + +Online: https://docs.sqlbuild.com/cli/reconcile + +# sqb reconcile + +Detects and repairs inconsistencies between the virtual state store and the warehouse. Virtual mode only. + +## Usage + +```bash +sqb reconcile [--virtual-env ] [--model ] +sqb reconcile repair-view --virtual-env --model +sqb reconcile attach --virtual-env --model --physical-relation +``` + +## Subcommands + +### (default) + +Run a diagnostic report without changing anything: + +```bash +sqb reconcile --virtual-env dev +``` + +### repair-view + +Recreate a logical VDE view from trusted state refs: + +```bash +sqb reconcile repair-view --virtual-env dev --model fact_orders +``` + +No confirmation needed. Idempotent (`CREATE OR REPLACE VIEW`). + +### attach + +Rebind a VDE model ref to a different tracked physical relation: + +```bash +sqb reconcile attach --virtual-env dev --model fact_orders \ + --physical-relation dev__sqb_physical.fact_orders__v_8f3a9c12 +``` + +Requires confirmation by default. + +## Examples + +```bash +# Diagnose issues +sqb reconcile --virtual-env dev + +# Fix a missing view +sqb reconcile repair-view --virtual-env dev --model fact_orders + +# Rebind a model to a different physical version +sqb reconcile attach --virtual-env dev --model fact_orders \ + --physical-relation dev__sqb_physical.fact_orders__v_71d0e4ab +``` + +See [Reconcile](../concepts/virtual-environments/reconcile.md) for details on guards and when to use each subcommand. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rollback.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rollback.md new file mode 100644 index 0000000000..35e21054c8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rollback.md @@ -0,0 +1,48 @@ + + +# rollback + +> Roll back a VDE to a prior finalized checkpoint. + +Online: https://docs.sqlbuild.com/cli/rollback + +# sqb rollback + +Restores a virtual environment to a prior finalized checkpoint by rebinding its refs and refreshing logical views. Virtual mode only. + +## Usage + +```bash +sqb rollback [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--virtual-env` | Target VDE name (defaults to active target name) | +| `--checkpoint-id` | Restore a specific checkpoint instead of the previous one | +| `--select`, `-s` | Roll back specific models only | +| `--exclude` | Exclude specific models | +| `--allow-partial-rollback` | Accept a working VDE when partial rollback leaves downstream stale | +| `--include-stale-upstreams` | Add required upstream refs from the checkpoint | +| `--verbose`, `-v` | Show uncapped model sets | +| `--var` | Set project variables | + +## Examples + +```bash +# Roll back to the previous finalized checkpoint +sqb rollback + +# Roll back a specific VDE +sqb rollback --virtual-env pr_123 + +# Roll back to a specific checkpoint +sqb rollback --checkpoint-id + +# Partial rollback +sqb rollback --select fact_orders --allow-partial-rollback +``` + +See [Rollback](../concepts/virtual-environments/rollback.md) for details on checkpoints, rollback behavior, and guards. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md new file mode 100644 index 0000000000..fafdb2cb7c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md @@ -0,0 +1,52 @@ + + +# rules + +> List, inspect, or run compiler-integrated Rules and generate project guidance. + +Online: https://docs.sqlbuild.com/cli/rules + +# sqb rules + +Configured Rules run automatically through `sqb compile` and build planning. The `rules` command +provides focused catalogue, inspection, execution, and guidance operations. + +## Usage + +```bash +sqb rules list +sqb rules show +sqb rules run [--select ] [--exclude ] +sqb rules skills [--check] +``` + +Add `--json` after `rules` for machine-readable list or run output: + +```bash +sqb rules --json list +sqb rules --json run SQBRSQL +``` + +Lifecycle progress is written to stderr so stdout remains valid JSON. + +## Commands + +- `list`: print built-in and repository-defined Rules. +- `show`: print metadata for one exact code. +- `run`: compile and evaluate only one exact code or family prefix. +- `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. + +`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. + +## Exit codes + +| Command | Code | Meaning | +|---|---:|---| +| `rules list` | 0 | Catalogue printed | +| `rules show` | 0 | Exact code found | +| `rules show` | 2 | Unknown code | +| `rules run` | 0 | No findings | +| `rules run` | 1 | Findings reported or compilation failed | +| `rules run` | 2 | Unknown code or family | +| `rules skills --check` | 0 | Guidance is fresh | +| `rules skills --check` | 1 | Guidance is missing or stale | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md new file mode 100644 index 0000000000..e9fec8b062 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -0,0 +1,168 @@ + + +# scenario + +> Run end-to-end scenario tests against the warehouse or locally with DuckDB. + +Online: https://docs.sqlbuild.com/cli/scenario + +## Contents + +- sqb scenario test +- sqb scenario capture +- Runtime artifacts + +# sqb scenario + +Run end-to-end scenario tests. Scenarios materialize fixture inputs as physical relations, build the real project graph against them, and evaluate expected outputs and assertions. See [Scenarios](../concepts/scenarios.md) for concepts and authoring details. + +## sqb scenario test + +Run scenario tests against the warehouse. + +```bash +sqb scenario test [flags] +``` + +### Flags + +| Flag | Description | +|------|-------------| +| `--select`, `-s` | Select scenarios to run | +| `--exclude` | Exclude scenarios from the selection | +| `--retain` | Keep scenario-owned warehouse artifacts for inspection | +| `--local` | Run locally against DuckDB using captured JSONL snapshots | +| `--strict` | Treat missing/stale local snapshots as errors instead of skips | +| `--sync-snapshots` | Capture missing/stale snapshots before local run (requires `--local`) | +| `--refresh` | Recapture all selected snapshots before local run (requires `--local`) | +| `--force` | Bypass snapshot capture safety limits | +| `--max-snapshot-rows` | Override per-relation row limit for capture | +| `--max-snapshot-total-rows` | Override total row limit for capture | +| `--max-snapshot-bytes` | Override per-relation byte limit for capture | +| `--max-snapshot-total-bytes` | Override total byte limit for capture | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | + +### Selectors + +Scenarios are selected with `--select`. `--exclude` removes scenarios from the selection. Without any selectors, all discovered scenarios run. + +| Selector | Example | +|----------|---------| +| Scenario name | `sqb scenario test --select daily_revenue_minimal` | +| Multiple names | `sqb scenario test --select daily_revenue_minimal --select daily_revenue_multi_order` | +| `.sql` file path | `sqb scenario test --select tests/scenarios/revenue/daily_revenue_minimal.sql` | +| Folder | `sqb scenario test --select tests/scenarios/revenue` | +| Scenario-root-relative folder | `sqb scenario test --select revenue` | +| Exclude | `sqb scenario test --select revenue --exclude daily_revenue_multi_order` | + +Mixed selector types are supported and the result is de-duplicated by scenario name. + +### Remote examples + +```bash +# Run all scenarios +sqb scenario test + +# Run one scenario +sqb scenario test --select daily_revenue_minimal + +# Run and retain warehouse artifacts +sqb scenario test --select daily_revenue_minimal --retain + +# Run all scenarios in a folder +sqb scenario test --select revenue +``` + +### Local examples + +```bash +# Run locally (requires prior capture) +sqb scenario test --local + +# Run locally, capture missing/stale snapshots first +sqb scenario test --local --sync-snapshots + +# Run locally, recapture everything first +sqb scenario test --local --refresh + +# Fail on missing/stale snapshots instead of skipping +sqb scenario test --local --strict +``` + +### Output + +Remote scenarios report per-scenario PASS/FAIL with nested check rows: + +``` +daily_revenue_minimal PASS + check expected daily_revenue PASS + check assertion no_negative_revenue PASS + +PASS=1 FAIL=0 TOTAL=1 +``` + +Local scenarios add ERROR and SKIP statuses: + +``` +PASS=2 FAIL=0 ERROR=0 SKIP=1 TOTAL=3 +``` + +Failed scenarios suggest rerunning with `--retain` for inspection. Local runs always keep the DuckDB file at `target/run/scenarios//local.duckdb`. + +## sqb scenario capture + +Capture scenario input fixtures from the warehouse as JSONL snapshots for local replay. + +```bash +sqb scenario capture [flags] +``` + +### Flags + +| Flag | Description | +|------|-------------| +| `--select`, `-s` | Select scenarios to capture | +| `--exclude` | Exclude scenarios from the selection | +| `--retain` | Keep warehouse fixture artifacts after capture | +| `--force` | Bypass snapshot capture safety limits | +| `--max-snapshot-rows` | Override per-relation row limit | +| `--max-snapshot-total-rows` | Override total row limit | +| `--max-snapshot-bytes` | Override per-relation byte limit | +| `--max-snapshot-total-bytes` | Override total byte limit | +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | + +### Examples + +```bash +# Capture all scenarios +sqb scenario capture + +# Capture one scenario +sqb scenario capture --select daily_revenue_minimal + +# Capture and retain warehouse artifacts +sqb scenario capture --select daily_revenue_minimal --retain +``` + +Snapshots are written to `tests/_scenario_snapshots//` with a `scenario.json` manifest and JSONL files for each fixture relation. These files can be committed to version control. + +## Runtime artifacts + +Both remote and local scenario runs write runtime artifacts to `target/run/scenarios//`: + +``` +target/run/scenarios/daily_revenue_minimal/ + cleanup/ + prepare.sql + final.sql + fixtures/ + ref__stg_orders.sql + ref__stg_payments.sql + models/ + marts/daily_revenue.sql + checks/ + expected__daily_revenue.sql + assertion__no_negative_revenue.sql +``` + +Local runs additionally write to a `local/` subdirectory and create `local.duckdb`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md new file mode 100644 index 0000000000..c6fcebc506 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md @@ -0,0 +1,413 @@ + + +# scope + +> Inspect declaration visibility, usage, placement, and move impact offline. + +Online: https://docs.sqlbuild.com/cli/scope + +## Contents + +- Usage +- Flags +- Reading A Report +- Expected-output access +- Nearby And Explain +- Prospective Paths +- Move Preview +- Folder Browsing +- Pagination +- Paths And JSON +- Partial Projects +- Output Safety +- Native SQLBuild Only + +`sqb scope` explains the lexical environment around native SQLBuild macros, enums, and constants. Use it to see what a resource can access, what it actually uses, why a declaration is visible or inaccessible, and whether a move would cross a scope boundary. + +The command is read-only and offline. It does not connect to a warehouse, require warehouse credentials, or inspect relations. It reads the compiler-owned declaration-scope index and reuses a deterministic cache when the relevant source and configuration fingerprint has not changed. Building a cold cache uses the normal compiler expansion path, which loads the configured adapter and may execute authored Python macros; a warm cache hit reconstructs scope facts without importing macro modules. + +Text output is deterministic, bounded, and requires no interactive pager. + +## Usage + +```bash +# Inspect an existing resource, declaration, or exact resource path +sqb scope TARGET [flags] + +# Inspect the scope at a prospective file or directory +sqb scope --at PATH [flags] + +# Preview moving an existing resource to a new path +sqb scope TARGET --as-path PATH [flags] +``` + +`TARGET` is either a kind-qualified identity or an exact project-relative resource path: + +```bash +sqb scope model:stg_orders +sqb scope test:stg_orders__excludes_cancelled +sqb scope scenario:daily_revenue +sqb scope hook:grant_select +sqb scope function:normalize_status +sqb scope audit:positive_order_value +sqb scope source:raw__orders + +sqb scope macro:normalize_order_status +sqb scope enum:order_status +sqb scope constant:minimum_order_value + +sqb scope models/staging/orders/stg_orders.sql +``` + +Bare names such as `stg_orders` and `order_status` are rejected even if they are currently unique. Qualification keeps commands stable when names collide across resource or declaration kinds. Public declaration identities are `macro:`, `enum:`, and `constant:`. + +Model-private constants and enums include their owner in the introspection identity: + +```text +enum:model:stg_orders._state +constant:model:stg_orders._minimum_value +``` + +These identities are for inspection. SQL authored inside the owning model continues to use `@enum("_state")` and `@const("_minimum_value")`. + +## Flags + +| Flag | Description | +|------|-------------| +| `--at PATH` | Inspect path-derived visibility for a prospective file or directory. A trailing `/` denotes a directory whose direct children are being considered. | +| `--as-path PATH` | Preview moving the existing target resource to `PATH`. No file or configuration is changed. | +| `--browse PATH` | Show direct child declaration folders and recursive counts without listing declarations. | +| `--list PATH` | Recursively list declarations under one declaration folder. | +| `--defined-under PATH` | Keep declarations whose definitions are under the project-relative path. | +| `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | +| `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | +| `--used-only` | Keep declarations used by the target. | +| `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | +| `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | +| `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | +| `--explain QUALIFIED_NAME` | Explain one qualified declaration in detail at the target. | +| `--globals POLICY` | Control global declarations in the main report: `summary` (default), `used`, or `all`. | +| `--page-size N` | Return at most `N` declarations per paged section (default: `100`). | +| `--after CURSOR` | Continue after a declaration identity returned as the previous section's next cursor. | +| `--paths MODE` | Render paths as `relative`, `compact`, or `none`. | +| `--json` | Write the canonical versioned JSON report instead of the text tree. | + +Filters combine in a fixed order: definition path, kind, glob, then usage. `--dependency-depth` +expands the filtered used declarations afterward. Parent-folder visibility is never depth-limited: +every unprefixed declaration role above the resource contributes to its scope. + +## Reading A Report + +The default text report keeps distinct facts in distinct sections: + +- **Available** contains declarations visible through the resource's own lexical path. +- **Used** contains declarations consumed by the resource, including tracked declaration dependencies. +- **Relationship grants** contains eligible file-based enums and constants made available through expected-output sections in a test or scenario. +- **Nearby unavailable** is opt-in and explains close declarations that are outside the target's scope. +- **Scope chain** starts with the resource's owner folder, then shows parent folders and the project + declaration roles. Its compact labels identify the access rule checked at each path. + +Ordinary text rows show each declaration's qualified identity, definition location, visibility, +reason for appearing, declaration-role root, and safe type or signature metadata where useful. +`--explain` additionally shows its owning path, narrowest required placement, current consumers, +dependencies, and the consumers affected by a placement mismatch. JSON includes the complete +structured declaration report. + +### See the whole scope at once + +Suppose orders and returns models use a macro published from their shared `commerce` folder, and +that macro composes two project-wide macros: + +```text +project/ +├── macros/ +│ └── currency.py add_tax, round_money +└── models/ + └── commerce/ + ├── macros/ + │ └── orders.py formatted_order_total + ├── orders.sql + └── returns/ + └── returns.sql +``` + +```python +# models/commerce/macros/orders.py +from macros.currency import add_tax, round_money + +def formatted_order_total(expression: str) -> str: + return round_money(add_tax(expression)) +``` + +```sql +-- models/commerce/orders.sql +MODEL(); + +SELECT @formatted_order_total("subtotal") AS total +``` + +`returns/returns.sql` calls the same macro, so the declaration must remain available throughout +`models/commerce/` and its descendants. + +The default report puts actual usage next to the complete directory-derived scope. Unused globals +stay collapsed, so the useful facts remain visible even in a large project: + +```console +$ sqb scope model:orders +Scope + Target: model:orders + Resource: model:orders + Path: models/commerce/orders.sql + +Used (1) + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 + +Scope chain + ├─ exact-owner-private models/commerce (1) + ├─ descendant-public models (0) + └─ project global (2) + +Available (1 of 3, 2 collapsed) + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 + … 2 globals collapsed; run sqb scope model:orders --globals all + +Relationship grants (0 of 0) + (none) + +Nearby unavailable (0 of 0) + (none) + +Diagnostics (0) + (none) +Completeness: complete +``` + +This answers several questions without opening declaration files: what the model actually uses, +where that declaration came from, which owner and parent folders SQLBuild checks, how many +project-wide declarations are available, and whether the result is complete. The scope-chain count +at `models/commerce` includes declarations owned there; the `exact-owner-private` chain label does +not mean every declaration in that count is private. + +### Follow composed macro dependencies + +`Used` is direct by default. Add `--dependency-depth` to follow declarations used by those +declarations. In this example, the model calls `formatted_order_total`, whose Python body calls +`add_tax` and `round_money`: + +```console +$ sqb scope model:orders --used-only --dependency-depth 1 +Scope + Target: model:orders + Resource: model:orders + Path: models/commerce/orders.sql + +Used (3) + ├─ ● macro:add_tax [macro; project; dependency; params 1; role macros] macros/currency.py:1:1 + ├─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 + └─ ● macro:round_money [macro; project; dependency; params 1; role macros] macros/currency.py:5:1 +``` + +The remaining report sections follow the `Used` section as usual. The dependency graph comes from +actual Python calls, including nested calls and calls reached through private helpers. Merely +importing a macro does not make it used. Increase the depth to follow longer chains. + +Explain the composed macro to inspect its direct graph and placement facts: + +```console +$ sqb scope model:orders --explain macro:formatted_order_total +Explanation + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:5:1 + Owner: (none) + Owning path: models/commerce + Consumers: model:orders, model:returns + Dependencies: macro:add_tax, macro:round_money + Grants: (none) + Required scope: descendant-public + Required path: models/commerce + Promotion impact: (none) +``` + +The complete command also prints the ordinary report sections above the `Explanation` section. +Use `--json` when another tool needs the same graph and placement facts as structured data. + +Global declarations are an intentional project-wide API and can be numerous. The default `--globals summary` always retains globals used by the target but collapses the unused global inventory with exact counts. Use `--globals used` for only used globals or `--globals all` when a bounded full list is appropriate. + +```bash +# Include every global declaration in the paged report +sqb scope model:stg_orders --globals all + +# Focus on constants and enums defined in finance +sqb scope model:stg_orders \ + --kind constant \ + --kind enum \ + --defined-under models/finance + +# Find visible or used settlement declarations +sqb scope model:stg_orders --match '*settlement*' +sqb scope model:stg_orders --used-only +``` + +## Expected-output access + +Tests and scenarios have two independent ways to reach declarations. Their own folder paths provide +ordinary visibility. For tests, each explicit `__expected__` section adds the eligible +file-based macros, enums, and constants available to that model. Scenarios receive the model's +file-based enums and constants but not its macros. These relationships include eligible +exact-owner-private declarations. + +`sqb scope` reports this additional access separately under **Relationship grants** and names the +model that provides it. With multiple expected models, SQLBuild makes the deterministic union of +their eligible declarations available while compiling the whole test or scenario. It never includes +declarations defined inside a model's `MODEL()` header. A test filename, mirrored path, or mock does +not provide this access by itself. + +For macro-mode tests, **Relationship grants** also reports +`tested_macro through macro:`. This grants the tested macro and the scoped file-based +declarations available from its production owner, as inferred from calls in `__macro_actual__`. + +```bash +sqb scope test:orders__completed_only +sqb scope scenario:daily_revenue --used-only +``` + +## Nearby And Explain + +Nearby discovery is deliberately opt-in and bounded. It considers relevant declarations in the +same authored resource tree, parent folders, close neighboring branches and child folders, and +folders connected through expected-output relationships. It does not dump every private +declaration in the project. + +```bash +# Find declarations just outside the model's effective scope +sqb scope model:stg_orders --include-nearby + +# Include descendants and siblings two directory levels away +sqb scope model:stg_orders --include-nearby --nearby-depth 2 + +# Explain one declaration's visibility and placement at this model +sqb scope model:stg_orders --explain enum:customer_status +``` + +An explanation distinguishes a known but unavailable declaration from an unknown name. It reports +where the declaration is defined, why it is or is not available, which files use it, access through +expected output, required placement, and consumers affected by a placement mismatch. It does not +move or rewrite the declaration. + +`--dependency-depth` is separate from `--nearby-depth`: it follows tracked declaration dependencies from the used section rather than filesystem proximity. + +```bash +sqb scope model:stg_orders --used-only --dependency-depth 2 +``` + +## Prospective Paths + +Use `--at` before creating a resource. A prospective file receives the declarations implied by that exact authored path. A prospective directory describes what a direct child resource would receive. + +```bash +sqb scope --at models/staging/orders/new_model.sql +sqb scope --at tests/unit/staging/orders/ +``` + +The path must be project-relative, below a configured authored resource root, and use the appropriate resource suffix (`.sql`, or `.yml`/`.yaml` for sources). Paths outside those roots produce a diagnostic rather than borrowing scope from a nearby directory. + +Prospective reports are intentionally partial: static path visibility is available, but runtime +usage and expected-output relationships do not exist yet. The report marks those sections +incomplete and exits nonzero while preserving the useful static result. + +## Move Preview + +`--as-path` calculates the visibility delta for moving one existing resource. It reports retained, +gained, and lost declarations; direct usages that the move would invalidate; the new resource-tree +root; declarations private to the resource's owner folder; and expected-output access retained +independently of folder visibility. + +```bash +sqb scope model:stg_orders \ + --as-path models/marts/orders/stg_orders.sql +``` + +The destination must be a valid project-relative file path for that resource kind. This is a pure preview: `sqb scope` never moves files, edits declarations, or changes configuration. + +## Folder Browsing + +Browse and list are separate so a project with 10,000 or more declarations remains safe to explore. `--browse` returns only direct child declaration folders. Each folder has exact recursive declaration, usage, and kind counts plus its direct child-folder count; no arbitrary alphabetical prefix of declarations is printed. + +Global roots appear in the browse namespace as `global/macros`, `global/constants`, and `global/enums`, regardless of how files are organized recursively beneath the top-level declaration roots. + +```bash +# Start with folder summaries only +sqb scope model:stg_orders --browse . + +# Walk down without listing declarations +sqb scope model:stg_orders --browse global +sqb scope model:stg_orders --browse global/macros/finance + +# List declarations only after choosing a bounded domain +sqb scope model:stg_orders --list global/macros/finance/payments +``` + +`--list` is recursive and supports `--kind`, `--match`, `--defined-under`, `--used-only`, `--page-size`, and `--after`. Browse output itself stays folder-first. + +## Pagination + +Flat lists and report sections use qualified declaration identities as stable lexical cursors, not page numbers. In canonical JSON, every paged section reports its total, returned count, completeness, truncation state, and `next_cursor`. Repeat the same command and filters with that identity as `--after`: + +```bash +# First page +sqb scope model:stg_orders \ + --list global/macros/legacy \ + --page-size 50 + +# Continue with the next_cursor from the first result +sqb scope model:stg_orders \ + --list global/macros/legacy \ + --page-size 50 \ + --after macro:legacy_batch_0049 +``` + +Keep all semantic filters unchanged while continuing. Qualified cursors avoid cross-kind ambiguity. A bare cursor is accepted only when it resolves uniquely; invalid or ambiguous cursors produce a diagnostic and nonzero status rather than silently selecting a different page. + +For automation, read each section's `next_cursor` from JSON and continue until it is `null`: + +```bash +sqb scope model:stg_orders \ + --list global/macros/legacy \ + --page-size 50 \ + --json > scope-page.json +``` + +## Paths And JSON + +`--paths relative` shows normalized project-relative paths. `compact` shortens repeated path context +in text output. `none` omits declaration definition locations and replaces structural and header +paths with `(hidden)`. Paths in stable machine output never expose an absolute workspace root. + +`--json` emits the canonical schema rather than serializing the visual tree. The top-level `schema_version` is currently `1`. Reports include the target, scope chain, declaration sections, applied filters, section totals, collapsed and truncated flags, cursors, diagnostics, and aggregate and section-level completeness. Move previews and explanations appear when requested. + +JSON is deterministically ordered, ASCII, newline-terminated, and byte-stable for identical inputs. It contains no ANSI formatting. Consumers should check `schema_version` before relying on fields. + +## Partial Projects + +Scope inspection remains useful while a project is broken. SQLBuild retains valid facts and marks +affected report sections complete or incomplete. Text and JSON never present a partial section as +complete, and diagnostics identify faults that prevented the missing facts from being indexed. + +Diagnostics are stable and include project-relative source locations when available. A partial result, invalid target or cursor, or any error diagnostic produces a nonzero exit status after the available report is written. This lets an editor or agent consume path visibility while still treating incomplete analysis as a failed check. + +## Output Safety + +Scope reports describe declarations without exposing authored values or runtime secrets: + +- Constants show logical type, nullability, collection kind, item count, and rendering mode, not values. +- Enums show scalar type, member count, and a bounded preview of member names, not member values. +- Macros show parameters and tracked declaration dependencies, not source bodies, callables, or source digests. +- Credentials, connection fields, environment variables, warehouse data, and absolute machine paths are not inspected or emitted. + +There is no `--show-values` option. + +## Native SQLBuild Only + +`sqb scope` inspects native SQLBuild authored resources and declarations. It does not discover or emulate dbt models, dbt or Jinja macros, package dispatch, dbt manifests, dbt selectors, dbt tests, or dbt schema YAML visibility. An external dbt graph dependency does not contribute declarations or lexical scope. + +For declaration directory rules, placement checks, test access through expected output, and scoped +macro imports, see [Declarations and Scopes](../concepts/declaration-scopes.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/seed.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/seed.md new file mode 100644 index 0000000000..ae01a8a65c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/seed.md @@ -0,0 +1,34 @@ + + +# seed + +> Load seed CSV files into the warehouse. + +Online: https://docs.sqlbuild.com/cli/seed + +# sqb seed + +Loads seed CSV files into the warehouse as tables. Seeds are fully replaced on every run. + +## Usage + +```bash +sqb --project-dir seed [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--select`, `-s` | Select specific seeds by name | +| `--exclude` | Exclude specific seeds | + +## Examples + +```bash +# Load all seeds +sqb seed + +# Load a specific seed +sqb seed --select seed:waffle_types +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/skills.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/skills.md new file mode 100644 index 0000000000..6ed7315808 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/skills.md @@ -0,0 +1,105 @@ + + +# skills + +> Install SQLBuild skill files for AI coding agents. + +Online: https://docs.sqlbuild.com/cli/skills + +# sqb skills + +Install or update SQLBuild skill files so AI coding agents (Claude Code, OpenCode, Cursor, etc.) understand your project's framework, syntax, and conventions. + +Write the packaged SQLBuild skill file to agent-specific locations in your project. + +```bash +sqb skills [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--target` | Specify agent targets to install for. Can be passed multiple times. | +| `--global` | Install to global agent config directories instead of project-local | +| `--force` | Overwrite existing skill files even if they were not generated by SQLBuild | + +## Targets + +Three agent targets are supported: + +| Target | Local path | Global path | +|--------|-----------|-------------| +| `opencode` | `.opencode/skills/sqlbuild/SKILL.md` | `~/.config/opencode/skills/sqlbuild/SKILL.md` | +| `claude` | `.claude/skills/sqlbuild/SKILL.md` | `~/.claude/skills/sqlbuild/SKILL.md` | +| `agents` | `.agents/skills/sqlbuild/SKILL.md` | `~/.agents/skills/sqlbuild/SKILL.md` | + +By default, SQLBuild installs `agents` and `claude`. OpenCode consumes the shared `.agents` +skill, while Claude requires its `.claude` copy. The `opencode` target remains available as an +explicit override for installations that require `.opencode`: + +```bash +# Install the agents and claude targets (default) +sqb skills + +# Install for Claude Code only +sqb skills --target claude + +# Install for OpenCode and Claude +sqb skills --target opencode --target claude + +# Install globally +sqb skills --global +``` + +## Overwrite behavior + +Generated skill files include an ownership marker. `sqb skills` will: + +- Overwrite files it previously generated (safe to rerun) +- Refuse to overwrite files that were manually created or edited (no marker) +- Overwrite any file when `--force` is passed + +Normal project commands append a non-blocking notice when an installed, SQLBuild-owned generated +skill is stale. The command still runs normally. + +## Configuration + +Default targets can be set in `sqlbuild_project.toml` so the team shares the same agent config: + +```toml +[skills] +targets = ["agents", "claude"] +auto_update = true +``` + +CLI `--target` flags override the TOML config. + +Set `auto_update = true` to refresh stale generated skills during normal project commands. Auto +update reads the skill bundled with the installed SQLBuild package and rewrites only files marked +as SQLBuild-owned. It never overwrites a custom file or an unowned path collision. + +## Playground + +The playground command automatically runs `sqb skills` after creating the project, so AI agents are ready to use immediately: + +```bash +sqb playground waffle-shop +cd waffle-shop +# Agent skill files are already installed +``` + +## Project Rules skills + +`sqb skills` installs general SQLBuild framework guidance. `sqb rules skills` generates +project-specific guidance from the active Rules, options, thresholds, naming vocabulary, and scoped +deviations: + +```bash +sqb rules skills +sqb rules skills --check +``` + +Use `--check` in CI to detect missing or stale Rules guidance without rewriting files. Rules use +the separate `sqlbuild-rules` skill path and refuse to overwrite divergent or unowned content. See +[Compiler-integrated Rules](../concepts/rules.md) and the [Rules CLI reference](rules.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/state.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/state.md new file mode 100644 index 0000000000..f0cfeea979 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/state.md @@ -0,0 +1,128 @@ + + +# state + +> Manage the virtual mode state store. + +Online: https://docs.sqlbuild.com/cli/state + +## Contents + +- Usage +- Subcommands + +# sqb state + +Manages the virtual mode state store lifecycle, locks, and checkpoints. Requires `virtual_environments = true`. + +## Usage + +```bash +sqb state [flags] +``` + +## Subcommands + +### init + +Create state tables in the configured schema. + +```bash +sqb state init +``` + +### migrate + +Back up the current state schema and re-initialize state tables. Creates a backup schema (e.g. `sqlbuild_state__backup_`) before re-initializing. + +```bash +sqb state migrate +``` + +### rollback + +Restore state from a backup. + +```bash +# Restore from the latest backup +sqb state rollback + +# Restore from a specific backup +sqb state rollback --backup-id +``` + +### reset + +Drop all state tables. Requires `allow_reset = true` in the target state config and `--auto-approve` on the command line. + +```bash +sqb state reset --auto-approve +``` + +### adopt + +Convert existing stateless warehouse relations into versioned physical relations with VDE views. Interactive only - requires typed confirmation. + +```bash +sqb state adopt [--allow-copy] +``` + +| Flag | Description | +|------|-------------| +| `--allow-copy` | Allow CTAS copy fallback when native rename is not available | + +See [Adopt and Detach](../concepts/virtual-environments/adopt-detach.md) for details. + +### detach + +Collapse a VDE back into stateless warehouse relations. Interactive only - requires typed confirmation. The VDE must be finalized. + +```bash +sqb state detach [--allow-copy] +``` + +| Flag | Description | +|------|-------------| +| `--allow-copy` | Allow CTAS copy fallback when native rename is not available | + +### locks + +List active locks in the state store. + +```bash +sqb state locks +``` + +### locks clear + +Clear a stuck lock. + +```bash +sqb state locks clear --force +``` + +Only clear locks when you are certain the holding operation is no longer running. + +### checkpoints list + +List finalized checkpoints for a VDE. + +```bash +sqb state checkpoints list [--virtual-env ] +``` + +### checkpoints show + +Show the model refs stored in a checkpoint. + +```bash +sqb state checkpoints show [--virtual-env ] +``` + +### checkpoints diff + +Diff current VDE refs against a checkpoint. + +```bash +sqb state checkpoints diff [--virtual-env ] +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md new file mode 100644 index 0000000000..e8cc148409 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -0,0 +1,44 @@ + + +# test + +> Run SQL unit tests and multi-model tests in isolation. + +Online: https://docs.sqlbuild.com/cli/test + +# sqb test + +Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without +building models. Useful for validating test logic independently. + +## Usage + +```bash +sqb --project-dir test [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--select`, `-s` | Select tests targeting specific models | +| `--exclude` | Exclude tests targeting specific models | + +Parameterized cases use parent-level selection. Selecting a target model includes every case in +each matching `TEST` template; case names are not model selectors. + +Text output identifies each case as ` []` and includes its source path and safe typed +parameters. Structured JSON emits one check per case with stable source/block/case identity, +declared parameter types and nullability, typed values, and a content fingerprint. Exact decimals +are strings in JSON so their scale is preserved. + +## Examples + +```bash +# Run all tests +sqb test + +# Run tests for a specific model +sqb test --select stg_orders +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md new file mode 100644 index 0000000000..5a0c190078 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -0,0 +1,120 @@ + + +# Overview + +> Supported database engines and their connection configuration. + +Online: https://docs.sqlbuild.com/concepts/adapters + +## Contents + +- Custom adapters + +SQLBuild uses adapters to connect to different database engines. + +| Adapter | Status | Install | +|---------|--------|---------| +| [DuckDB](adapters/duckdb.md) | Supported | included by default | +| [MotherDuck](adapters/motherduck.md) | Supported | included by default (uses DuckDB) | +| [Snowflake](adapters/snowflake.md) | Supported | `sqlbuild[snowflake]` | +| [BigQuery](adapters/bigquery.md) | Supported | `sqlbuild[bigquery]` | +| [Databricks](adapters/databricks.md) | Supported | `sqlbuild[databricks]` | +| [PostgreSQL](adapters/postgres.md) | Supported | `sqlbuild[postgres]` | +| [SQL Server](adapters/sqlserver.md) | Supported | `sqlbuild[sqlserver]` | +| ClickHouse | Coming soon | | +| Redshift | Coming soon | | +| Trino | Coming soon | | +| Spark | Coming soon | | +| Athena | Coming soon | | + +Set the adapter in `sqlbuild_project.toml`: + +```toml +name = "my_project" +adapter = "duckdb" +``` + +Or override it per developer in `sqlbuild_local.toml`: + +```toml +adapter = "duckdb" +``` + +## Custom adapters + +You can write your own adapter for any database engine by placing Python files under `adapters/` in your project directory. + +### Extending a built-in adapter + +The most common case is extending an existing adapter with custom behavior. Subclass the built-in adapter, set a new `adapter_name`, and override what you need: + +```python +# adapters/duckdb_plus.py +from sqlbuild.integrations.duckdb.client import DuckDbAdapter + +class DuckDbPlusAdapter(DuckDbAdapter): + adapter_name = "duckdb_plus" + + def connect(self, config): + connection = super().connect(config) + # custom setup - load extensions, set pragmas, etc. + self.execute(connection, "SET enable_progress_bar = true") + return connection +``` + +```toml +adapter = "duckdb_plus" +``` + +### Writing an adapter from scratch + +For a database engine with no built-in support, subclass `BaseAdapter`. It provides ANSI SQL defaults for most methods - you only need to implement `connect`, `execute`, `close`, and any methods where your engine differs from standard SQL: + +```python +# adapters/my_database.py +from sqlbuild.adapter.base.base_adapter import BaseAdapter + +class MyDatabaseAdapter(BaseAdapter): + adapter_name = "my_database" + sql_analysis_dialect_name = "postgres" # SQL dialect for validation and lineage + + def connect(self, config): + ... + + def execute(self, connection, sql): + ... + + def close(self, connection): + ... +``` + +Set `sql_analysis_dialect_name` to the SQL dialect name that matches your engine's SQL syntax. This enables compile-time SQL validation, column inference, column lineage, and local scenario replay for your adapter. If omitted, SQLBuild uses generic SQL parsing. SQL analysis is powered by [Polyglot](https://github.com/tobilg/polyglot), a Rust reimplementation of SQLGlot supporting 32+ dialects. + +For full control with no inherited defaults, subclass `StrictAdapter` instead. Every method is abstract and must be implemented explicitly. SQLBuild raises a clear error listing any unimplemented methods. + +### Discovery rules + +- SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) +- Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) +- Adapter names must be unique across all adapter files - duplicates raise an error +- Custom adapter names cannot shadow built-in names (`duckdb`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) + +### Adapter class hierarchy + +``` +StrictAdapter (fully abstract - all methods must be implemented) + └── BaseAdapter (ANSI SQL defaults - override only what differs) + ├── DuckDbAdapter + ├── SnowflakeAdapter + ├── BigQueryAdapter + ├── DatabricksAdapter + ├── PostgresAdapter + └── SqlServerAdapter +``` + +`StrictAdapter` composes four mixins that define the full adapter contract: + +- **ConnectionMixin** - `connect`, `close`, `begin`, `commit`, `rollback` +- **SchemaMixin** - `relation_exists`, `list_relations`, `describe_relation`, `ensure_schema` +- **MaterializationMixin** - `create_table_as`, `create_view`, `drop`, `rename`, `swap`, `load_seed` +- **DiffMixin** - `diff_schema`, `diff_rows`, `sample_unequal_rows`, `sample_side_only_rows` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md new file mode 100644 index 0000000000..50b5cf7f9e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md @@ -0,0 +1,65 @@ + + +# BigQuery + +> BigQuery adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/bigquery + +BigQuery requires the optional `google-cloud-bigquery` dependency: + +```bash +pip install 'sqlbuild[bigquery]' +# or +uv pip install 'sqlbuild[bigquery]' +``` + +## Connection config + +```toml +adapter = "bigquery" +default_target = "dev" + +[connections.gcp] +project = "my-gcp-project" +location = "europe-west2" + +[targets.dev] +connection = "gcp" +database = "my-gcp-project" +schema = "analytics_dev" +``` + +| Field | Description | +|-------|-------------| +| `project` | GCP project ID (required) | +| `location` | BigQuery dataset location - a region (`europe-west2`, `us-east1`) or multi-region (`US`, `EU`) | +| `credentials_path` | Path to a service account JSON key file. If omitted, uses Application Default Credentials (ADC). | + +## Authentication + +BigQuery supports two authentication methods: + +**Application Default Credentials (default):** Uses the credentials from `gcloud auth application-default login` or the environment's service account. No config needed beyond `project`. + +**Service account file:** + +```toml +[connections.gcp] +project = "my-gcp-project" +credentials_path = "/path/to/service-account.json" +``` + +## Typed constants + +BigQuery renders typed constant arrays with bracket syntax, objects as native JSON expressions, and exact decimals as `NUMERIC` expressions. For example: + +```sql +['GB', 'FR', 'HK'] +JSON '{"GB":"Great Britain"}' +NUMERIC '2.4700' +``` + +BigQuery does not support arrays of arrays. A nested list or set requested with `render_as array` fails at compile time rather than producing invalid BigQuery SQL. Parenthesized `value_list` rendering remains available for ordinary scalar collections used with `IN`. + +See [Collections and Rendering](../constants/collections-and-rendering.md#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md new file mode 100644 index 0000000000..e21871b989 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md @@ -0,0 +1,45 @@ + + +# Databricks + +> Databricks adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/databricks + +Databricks requires the optional `databricks-sql-connector` dependency: + +```bash +pip install 'sqlbuild[databricks]' +# or +uv pip install 'sqlbuild[databricks]' +``` + +## Connection config + +```toml +adapter = "databricks" +default_target = "dev" + +[connections.workspace] +server_hostname = "my-workspace.cloud.databricks.com" +http_path = "/sql/1.0/warehouses/abc123" +token = "dapi_my_access_token" + +[targets.dev] +connection = "workspace" +database = "my_catalog" +schema = "my_schema" +``` + +| Field | Description | +|-------|-------------| +| `server_hostname` | Databricks workspace hostname (required) | +| `http_path` | SQL warehouse or cluster HTTP path (required) | +| `token` | Personal access token (required) | + +The target's `database` selects the Unity Catalog catalog, and its `schema` selects the schema. + +## Session initialization + +On connect, SQLBuild runs `USE CATALOG` and `USE SCHEMA` statements from the active target to set +the authoritative session namespace. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md new file mode 100644 index 0000000000..988f8bddb6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -0,0 +1,62 @@ + + +# DuckDB + +> DuckDB adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/duckdb + +DuckDB is included as a core dependency. No extra installation needed. + +## Connection config + +```toml +adapter = "duckdb" +default_target = "dev" + +[connections.local] +database = "my_project.duckdb" + +[targets.dev] +connection = "local" +schema = "dev" +``` + +| Field | Description | +|-------|-------------| +| `database` | Path to the DuckDB database file. Use `:memory:` for in-memory databases. | +| `extensions` | List of DuckDB extensions to install and load on connect. | +| `settings` | Key-value pairs passed as `SET` statements on connect. | +| `attach` | List of additional databases to attach. | + +## Extensions and settings + +```toml +[connections.local] +database = "my_project.duckdb" +extensions = ["httpfs", "parquet"] + +[connections.local.settings] +memory_limit = "4GB" +``` + +## Attaching additional databases + +```toml +[connections.local] +database = "my_project.duckdb" + +[[connections.local.attach]] +path = "external_data.duckdb" +alias = "external" +read_only = true +``` + +## Table promotion mode + +DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: + +```toml +[settings] +table_promotion_mode = "staged" +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md new file mode 100644 index 0000000000..bc87e4f571 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md @@ -0,0 +1,79 @@ + + +# MotherDuck + +> MotherDuck adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/motherduck + +MotherDuck uses DuckDB's built-in `md:` connection support. No extra installation needed beyond the core DuckDB dependency. + +## Connection config + +```toml +adapter = "motherduck" +default_target = "dev" + +[connections.motherduck] +database = "my_database" +token = "your_motherduck_token" + +[targets.dev] +connection = "motherduck" +database = "my_database" +schema = "dev" +``` + +| Field | Description | +|-------|-------------| +| `database` | MotherDuck database name. Automatically prefixed with `md:` if not already present. Defaults to `md:` (your default MotherDuck database). | +| `token` | MotherDuck access token. Can also be set via environment variable. | + +## Authentication + +MotherDuck requires an access token. Generate one from the MotherDuck UI and pass it via the connection config or an environment variable: + +```toml +[connections.motherduck] +database = "my_database" +token = "${ENV:MOTHERDUCK_TOKEN}" +``` + +## Shared connections across targets + +Use targets to separate production and development databases on MotherDuck: + +```toml +adapter = "motherduck" + +[connections.motherduck] +token = "${ENV:MOTHERDUCK_TOKEN}" +database = "my_database" + +[targets.prod] +connection = "motherduck" +database = "prod_db" +schema = "prod" + +[targets.dev] +connection = "motherduck" +database = "dev_db" +schema = "dev" +``` + +## Local development with DuckDB + +Use `sqlbuild_local.toml` to override the adapter for local development against a plain DuckDB file: + +```toml +adapter = "duckdb" + +[connections.local] +database = "local_dev.duckdb" + +[targets.dev] +connection = "local" +schema = "dev" +``` + +This lets you develop and test locally with zero MotherDuck compute cost, then deploy to MotherDuck in production. SQLBuild's [scenario replay](../scenarios.md) also runs locally in DuckDB regardless of the production adapter. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md new file mode 100644 index 0000000000..6f260044d3 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md @@ -0,0 +1,66 @@ + + +# PostgreSQL + +> PostgreSQL adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/postgres + +PostgreSQL requires the optional `psycopg` dependency: + +```bash +pip install 'sqlbuild[postgres]' +# or +uv pip install 'sqlbuild[postgres]' +``` + +## Connection config + +```toml +adapter = "postgres" +default_target = "dev" + +[connections.postgres] +host = "localhost" +port = 5432 +user = "my_user" +password = "my_password" +dbname = "my_database" + +[targets.dev] +connection = "postgres" +database = "my_database" +schema = "analytics_dev" +``` + +| Field | Description | +|-------|-------------| +| `host` | PostgreSQL server hostname (default: `localhost`) | +| `port` | PostgreSQL server port (default: `5432`) | +| `user` | Database user | +| `password` | Database password | +| `dbname` | Database name | + +Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. + +## Shared connections across targets + +```toml +adapter = "postgres" + +[connections.postgres] +host = "localhost" +user = "my_user" +password = "${ENV:PG_PASSWORD}" +dbname = "analytics" + +[targets.prod] +connection = "postgres" +database = "analytics" +schema = "prod" + +[targets.dev] +connection = "postgres" +database = "analytics" +schema = "dev" +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md new file mode 100644 index 0000000000..5d910b454b --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -0,0 +1,68 @@ + + +# Snowflake + +> Snowflake adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/snowflake + +Snowflake requires the optional `snowflake-connector-python` dependency: + +```bash +pip install 'sqlbuild[snowflake]' +# or +uv pip install 'sqlbuild[snowflake]' +``` + +## Connection config + +```toml +adapter = "snowflake" +default_target = "dev" + +[connections.warehouse] +account = "my_org-my_account" +user = "my_user" +password = "my_password" +role = "TRANSFORM_ROLE" +warehouse = "TRANSFORM_WH" + +[targets.dev] +connection = "warehouse" +database = "ANALYTICS" +schema = "RAW" +``` + +Connection fields are passed directly to `snowflake.connector.connect()`. See the [Snowflake Connector documentation](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect) for all available options, including key-pair authentication, OAuth, and SSO. Put the authoritative database and schema on the target. + +## Session initialization + +On connect, SQLBuild applies the connection's role and warehouse, then applies the active +target's authoritative database and schema. These ensure the session context is set correctly +regardless of the user's default settings. + +## Shared connections across targets + +Use multiple targets to reuse one Snowflake connection while selecting different authoritative +databases and schemas: + +```toml +adapter = "snowflake" + +[connections.warehouse] +account = "my_org-my_account" +user = "my_user" +password = "my_password" +role = "TRANSFORM_ROLE" +warehouse = "TRANSFORM_WH" + +[targets.prod] +connection = "warehouse" +database = "PROD_DB" +schema = "prod" + +[targets.dev] +connection = "warehouse" +database = "DEV_DB" +schema = "dev" +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md new file mode 100644 index 0000000000..f46232bb3a --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md @@ -0,0 +1,80 @@ + + +# SQL Server + +> Microsoft SQL Server adapter configuration for SQLBuild. + +Online: https://docs.sqlbuild.com/concepts/adapters/sqlserver + +SQL Server requires the optional `pymssql` dependency: + +```bash +pip install 'sqlbuild[sqlserver]' +# or +uv pip install 'sqlbuild[sqlserver]' +``` + +## Connection config + +```toml +adapter = "sqlserver" +default_target = "dev" + +[connections.sqlserver] +host = "localhost" +port = 1433 +user = "sa" +password = "my_password" +database = "my_database" + +[targets.dev] +connection = "sqlserver" +database = "my_database" +schema = "analytics_dev" +``` + +| Field | Description | +|-------|-------------| +| `host` | SQL Server hostname (default: `localhost`). Also accepts `server` as an alias. | +| `port` | SQL Server port (default: `1433`) | +| `user` | Database user (default: `sa`). Also accepts `username` as an alias. | +| `password` | Database password | +| `database` | Database name (default: `master`). Also accepts `dbname` as an alias. | + +Connection fields are passed to `pymssql.connect()`. See the [pymssql documentation](https://pymssql.readthedocs.io/en/stable/ref/pymssql.html) for all available options. The target remains authoritative for database and schema qualification. + +SQL Server supports schema-only, full-row, and bounded `sqb diff` comparisons. + +## Typed constants + +SQL Server supports scalar and parenthesized `value_list` constants. Strings use Unicode literals where required, and objects render through a JSON expression such as: + +```sql +JSON_QUERY(N'{"GB":"Great Britain"}') +``` + +SQL Server has no supported first-class native array expression. Any list or set that resolves to `render_as array`, whether through its declaration or `[constants].collection_rendering`, fails at compile time. SQLBuild does not silently substitute a value list or quoted JSON string. + +See [Collections and Rendering](../constants/collections-and-rendering.md#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. + +## Shared connections across targets + +```toml +adapter = "sqlserver" + +[connections.sqlserver] +host = "localhost" +user = "sa" +password = "${ENV:MSSQL_PASSWORD}" +database = "analytics" + +[targets.prod] +connection = "sqlserver" +database = "analytics" +schema = "prod" + +[targets.dev] +connection = "sqlserver" +database = "analytics" +schema = "dev" +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md new file mode 100644 index 0000000000..56175d5e4f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -0,0 +1,316 @@ + + +# Audits + +> Violation and measurement checks that gate data and record quality outcomes. + +Online: https://docs.sqlbuild.com/concepts/audits + +## Contents + +- How audits work +- Measurement audits +- Audit factories +- Result history +- Built-in audits +- Custom generic audits +- Singular audits +- Source audits +- Severity +- Run scope +- Running audits standalone + +Audits are SQL queries that verify data quality. Violation audits return invalid rows. Measurement +audits return a value that SQLBuild evaluates against authored thresholds and sample policy. +SQLBuild can run audits before table promotion or incremental DML so error-severity failures do not +reach the target. + +## How audits work + +Violation audits pass when their query returns zero rows. Measurement audits produce one value and +optionally a sample count; their outcome is `pass`, `warn`, `fail`, or `insufficient`. + +For `error` severity audits: +- **Full table builds:** SQLBuild materializes into a staging table, runs audits against it, and only promotes to the target if all audits pass. If any fail, the staging table is kept for inspection and the production table is untouched. +- **Incremental models:** Delta-phase audits validate each batch before DML is applied. If an audit fails, the batch is not applied. + +For `warn` severity audits, the build continues and the failure is reported in the output. + +## Measurement audits + +A reusable measurement audit separates the aggregate query from optional bounded evidence: + +```sql +-- audits/generic/valid_order_rate.sql +AUDIT ( + evaluation measurement, + value valid_rate, + sample_count total_rows, + sample_unit rows +); + +MEASURE ( + SELECT + COUNT(*) AS total_rows, + 100.0 * AVG(CASE WHEN @condition THEN 1 ELSE 0 END) AS valid_rate + FROM @relation +); + +EVIDENCE ( + SELECT * FROM @relation WHERE NOT (@condition) +); +``` + +Attach threshold and sample policy where the audit is used: + +```sql +MODEL ( + audits [ + valid_order_rate ( + condition "order_id IS NOT NULL", + minimum_samples 100, + evidence_limit 20, + thresholds (warn (below 99.9), error (below 99)) + ) + ] +); +``` + +`minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a +pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and +threshold determine the outcome. + +## Audit factories + +Use a Python audit factory when many related audit instances should be generated from one reviewed +declaration: + +```python +from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory + +@audit_factory +def order_quality(): + return [ + AuditCase( + name="positive_amount", + definition="expression_is_true", + arguments={"expression": "amount > 0"}, + severity=AuditSeverity.ERROR, + ) + ] +``` + +Attach it with `MODEL (audit_factories [order_quality])`. Generated cases compile to the same audit +contract as directly authored instances. + +## Result history + +Native warehouse adapters best-effort append confirmed audit outcomes to +`_sqlbuild_audit_results`. Rows are immutable and use deterministic IDs, so retrying the same result +is idempotent. Projection failure is reported separately and does not change the audit outcome or +command exit code. Lifecycle sinks can also consume the corresponding `audit_completed` fact. + +## Built-in audits + +SQLBuild includes four generic audits out of the box. You do not need to define these in `audits/generic/` - they are available automatically: + +| Audit | Description | Parameters | +|-------|-------------|------------| +| `not_null` | Fails if any row has a NULL value in the column | Column-level only | +| `unique` | Fails if any non-NULL value appears more than once | Column-level only | +| `accepted_values` | Fails if any non-NULL value is not in the allowed list | `values` - list of allowed values | +| `relationships` | Fails if any non-NULL value does not exist in the referenced column | `to` - target relation, `field` - target column | + +### Using built-in audits + +Attach them in the `MODEL()` header like any generic audit: + +```sql +MODEL ( + materialized view, + tags [staging], + columns ( + order_id (audits [not_null, unique]), + customer_id (audits [not_null]), + status ( + audits [ + accepted_values (values ["placed", "preparing", "ready", "completed", "cancelled"]), + ], + ), + payment_method ( + audits [ + relationships (to "stg_payments", field "method"), + ], + ), + ), +); +``` + +### Overriding built-in audits + +If you define a generic audit with the same name as a built-in (e.g. `audits/generic/not_null.sql`), your definition takes precedence. SQLBuild emits a warning so you're aware of the override: + +``` +warning[P003]: project audit 'not_null' overrides built-in audit 'not_null' +``` + +## Custom generic audits + +Beyond the built-ins, you can define reusable SQL templates under `audits/generic/`. They use `@parameter` placeholders that are resolved by the audit engine at compile time. + +```sql +-- audits/generic/expression_is_true.sql +AUDIT (); + +SELECT * +FROM @relation +WHERE NOT (@expression) +``` + +### Audit parameters + +Generic audit SQL uses `@name` for parameter placeholders. These are resolved by the audit engine, not the general SQL interpolation system: + +| Parameter | Description | +|-----------|-------------| +| `@column` | The column name (auto-populated for column-level audits) | +| `@relation` | The target relation (auto-populated from the attached model or source) | +| `@'name'` | A quoted parameter passed from the audit declaration (e.g. `@'values'`) | +| `@name` | An unquoted parameter passed from the audit declaration (e.g. `@expression`) | + +Generic and singular audit SQL uses macros, constants, and enums available from the audit file under +`audits/`, not from a model or source that uses the audit. See +[How Visibility Works](declaration-scopes/visibility.md). + +### Attaching custom generic audits + +```sql +MODEL ( + materialized table, + audits [ + expression_is_true ( + name "revenue_is_non_negative", + expression "total_revenue_cents >= 0", + ), + ], +); +``` + +## Singular audits + +Singular audits are standalone SQL files. Their canonical home is `audits/singular/`, and they +reference models directly. They're useful for one-off checks that don't fit a reusable template. +For backward compatibility, singular audits directly under `audits/` or another non-`generic` +child directory continue to compile. + +```sql +-- audits/singular/orders_have_payments.sql +AUDIT ( + name "completed_orders_have_payments", + severity error +); + +SELECT o.order_id +FROM __ref("fact_orders") o +LEFT JOIN __ref("stg_payments") p ON o.order_id = p.order_id +WHERE p.payment_id IS NULL + AND o.order_status = 'completed' +``` + +SQLBuild automatically infers which model a singular audit attaches to based on the `__ref()` calls in the query. If the audit references a single model, it attaches to that model. If it references multiple models, SQLBuild attaches it to the latest (most downstream) model in the DAG. If attachment can't be inferred, the audit runs at the end of the build. + +## Source audits + +Sources support the same audit system as models. Audits attached to sources run *before* any dependent model is built: + +```yaml +sources: + - name: raw__orders + columns: + - name: id + audits: + - not_null + - unique + audits: + - expression_is_true: + name: no_future_orders + expression: "ordered_at <= CURRENT_TIMESTAMP" +``` + +If a source audit with `error` severity fails, all downstream models that depend on that source are blocked. This lets you catch data quality issues at the source before any transformations run. + +## Severity + +| Severity | Behavior | +|----------|----------| +| `error` | Blocks the build. Staging table is not promoted, DML is not applied. | +| `warn` | Reports a warning but allows the build to continue. | + +Set the default severity in `sqlbuild_project.toml`: + +```toml +[settings] +default_audit_severity = "warn" +``` + +Override per audit instance in the `MODEL()` header: + +```sql +columns ( + order_id (audits [not_null (severity error)]), +), +``` + +## Run scope + +Audits on incremental models can run at different lifecycle phases: + +| Scope | Behavior | +|-------|----------| +| `final` | Run once against the staged table before promotion (default). | +| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete. | + +```sql +MODEL ( + materialized incremental, + ... + columns ( + activity_hour (audits [not_null (run_scope delta_and_final)]), + ), + audits [ + expression_is_true ( + name "orders_placed_is_non_negative", + expression "orders_placed >= 0", + run_scope delta_and_final, + ), + ], +); +``` + +Delta-phase audits with `error` severity block DML before the target is updated. This is visible in the build output as `audit (d)` for delta-phase and `audit (f)` for final-phase: + +``` + 10/13 table hourly_order_activity (delete_insert) OK 0.16s + audit (d) expression_is_true PASS 4/4 + audit (d) not_null (activity_hour) PASS 4/4 + audit (f) expression_is_true PASS + audit (f) not_null (activity_hour) PASS +``` + +The `4/4` indicates the audit passed for all 4 microbatch batches. + +If a model is not incremental, `delta_and_final` degrades to `final` automatically. + +## Running audits standalone + +```bash +sqb audit +``` + +This runs all audits without rebuilding any models. + +Standalone audits run serially unless concurrency is configured explicitly, through +`SQLBUILD_CONCURRENCY`, or in project settings. For example, `sqb audit --concurrency 8` runs up +to eight selected audits at once, using one warehouse connection per active worker. Increase this +limit deliberately because parallel queries can increase warehouse load and cost. See +[`sqb audit`](../cli/audit.md) for precedence, ordering, and cancellation details. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/column-lineage.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/column-lineage.md new file mode 100644 index 0000000000..11d7e2a6a6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/column-lineage.md @@ -0,0 +1,190 @@ + + +# Column Lineage + +> Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. + +Online: https://docs.sqlbuild.com/concepts/column-lineage + +## Contents + +- Why column lineage matters +- How it works +- Transform types +- Confidence levels +- Analysis modes +- Using column lineage +- Limitations + +## Why column lineage matters + +**Impact analysis** - Before changing a source column, see exactly which downstream models and columns are affected. A rename or type change in `raw__orders.id` can be traced through every model that consumes it, even indirectly. + +**Debugging data issues** - When a column has unexpected values, trace it upstream to find where the data originates and what transformations it passes through. Instead of reading SQL files and mentally joining dependencies, ask SQLBuild to show the path. + +**Documentation** - Column lineage provides machine-readable metadata about your pipeline. The JSON output can feed data catalogs, governance tools, or custom dashboards. + +## How it works + +SQLBuild analyzes column lineage statically at compile time. No warehouse connection is needed. The analyzer parses each model's SQL, resolves `ref()` and `source()` calls, and traces columns through `SELECT` lists, CTEs, JOINs, subqueries, and expressions. + +Column lineage requires SQL analysis to be enabled in project settings (it is by default). + +## Transform types + +Each lineage edge is classified by how the column is transformed: + +| Transform | Description | Example | +|-----------|-------------|---------| +| `direct` | Column passes through unchanged | `SELECT order_id FROM ...` | +| `cast` | Column is explicitly cast to a different type | `SELECT CAST(id AS BIGINT)` | +| `expression` | Column is used in a computed expression | `SELECT amount * 100 AS amount_cents` | +| `aggregation` | Column is used inside an aggregate function | `SELECT SUM(amount) AS total` | +| `star` | Column is included via `SELECT *` | `SELECT * FROM ...` | +| `constant` | Output column is a literal value with no upstream dependency | `SELECT 'active' AS status` | + +Transform classification helps you understand the nature of each dependency. A `direct` edge means the column is a simple passthrough - safe to rename if you rename the source. An `expression` or `aggregation` edge means the column is computed - the upstream value is an input to a calculation, not a 1:1 mapping. + +## Confidence levels + +Each edge also carries a confidence level indicating how certain the analyzer is about the traced dependency: + +| Confidence | Meaning | +|------------|---------| +| `high` | The lineage path is fully resolved through known SQL constructs | +| `medium` | The path is likely correct but involves constructs the analyzer handles with heuristics | +| `low` | The path is best-effort - complex SQL patterns or unsupported constructs may reduce accuracy | + +## Analysis modes + +Column lineage supports two analysis modes that trade off speed against depth of analysis. + +**Rich mode** uses the SQL analysis optimizer to resolve columns through CTEs, subqueries, and multi-level nesting with full transform classification. Thorough, but slower because the optimizer runs per column per model. + +**Fast mode** parses the SQL AST directly to extract column mappings, resolve CTE references, and classify transforms. It handles the same SQL patterns that most column lineage tools support and is fast enough to run on every compile. + +`sqb compile` defaults to fast mode because it runs frequently and analyzes the entire project. `sqb lineage` defaults to rich mode because it targets a specific column in a scoped slice of the DAG, where the deeper analysis is worth the cost. Both are overridable: + +```bash +sqb compile --lineage-mode rich +sqb lineage fact_orders.payment_amount_cents --mode fast +``` + +## Using column lineage + +### Interactive tracing with `sqb lineage` + +The target syntax is `model_name.column_name`. Trace a column upstream to see where its values come from: + +```bash +sqb lineage daily_revenue.total_revenue_cents +``` + +``` +Column trace daily_revenue.total_revenue_cents upstream + +└── stg_payments.amount_cents (aggregation) + └── raw__payments.amount_cents (direct) +``` + +Each hop is annotated with how the value was derived. Here `total_revenue_cents` is an aggregation of `stg_payments.amount_cents`, which is a direct passthrough from the source. + +Use `--direction downstream` to trace the other way - every column derived from this one: + +```bash +sqb lineage stg_payments.amount_cents --direction downstream +``` + +``` +Column trace stg_payments.amount_cents downstream + +├── daily_revenue.avg_order_value_cents (aggregation) +├── daily_revenue.total_revenue_cents (aggregation) +├── daily_revenue.total_revenue_dollars (aggregation) +├── dim_customers.lifetime_spend_cents (aggregation) +└── fact_orders.payment_amount_cents (direct) +``` + +Column lineage supports `upstream` (default) and `downstream` directions (not `both` - model lineage supports `both`). + +### Model lineage + +`sqb lineage` also traces model-level dependencies when the target has no column (no dot): + +```bash +sqb lineage fact_orders +``` + +``` +Lineage model fact_orders models/marts/fact_orders.sql upstream + +├── model stg_orders models/staging/stg_orders.sql +│ └── source raw__orders sources/raw.yml +├── model stg_payments models/staging/stg_payments.sql +│ └── source raw__payments sources/raw.yml +├── seed waffle_types seeds/waffle_types.csv +└── udf udf__is_completed_order +``` + +Each node is tagged with its resource type (`model`, `source`, `seed`, `udf`) and file path. Use `--direction both` to show upstream and downstream together: + +```bash +sqb lineage daily_revenue --direction both +``` + +``` +Lineage model daily_revenue models/marts/daily_revenue.sql both + +upstream +├── model stg_orders models/staging/stg_orders.sql +│ └── source raw__orders sources/raw.yml +└── model stg_payments models/staging/stg_payments.sql + └── source raw__payments sources/raw.yml +downstream +``` + +### Options + +| Flag | Description | +| --------------- | --------------------------------------------------------------------------------- | +| `--direction` | `upstream` (default), `downstream`, or `both`. `both` is model lineage only. | +| `--depth` | How many hops to traverse: an integer or `all` (default `all`). | +| `--format` | `tree` (default), `list` (an edge list of `a -> b` pairs), or `json`. | +| `--mode` | Column lineage mode: `rich` (default) or `fast`. | + +See the [lineage CLI reference](../cli/lineage.md) for full flag documentation and output format examples. + +### Batch analysis with `sqb compile` + +Every compile run computes column lineage for analysis and reports a summary: + +```bash +# Default: fast column lineage +sqb compile + +# Skip column lineage +sqb compile --lineage-mode none + +# JSON report includes per-model lineage summary +sqb compile --json +``` + +In the JSON compile report, each model includes a `lineage` field with `column_count`, `edge_count`, and `has_star` metadata. It does not contain the full edge graph; use `sqb lineage [.] --format json` when you need structured lineage details. + +See the [compile CLI reference](../cli/compile.md) for details on the compile report format. + +### Integration with contract validation + +Column lineage feeds into compile-time contract validation. When a model declares columns in its `MODEL()` header, the compiler uses inferred column information to check that: + +- Every declared column exists in the query output +- Column types match the declared types (when `type_enforcement` is enabled) + +These checks run automatically during `sqb compile` and report diagnostics with source-annotated error messages. + +## Limitations + +- Column lineage requires SQL analysis to be enabled (`sql_analysis = true` in settings, which is the default) +- Complex SQL patterns (deeply nested correlated subqueries, dynamic SQL, adapter-specific functions) may reduce accuracy or confidence +- `SELECT *` is tracked as a `star` transform - the analyzer knows the column passes through but the mapping is less precise than explicit column references +- Column lineage is computed statically from SQL text. Runtime-only column additions (e.g. from dynamic UDFs) are not tracked diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants.md new file mode 100644 index 0000000000..0221dddd05 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants.md @@ -0,0 +1,84 @@ + + +# Constants + +> Define reusable compiler-validated values and reference them safely from SQL. + +Online: https://docs.sqlbuild.com/concepts/constants + +Constants give a name to a value used in SQL. SQLBuild validates the value during compilation and +asks the active adapter to render it safely for its SQL dialect. Constant values are data, never raw +SQL snippets. + +## Create a constant + +Put project-wide constants under the top-level `constants/` directory. Files are discovered +recursively, and one file may contain more than one declaration. + +```text +my_project/ +├── constants/ +│ ├── commerce/ +│ │ └── thresholds.sql +│ └── reporting_day.sql +├── models/ +└── sqlbuild_project.toml +``` + +```sql +-- constants/commerce/thresholds.sql +CONSTANT (name min_items, value 7); +CONSTANT (name fallback_source, value "web"); +CONSTANT (name enabled, value true); +CONSTANT (name ratio, value 0.75); +CONSTANT (name missing_value, value null); +``` + +Folders below `constants/` are organizational. They do not change where project-wide constants are +available. + +## Use a constant + +Reference a constant with `@const("name")`: + +```sql +SELECT * +FROM order_batches +WHERE item_count >= @const("min_items") + AND source = @const("fallback_source") +``` + +References work in model queries, SQL hooks, SQL functions, audits, unit tests, scenarios, and +inline source expressions. An unknown constant fails compilation. + +## Scalar values + +Constants support strings, signed integers, booleans, finite floating-point numbers, exact +decimals, and `null`. Integers use a portable signed 64-bit range. `NaN` and positive or negative +infinity are rejected. + +Use `type decimal` with a quoted value when decimal precision must be exact: + +```sql +CONSTANT ( + name usd_rate, + type decimal, + value "2.4700", +); +``` + +SQLBuild parses the quoted value directly as a decimal rather than first converting it to a binary +float. An incompatible `type` and `value` fails compilation. + +## Naming rules + +Public constant names must be unique among public constants and cannot begin with `_`. Enum, +constant, and macro names use separate namespaces, so an enum and a constant may share a name. + +## More constant features + + Define lists, sets, and objects, then choose value-list or native-array rendering. + Keep a constant inside one model when no other resource should use it. + +To limit a constant to one folder, or to that folder and its child folders, see +[Declarations and Scopes](declaration-scopes.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants/collections-and-rendering.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants/collections-and-rendering.md new file mode 100644 index 0000000000..cde2d8aac4 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/constants/collections-and-rendering.md @@ -0,0 +1,163 @@ + + +# Collections and Rendering + +> Define list, set, and object constants and control their adapter-specific SQL rendering. + +Online: https://docs.sqlbuild.com/concepts/constants/collections-and-rendering + +## Contents + +- Lists +- Sets +- Objects +- Collection rules +- Value-list rendering +- Native-array rendering +- Project default +- Stable values + +Constants can hold collections as well as scalar values. Use this page when a constant represents +a reusable value list, native array, set, or structured object. + +## Lists + +Square brackets declare an ordered list. Lists preserve authored order and allow duplicates: + +```sql +CONSTANT ( + name supported_countries, + value ["GB", "FR", "HK"], +); +``` + +## Sets + +Curly braces declare a set. Sets reject duplicate typed values and use a stable order when SQLBuild +renders or fingerprints them: + +```sql +CONSTANT ( + name unique_countries, + value {"GB", "FR", "HK"}, +); +``` + +`{true, 1}` is valid because a boolean and an integer are different logical types. +`{"GB", "FR", "GB"}` fails compilation instead of silently discarding the duplicate. + +## Objects + +Parenthesized key-value entries declare a string-keyed object. Values may be scalars, lists, sets, +or other objects: + +```sql +CONSTANT ( + name country_rules, + value ( + GB ( + label "Great Britain", + threshold 2.47, + enabled true, + regions ["ENG", "SCT", "WLS"], + ), + FR ( + label "France", + threshold 2.5, + enabled true, + regions ["IDF", "NAQ"], + ), + ), +); +``` + +Object keys must be unique. Objects are logical JSON values rather than portable homogeneous SQL +maps or structs. + +## Collection rules + +Lists and sets must be non-empty and have one compatible element type. Nullable elements do not +determine the type, so `[1, null, 2]` is valid, while these declarations fail: + +```sql +CONSTANT (name empty_values, value []); -- no element type +CONSTANT (name unknown_values, value [null]); -- no non-null element type +CONSTANT (name mixed_values, value [1, "two"]); -- incompatible types +``` + +Objects may contain different value types because each key is checked independently. SQLBuild also +applies nesting-depth, element-count, and rendered-size safety limits. + +## Value-list rendering + +Lists and sets render as a parenthesized value list by default. This is designed for `IN`: + +```sql +WHERE country_code IN @const("supported_countries") +``` + +```sql +WHERE country_code IN ('GB', 'FR', 'HK') +``` + +Every element is escaped by the active adapter. + + A value-list constant is intended for a value-list position such as `IN (...)`. It is not a + portable standalone projection. Use native-array rendering when the constant must be an array + expression. + +## Native-array rendering + +Set `render_as array` to request an adapter-native array: + +```sql +CONSTANT ( + name supported_countries_array, + value ["GB", "FR", "HK"], + render_as array, +); +``` + +SQLBuild does not rewrite array membership operations. Use the operators and functions provided by +your adapter. + +Sets support the same `value_list` and `array` modes as lists. Scalar and object constants reject +`render_as` because those rendering modes do not apply to them. + +| Adapter | Native array expression | Object/JSON expression | +|---------|-------------------------|------------------------| +| DuckDB | `['GB', 'FR', 'HK']` | `json('{"GB":"Great Britain"}')` | +| MotherDuck | `['GB', 'FR', 'HK']` | `json('{"GB":"Great Britain"}')` | +| Snowflake | `ARRAY_CONSTRUCT('GB', 'FR', 'HK')` | `PARSE_JSON('{"GB":"Great Britain"}')` | +| BigQuery | `['GB', 'FR', 'HK']` | `JSON '{"GB":"Great Britain"}'` | +| Databricks | `array('GB', 'FR', 'HK')` | `parse_json('{"GB":"Great Britain"}')` | +| PostgreSQL | `ARRAY['GB', 'FR', 'HK']` | `'{"GB":"Great Britain"}'::JSONB` | +| SQL Server | Unsupported | `JSON_QUERY(N'{"GB":"Great Britain"}')` | + +BigQuery does not support arrays containing arrays. SQL Server has no native array representation. +Unsupported requests fail compilation rather than silently changing representation. + +## Project default + +Set the default for list and set constants in `sqlbuild_project.toml`: + +```toml +[constants] +collection_rendering = "array" +``` + +SQLBuild chooses the rendering mode in this order: + +1. The declaration's `render_as` field +2. Project `[constants].collection_rendering` +3. The `value_list` default + +One constant has one representation throughout a compilation. Changing its value or rendering mode +changes the identity of SQL that uses it. + +## Stable values + +- List order and duplicates are preserved. +- Set order is ignored; membership is stored in a stable order. +- Object key order is ignored; keys are stored in a stable order. +- Changing set membership or object values changes dependent query identity. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md new file mode 100644 index 0000000000..d9119a290a --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md @@ -0,0 +1,59 @@ + + +# Adding SQLBuild models + +> Grow into SQLBuild's own models, tests, audits, and scenarios downstream of your dbt project. + +Online: https://docs.sqlbuild.com/concepts/dbt-compatibility/adding-sqlbuild-models + +The dbt compatibility bridge does not require SQLBuild models. You can optionally add SQLBuild models, tests, audits, and scenarios downstream of dbt outputs. + +This is purely additive. Your dbt models stay in dbt, and the layout gains a `models/` directory in the SQLBuild project. + +``` +my-workspace/ + analytics/ # your existing dbt project, untouched + dbt_project.yml + models/ + target/ + manifest.json + sqlbuild_project/ # created by SQLBuild + sqlbuild_project.toml + models/ + marts/downstream_orders.sql # references dbt models via __dbt_ref + tests/ + unit/test_downstream_orders.sql +``` + +## Referencing dbt models + +SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")`: + +```sql +MODEL ( + tags [finance], + columns (order_id (audits [not_null])), +); + +SELECT order_id FROM __dbt_ref("analytics", "fact_orders") +``` + +This resolves to the qualified warehouse table name from the dbt manifest (e.g. `analytics.fact_orders`). The dbt model becomes an upstream dependency in the combined graph, so SQLBuild models downstream of it build against its output. + +SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: + +```sql +MODEL (tags [marts]); + +SELECT order_id FROM __ref("downstream_orders") +``` + +## Tests, audits, and scenarios + +SQLBuild models added downstream of dbt get SQLBuild's full validation surface: + +- **Unit tests** can mock dbt model dependencies with `__dbt_ref__` fixture CTEs, so you can test a SQLBuild model without a warehouse connection or a compiled dbt manifest. See [Testing](../testing.md). +- **Audits** declared inline in the `MODEL()` header block promotion when they fail, the same as in a standalone SQLBuild project. See [Audits](../audits.md). +- **Scenarios** run the combined graph against fixture inputs for end-to-end checks. See [Scenarios](../scenarios.md). + +For the full model authoring reference (materializations, incremental strategies, contracts, and more), see [Models](../models.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md new file mode 100644 index 0000000000..ba4e816ded --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md @@ -0,0 +1,150 @@ + + +# Using SQLBuild with dbt + +> Run dbt and SQLBuild side by side with coordinated selection and SQLBuild models downstream. + +Online: https://docs.sqlbuild.com/concepts/dbt-compatibility/overview + +## Contents + +- Start with your existing dbt project +- How it works +- Configuration +- Prerequisites +- Debugging +- On this topic + +Use the dbt compatibility bridge to coordinate dbt selections with SQLBuild-owned models downstream of their outputs. + +SQLBuild reads the dbt manifest and drives the `dbt` CLI as a subprocess. dbt remains responsible for compiling and executing dbt-owned models; SQLBuild statically analyzes and executes only SQLBuild-owned models downstream. `sqb dbt` runs in direct mode, so change-aware execution and virtual environments are not supported by the bridge. + +## Start with your existing dbt project + +From inside your dbt project, run a `sqb dbt` command. Scope dbt work with familiar `--select` values, or omit selection to plan the whole project. + +```bash +sqb dbt plan --select path:models/marts +``` + +The first time you do this, SQLBuild bootstraps itself. If there is no `sqlbuild_project.toml`, it reads your `dbt_project.yml` and profile and creates a minimal twin project in a `sqlbuild_project/` directory next to your dbt project. It reuses your dbt profile for the warehouse connection, so there are no separate credentials to configure. + +``` +my-workspace/ + analytics/ # your existing dbt project, untouched + dbt_project.yml + models/ + target/ + manifest.json + sqlbuild_project/ # created by SQLBuild + sqlbuild_project.toml # points at the dbt project, no models of its own +``` + +The generated `sqlbuild_project.toml` looks like this: + +```toml +name = "analytics" +adapter = "snowflake" +default_target = "dev" + +[dbt] +project_dir = "../analytics" +profiles_dir = "/Users/you/.dbt" +target_path = "../analytics/target" +target = "dev" + +[connections.dbt_dev] +source = "dbt_profile" +profile = "analytics" +target = "dev" + +[targets.dev] +connection = "dbt_dev" +database = "ANALYTICS" +schema = "analytics" +``` + +`source = "dbt_profile"` on the named connection tells SQLBuild to connect using your dbt +profile, so it talks to the same warehouse dbt does. The target references that connection and +remains authoritative for its database and schema. + +`sqb dbt build --select path:models/marts` compiles the project, resolves the selection, runs the selected dbt models, then runs any SQLBuild models you have added against the dbt outputs. + +## How it works + +1. SQLBuild runs `dbt compile` to produce a `manifest.json` with model metadata +2. SQLBuild reads the manifest to understand dbt model names and their qualified warehouse tables +3. SQLBuild resolves your `--select`/`--exclude` against dbt by running `dbt ls`, so dbt-native selectors like `state:modified` and `package:` are evaluated by dbt itself, not reimplemented +4. `sqb dbt plan/run/build` orchestrates the run: dbt executes the selected dbt work +5. (Optional) any SQLBuild models you have added run last, against the dbt outputs + +Each step calls the `dbt` CLI directly: `dbt compile` for the manifest, `dbt ls` for selection, and `dbt build`/`dbt run` for execution. + +### Flags + +`sqb dbt plan/run/build` declare the common flags directly. Anything declared goes **before** a `--` separator; any other raw dbt flag goes **after** it and is forwarded verbatim. A flag placed on the wrong side errors rather than silently reaching dbt. + +```bash +sqb dbt build --select path:models/marts --full-refresh +``` + +Declared flags are routed to the right place automatically: + +| Flag | Routed to | Notes | +| ---- | --------- | ----- | +| `--select` / `--exclude` | dbt | dbt resolves selection (see [Selection](selection.md)). | +| `--vars` | dbt **and** SQLBuild | The same vars feed dbt's compile and SQLBuild's own variable resolution, so both sides see identical values. | +| `--full-refresh` | dbt **and** SQLBuild | Requests a full rebuild on both sides. dbt and native SQLBuild models independently apply their model-level `full_refresh` setting, including `false` opt-outs. | +| `--threads` | dbt (`--threads`) and SQLBuild (`--concurrency`) | | +| `--target` / `--project-dir` / `--profiles-dir` / `--profile` / `--target-path` | dbt | Standard dbt locators. | + +#### `--vars` + +`--vars` accepts the same JSON object dbt accepts, and SQLBuild passes it to **both** the underlying `dbt` invocation and its own variable resolution. That means a value referenced as `@@my_var` in SQLBuild model SQL (see [Interpolation](../interpolation.md)) and as `{{ var('my_var') }}` in a dbt model both resolve to the value you passed. CLI vars take precedence over project and local config vars. You do not need to declare vars twice. + +```bash +sqb dbt build --select path:models/marts --vars '{"my_var": 1}' +``` + +#### Forwarding other dbt flags + +For any native dbt flag SQLBuild does not declare, put it after `--` and it is passed straight to the `dbt` invocation untouched: + +```bash +sqb dbt build -- --log-level debug +``` + +## Configuration + +The auto-generated project above is editable, and you can write `sqlbuild_project.toml` by hand. The `[dbt]` block (shown in the generated project above) accepts: + +| Field | Description | +|-------|-------------| +| `project_dir` | Path to the dbt project root (where `dbt_project.yml` lives) | +| `profiles_dir` | Path to the directory containing `profiles.yml` | +| `target_path` | Path to dbt's `target/` directory (where `manifest.json` is written) | +| `target` | dbt target name override (optional) | + +Paths can be absolute or relative to the SQLBuild project root. + +## Prerequisites + +- dbt must be installed and available on `PATH` as `dbt` +- Both projects must target the same warehouse and schema/database context + +SQLBuild uses your own `dbt` install; it does not bundle or install dbt. If your `dbt` is not reachable as a bare `dbt` on `PATH` (for example, you run it via `uv`, `poetry`, or a wrapper), set the `DBT_EXECUTABLE` environment variable to the executable SQLBuild should call. + +SQLBuild runs `dbt compile` automatically as part of `sqb dbt plan/run/build` to produce the manifest. You do not need to compile the dbt project manually. + +## Debugging + +`sqb dbt debug` runs both projects' diagnostics: `dbt debug` (verifying the dbt project config and warehouse connection) followed by `sqb debug` (verifying the SQLBuild project config and connection). + +```bash +sqb dbt debug +``` + +## On this topic + +- [Selection](selection.md) - how `--select` and `--exclude` route work across both graphs. +- [Adding SQLBuild models](adding-sqlbuild-models.md) - optionally write SQLBuild models, tests, audits, and scenarios downstream of dbt. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/selection.md new file mode 100644 index 0000000000..5fb93f5448 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/selection.md @@ -0,0 +1,75 @@ + + +# Selection + +> How select and exclude flags route work across the dbt and SQLBuild graphs. + +Online: https://docs.sqlbuild.com/concepts/dbt-compatibility/selection + +The `sqb dbt` commands use `--select` and `--exclude` to scope what runs. Selectors work across both dbt and SQLBuild, with the system determining which side owns each selector and how to route work. + +## SQLBuild-recognized selectors + +These selectors match SQLBuild models directly: + +| Selector | Example | Behavior | +|----------|---------|----------| +| Model name | `fact_orders` | Selects that SQLBuild model. Auto-includes its immediate dbt upstream dependencies. | +| Leading `+` | `+fact_orders` | Selects the model plus walks upstream through both SQLBuild and dbt models. | +| Trailing `+` | `fact_orders+` | Selects the model plus all downstream SQLBuild models. | +| Both `+` | `+fact_orders+` | Full upstream (including dbt) and downstream expansion. | +| Tag | `tag:nightly` | Selects all SQLBuild models with that tag. Auto-includes dbt dependencies. | +| Tag with `+` | `+tag:nightly` | Tag match plus upstream expansion through the combined graph. | +| Path | `path:models/marts` | Selects SQLBuild models under that project-relative directory - the same syntax as dbt's `path:` selector. | +| Path with `+` | `+path:models/marts` | Path match plus upstream/downstream expansion. | + +When a SQLBuild model is selected, its immediate dbt upstream dependencies are always included so dbt can build the tables that SQLBuild models read from. + +## dbt-only selectors + +Selectors that SQLBuild does not recognize (like `state:modified`, `package:stripe`, `source:stripe.charges`) are passed to `dbt ls` to resolve: + +| Selector | Example | Behavior | +|----------|---------|----------| +| Without `+` | `state:modified` | dbt-only work. No SQLBuild models selected. | +| With trailing `+` | `state:modified+` | SQLBuild runs `dbt ls` to find which dbt models match, then walks downstream into SQLBuild territory. | +| With both `+` | `+state:modified+` | Same downstream expansion, plus upstream dbt expansion. | + +This means you can use dbt-native selectors like `state:modified+` to trigger rebuilds of SQLBuild models that depend on changed dbt models. If `dbt ls` returns no matching models, no SQLBuild work is triggered. + +## Exclude + +`--exclude` removes matching SQLBuild models from the final selection: + +```bash +sqb dbt build --select fact_orders+ --exclude tag:nightly +``` + +## Examples + +```bash +# Build a specific SQLBuild model and its dbt dependencies +sqb dbt build --select downstream_orders + +# Build everything downstream of a SQLBuild model +sqb dbt build --select downstream_orders+ + +# Build a SQLBuild model with full upstream dbt chain +sqb dbt build --select +downstream_orders + +# Build SQLBuild models downstream of modified dbt models +sqb dbt build --select state:modified+ + +# Build all SQLBuild models tagged "nightly" with their dbt dependencies +sqb dbt build --select tag:nightly + +# Build SQLBuild models under a path +sqb dbt build --select path:models/marts +``` + +## Execution order + +For `sqb dbt run` and `sqb dbt build`: + +1. **dbt runs** - a single `dbt run/build` command executes with the user's selectors merged with any additional dbt models required by selected SQLBuild models +2. **SQLBuild runs** - selected SQLBuild models execute against the now-built dbt tables diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes.md new file mode 100644 index 0000000000..5684b65381 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes.md @@ -0,0 +1,108 @@ + + +# Overview + +> Limit enums, constants, and macros to the parts of a project that use them. + +Online: https://docs.sqlbuild.com/concepts/declaration-scopes + +Most projects can keep enums, constants, and macros in their ordinary top-level directories. Those +declarations are available throughout the project. + +When an enum, constant, or macro should be available only within one folder or its child folders, +you can keep it near the SQL that uses it. SQLBuild uses the declaration directory's location to +decide which files can access it. No TOML configuration is required. + +| Where the declaration lives | Who can use it | +|-------------------------------------|--------------------| +| Top-level `macros/`, `constants/`, or `enums/` | The whole project | +| Nested `_sqlbuild/macros/`, `_sqlbuild/constants/`, or `_sqlbuild/enums/` | Files in the owner folder and folders below it | +| Nested `_sqlbuild/_macros/`, `_sqlbuild/_constants/`, or `_sqlbuild/_enums/` | Files directly in the owner folder only | +| An underscored constant or enum inside `MODEL()` | That model and its inline SQL hooks only | + +Macro declarations are Python files (`.py`). Constant and enum declarations are SQL files +(`.sql`). + +## Example + +The annotation beside each directory shows where its contents are available: + +```text +models/ +├── constants/ published throughout models/ +│ └── warehouse.sql +└── commerce/ + ├── _sqlbuild/ + │ ├── macros/ published throughout commerce/ + │ │ └── currency.py + │ └── _enums/ exact commerce/ directory only + │ └── grain.sql + ├── orders.sql sees warehouse, currency, and grain + ├── finance/ + │ ├── _sqlbuild/ + │ │ └── macros/ published throughout finance/ + │ │ └── tax.py + │ └── revenue.sql sees warehouse, currency, and tax + └── fulfillment/ + └── shipments.sql sees warehouse and currency +``` + +The folder containing `_sqlbuild/` is the declaration owner. An unprefixed role applies to that +owner and everything below it. An underscored role applies only to files directly in the owner. +Legacy declaration roles directly below an owner remain supported. + +`_sqlbuild/` is reserved for the six declaration-role directories shown above. Other direct files or +folders are rejected. It must also sit below a concrete owner directory. A project-root +`_sqlbuild/` or authored-root path such as `models/_sqlbuild/` is invalid; declarations at that +boundary belong in the project-wide `macros/`, `enums/`, or `constants/` roots. + +| Resource | Visible from the example tree | Not visible | +|----------|-------------------------------|-------------| +| `orders.sql` | Warehouse constant, commerce macro, commerce-folder-only enum | Finance macro | +| `finance/revenue.sql` | Warehouse constant, commerce macro, finance macro | Commerce-folder-only enum | +| `fulfillment/shipments.sql` | Warehouse constant, commerce macro | Commerce-folder-only enum, finance macro | + +Declarations do not flow upward or sideways into sibling directories. + +## Terms used in text output + +The CLI's text output uses short labels for the same four rules: + +| Canonical label | Plain-language meaning | +|-----------------|------------------------| +| `project` | Available throughout the project | +| `descendant-public` | Available in this folder and folders below it | +| `exact-owner-private` | Available directly in this folder only | +| `model-private` | Available to one model only | + +Here, an **owner folder** is simply the folder directly containing a model, test, hook, function, or +other authored resource. The [Scope Explorer](declaration-scopes/explorer.md) shows the owner +folder, its parents, and project-wide declarations separately. + +JSON keeps the underlying enum values for integrations. Its `scope` field uses `global`, +`inherited`, `local`, or `private`, while derived `visibility` values use underscores, such as +`descendant_public` and `exact_owner_private`. Consumers should use the fields documented in the +versioned JSON schema rather than parsing text labels. + +## Choose a location + +| Where the value is needed | Placement | +|---------------------------|-----------| +| One model only | In that model's `MODEL()` header | +| Files directly in one folder | In an underscored role under the owner's `_sqlbuild/` folder | +| Files across one folder tree | In an unprefixed role under the nearest shared owner's `_sqlbuild/` folder | +| Different resource trees, such as models and tests | In a top-level declaration role | + +SQLBuild computes the lowest common owner of every declaration's runtime consumers. A project-wide +declaration is valid only when no narrower supported owner contains all consumers. This keeps the +top-level roles as a genuine project API instead of a neutral dumping ground. + +## Explore the feature + + See which declarations a model, test, hook, or function can use. + Choose the narrowest folder that contains every real use. + Ask SQLBuild what a file can use and preview how moving it would change that answer. + +Learn the features themselves in [Enums](enums.md), [Constants](constants.md), and +[Writing Macros](macros.md). See [Interpolation](interpolation.md) for project, +environment, and runtime context values, which are separate from declarations. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/explorer.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/explorer.md new file mode 100644 index 0000000000..bdcd7c657f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/explorer.md @@ -0,0 +1,345 @@ + + +# Scope Explorer + +> Inspect visibility, explain resolution, browse declarations, and preview moves offline. + +Online: https://docs.sqlbuild.com/concepts/declaration-scopes/explorer + +## Contents + +- Example project +- See available and used declarations +- Follow composed dependencies +- Find declarations outside the scope +- See declarations available through expected output +- Preview a resource move +- Check a prospective path +- Browse large declaration sets +- Automation + +Scope Explorer answers questions about declaration visibility through `sqb scope`. It is read-only +and offline: it never connects to the warehouse, moves files, or edits configuration. + +Use this page for common workflows. See the [`sqb scope` CLI reference](../../cli/scope.md) for every flag, +filter, text section, and JSON field. + +## Example project + +The examples below use two domains with project-wide currency helpers: + +```text +project/ +├── macros/ +│ └── currency.py add_tax, round_money +├── models/ +│ ├── commerce/ +│ │ ├── constants/limits.sql minimum_order_value +│ │ ├── enums/status.sql order_status +│ │ ├── macros/orders.py formatted_order_total +│ │ ├── orders.sql +│ │ └── returns/returns.sql +│ └── finance/ +│ ├── enums/status.sql finance_status +│ ├── macros/margin.py calculate_margin +│ ├── finance_summary.sql +│ └── reports/margin_report.sql +└── tests/ + └── unit/orders__completed_only.sql +``` + +`formatted_order_total` composes the project-wide functions through ordinary Python calls: + +```python +from macros.currency import add_tax, round_money + +def formatted_order_total(expression: str) -> str: + return round_money(add_tax(expression)) +``` + +`orders.sql` uses the commerce constant and macro. The returns model also uses all three commerce +declarations, so they are correctly published throughout that domain. The two finance models use +both finance declarations. This gives every scoped declaration a real consumer and valid placement. + +## See available and used declarations + +Start with a model, test, scenario, hook, function, audit, source, or authored resource path to +inspect its directory-derived scope. A declaration identity instead opens its explanation. This +excerpt of the main report sections puts actual usage beside the complete directory-derived scope: + +```console +$ sqb scope model:orders +Scope + Target: model:orders + Resource: model:orders + Path: models/commerce/orders.sql + +Used (2) + ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + +Scope chain + ├─ exact-owner-private models/commerce (3) + ├─ descendant-public models (0) + └─ project global (2) + +Available (3 of 5, 2 collapsed) + ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 + ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + … 2 globals collapsed; run sqb scope model:orders --globals all + +Diagnostics (0) + (none) +Completeness: complete +``` + +`●` marks a declaration included in `Used`: a direct usage by default, or a followed declaration +dependency when `--dependency-depth` is nonzero. `○` marks a declaration present in the section but +not in `Used`. + +The compact labels translate to ordinary folder rules: + +| Output label | Meaning in this report | +|--------------|------------------------| +| `project` | Available throughout the project | +| `descendant-public` | Available from a folder at or above this resource | +| `exact-owner-private` | Available directly in this resource's owner folder only | +| `inherited_ancestor` | Visible because an unprefixed role publishes it down the folder tree | +| `role models/commerce/macros` | The declaration role begins at this path | + +In the scope chain, the first path is the resource's owner folder, followed by parent folders and +the project declaration roles. The counts show declarations defined at each path; the chain label +describes how SQLBuild reaches that path, not the visibility of every declaration counted there. +Unused project-wide declarations stay collapsed by default so a large project API does not hide the +owner-folder facts. + +## Follow composed dependencies + +`Used` is direct by default. Add `--dependency-depth` to follow declarations used by those +declarations. The expanded `Used` section is: + +```console +$ sqb scope model:orders --used-only --dependency-depth 1 +Used (4) + ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 + ├─ ● macro:add_tax [macro; project; dependency; params 1; role macros] macros/currency.py:1:1 + ├─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + └─ ● macro:round_money [macro; project; dependency; params 1; role macros] macros/currency.py:5:1 +``` + +SQLBuild derives these edges from actual Python calls, including nested calls and calls reached +through private helpers. Merely importing a macro does not make it used. Increase the depth to +follow longer chains. + +Explain the composed macro to see its direct dependencies, consumers, and required placement. The +ordinary report appears first; its `Explanation` section is: + +```console +$ sqb scope model:orders --explain macro:formatted_order_total +Explanation + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + Owner: (none) + Owning path: models/commerce + Consumers: model:orders, model:returns + Dependencies: macro:add_tax, macro:round_money + Grants: (none) + Required scope: descendant-public + Required path: models/commerce + Promotion impact: (none) +``` + +### Catch placement that is too broad + +If `returns.sql` stopped using `formatted_order_total`, only `orders.sql` would consume it. The same +explanation would then show that descendant-public placement is broader than necessary: + +```console +Explanation + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + Owner: (none) + Owning path: models/commerce + Consumers: model:orders + Dependencies: macro:add_tax, macro:round_money + Grants: (none) + Required scope: exact-owner-private + Required path: models/commerce + Promotion impact: model:orders + +Diagnostics (1) + ERROR S008 models/commerce/macros/orders.py: Declaration 'macro:formatted_order_total' is currently descendant-public at 'models/commerce' (models/commerce/macros/orders.py); required exact-owner-private at 'models/commerce'. Consumers: model:orders. Move it to 'models/commerce/_sqlbuild/_macros/' +Completeness: complete +``` + +This is actionable placement guidance, not only a visibility lookup. + +## Find declarations outside the scope + +Nearby discovery is useful when you know a declaration exists but not why the current resource +cannot use it. The relevant section is: + +```console +$ sqb scope model:orders --include-nearby +Nearby unavailable (2 of 2) + ├─ ○ enum:finance_status [enum; descendant-public; sibling_scope; members 3; type VARCHAR; role models/finance/enums] models/finance/enums/status.sql:1:1 + └─ ○ macro:calculate_margin [macro; descendant-public; sibling_scope; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 +``` + +The declarations are known, but they belong to a different folder branch. Here, the inspected +resource belongs to `models/commerce`, while the declarations belong to `models/finance`. +`sibling_scope` is the compact output label for that situation. Ask for one complete explanation +when you know a declaration's identity. After the ordinary report, the command adds: + +```console +$ sqb scope model:orders --explain macro:calculate_margin +Explanation + └─ ○ macro:calculate_margin [macro; descendant-public; sibling_scope; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 + Owner: (none) + Owning path: models/finance + Consumers: model:finance_summary, model:margin_report + Dependencies: (none) + Grants: (none) + Required scope: descendant-public + Required path: models/finance + Promotion impact: (none) +``` + +An explanation distinguishes a known but inaccessible declaration from an unknown name. It also +shows whether the declaration is placed more broadly than its real consumers require. + +## See declarations available through expected output + +Tests can use file-based macros, enums, and constants available to a model when they define that +model's expected output. Scenarios receive the model's file-based enums and constants but not its +macros. Neither receives declarations inside `MODEL()`. The access applies to the whole test or +scenario and remains separate from declarations visible through its own folder tree. The relevant +report sections are: + +```console +$ sqb scope test:orders__completed_only --used-only +Used (1) + └─ ● enum:order_status [enum; descendant-public; expected_model through model:orders; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 + +Relationship grants (1 of 1) + └─ ● enum:order_status [enum; descendant-public; expected_model through model:orders; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 +``` + +The compact reason `expected_model through model:orders` means **available through expected output +for model `orders`**. With multiple expected models, SQLBuild combines their eligible file-based +declarations into one deterministic set. Tests include macros in that union; scenarios include only +enums and constants. Neither receives declarations defined inside a model's `MODEL()` header. + +Macro-mode tests also report `tested_macro through macro:` relationships. Those grants contain +the tested macro and scoped file-based declarations available from its production owner. + +## Preview a resource move + +Use `--as-path` to see the scope delta before moving an existing model, test, hook, or function. The +ordinary report is followed by: + +```console +$ sqb scope model:orders --as-path models/finance/orders.sql +Move preview + Resource: model:orders + Destination: models/finance/orders.sql + Ownership root: models + Retained (2) + ├─ ○ macro:add_tax [macro; project; global; params 1; role macros] macros/currency.py:1:1 + └─ ○ macro:round_money [macro; project; global; params 1; role macros] macros/currency.py:5:1 + Gained (2) + ├─ ○ enum:finance_status [enum; descendant-public; inherited_ancestor; members 3; type VARCHAR; role models/finance/enums] models/finance/enums/status.sql:1:1 + └─ ○ macro:calculate_margin [macro; descendant-public; inherited_ancestor; params 2; role models/finance/macros] models/finance/macros/margin.py:4:1 + Lost (3) + ├─ ● constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 + ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 + └─ ● macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + Private retained (0) + (none) + Relationship retained (0) + (none) + Invalidated usages (2) + - constant:minimum_order_value + - macro:formatted_order_total +``` + +The move would gain finance declarations and lose commerce declarations. More importantly, +`Invalidated usages` separates the losses that break the model from declarations that happened to +be available but were never used. The preview does not move or rewrite any file. + +## Check a prospective path + +Inspect visibility before a resource exists: + +```console +$ sqb scope --at models/commerce/returns/new_return.sql +Scope + Target: models/commerce/returns/new_return.sql + Path: models/commerce/returns/new_return.sql + Status: prospective + +Used (0) + (none) + +Scope chain + ├─ exact-owner-private models/commerce/returns (0) + ├─ descendant-public models/commerce (3) + ├─ descendant-public models (0) + └─ project global (2) + +Available (3 of 5, 2 collapsed) + ├─ ○ constant:minimum_order_value [constant; descendant-public; inherited_ancestor; type integer; role models/commerce/constants] models/commerce/constants/limits.sql:1:1 + ├─ ○ enum:order_status [enum; descendant-public; inherited_ancestor; members 4; type VARCHAR; role models/commerce/enums] models/commerce/enums/status.sql:1:1 + └─ ○ macro:formatted_order_total [macro; descendant-public; inherited_ancestor; params 1; role models/commerce/macros] models/commerce/macros/orders.py:4:1 + … 2 globals collapsed; run sqb scope --at models/commerce/returns/new_return.sql --globals all + +Relationship grants (0 of 0) + (none) + +Nearby unavailable (0 of 0) + (none) + +Diagnostics (1) + ERROR S013 models/commerce/returns/new_return.sql: Runtime usage and relationship facts are unavailable for a prospective path +Completeness: partial +``` + +Visibility is available from the proposed path. Actual usage and expected-output relationships do +not exist yet, so the command preserves the useful static result while clearly marking it partial. + +## Browse large declaration sets + +In a larger project, browse returns folder summaries rather than an arbitrary prefix of the +declaration inventory. The `used` counts are direct usages by this target; browse does not apply +dependency-depth expansion: + +```console +$ sqb scope model:orders --browse global +Scope folders + Path: global + + ├─ constants/ 146 declarations, 18 used, 4 children; constant 146 + sqb scope model:orders --browse global/constants + sqb scope model:orders --list global/constants + ├─ enums/ 84 declarations, 11 used, 3 children; enum 84 + sqb scope model:orders --browse global/enums + sqb scope model:orders --list global/enums + └─ macros/ 231 declarations, 27 used, 8 children; macro 231 + sqb scope model:orders --browse global/macros + sqb scope model:orders --list global/macros + +Diagnostics (0) + (none) +Completeness: complete +``` + +Choose a bounded folder, then use `--list`. Definition-path, kind, glob, and actual-usage filters +can be combined, and paged sections use stable declaration identities as cursors. + +## Automation + +Use `--json` for editor integrations and repository tooling. Output has a versioned schema, stable +ordering, filters, pagination, and move-preview results. Constant values, credentials, and secret +connection settings are not included. + + Review the directory rules behind the report. + See all selectors, filters, pagination options, output sections, and JSON behavior. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/placement.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/placement.md new file mode 100644 index 0000000000..237daf3c38 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/placement.md @@ -0,0 +1,96 @@ + + +# Where to Put Declarations + +> Choose the narrowest folder that contains every real use. + +Online: https://docs.sqlbuild.com/concepts/declaration-scopes/placement + +Start with the ordinary project-wide directories unless you have a reason to limit access: + +```text +macros/ +enums/ +constants/ +``` + +Move a declaration closer to its users when it should be available only in one folder or one folder +tree. + +## Choose a location + +| Where it is needed | Location | +|--------------------|----------| +| One model only | Inside that model's `MODEL()` header, for enums and constants | +| SQL files directly in one directory | `_sqlbuild/_macros/`, `_sqlbuild/_enums/`, or `_sqlbuild/_constants/` | +| A directory and its descendants | `_sqlbuild/macros/`, `_sqlbuild/enums/`, or `_sqlbuild/constants/` | +| Different trees, such as models and tests | Top-level `macros/`, `enums/`, or `constants/` | + +## One directory + +These two models use the same constant and both sit directly in `commerce/`: + +```text +models/ +└── commerce/ + ├── _sqlbuild/ + │ └── _constants/ + │ └── minimum_value.sql + ├── orders.sql + └── customers.sql +``` + +Use `_sqlbuild/_constants/` because no descendant directory needs the value. + +## One directory tree + +These models use the same constant across two child directories: + +```text +models/ +└── commerce/ + ├── _sqlbuild/ + │ └── constants/ + │ └── reporting_day.sql + ├── finance/ + │ └── revenue.sql + └── fulfillment/ + └── shipments.sql +``` + +Use `_sqlbuild/constants/` in `commerce/` so both child directories can use it. + +## Different resource trees + +When a declaration is used directly from unrelated trees, make it project-wide: + +```text +my_project/ +├── constants/ +│ └── order_status.sql +├── models/ +│ └── commerce/orders.sql +└── tests/ + └── scenarios/order_lifecycle.sql +``` + +Top-level placement is valid when consumers genuinely cross resource trees or otherwise have no +shared owner folder. SQLBuild applies the same nearest-shared-folder analysis to project-wide +declarations, so a declaration used only under one narrower folder must move closer to those users. + +## What SQLBuild checks + +Every declaration must be used by real compiled SQL. A name appearing only in a comment, quoted +string, mock identity, or documentation does not count as use. + +For every declaration, including project-wide declarations, SQLBuild finds the nearest shared owner +folder of the files that use it. If the declaration is in a broader location, the error shows: + +- Where the declaration is now +- Which files use it +- Which directory form is required +- The destination directory + +These checks use the complete project rather than only the models selected by the current command. + + Use Scope Explorer to see what a file can access or preview moving the file. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md new file mode 100644 index 0000000000..3ce53b7c77 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md @@ -0,0 +1,187 @@ + + +# How Visibility Works + +> See which enums, constants, and macros are available to each SQL file. + +Online: https://docs.sqlbuild.com/concepts/declaration-scopes/visibility + +## Contents + +- Start from the SQL file +- Supported resource trees +- Which file controls visibility? +- Tests and expected output +- Macros importing macros +- Names do not shadow + +For most SQL, the rule is simple: + +> A file can use declarations available to the whole project, declarations in unprefixed roles +> under `_sqlbuild/` folders owned beside or above it, and declarations in an underscored role owned +> directly beside it. + +## Start from the SQL file + +SQLBuild starts from the file containing the SQL and walks up that file's directory tree. The folder +directly containing the file is its **owner folder**. + +```text +models/ +├── constants/ available throughout models/ +│ └── warehouse.sql +└── commerce/ + ├── _sqlbuild/ + │ ├── enums/ available throughout commerce/ + │ │ └── order_status.sql + │ └── _constants/ available directly in commerce/ only + │ └── minimum_value.sql + ├── orders.sql + └── history/ + └── archived_orders.sql +``` + +From this tree: + +| File | Can use | +|------|---------| +| `orders.sql` | `warehouse`, `order_status`, and `minimum_value` | +| `history/archived_orders.sql` | `warehouse` and `order_status` | + +`minimum_value` is not available in `history/` because `_sqlbuild/_constants/` applies only to files +directly in the `_sqlbuild/` owner's folder. + +## Supported resource trees + +Grouped declaration directories can be placed below these SQL resource roots: + +| Root | Contents | +|------|----------| +| `models/` | Models and inline model hooks | +| `tests/unit/` | Unit tests | +| `tests/scenarios/` | Scenarios | +| `hooks/sql/` | Named SQL hooks | +| `functions/sql/` | SQL functions | +| `audits/` | Audits | +| `sources/` | Inline source expressions | + +Each root is a separate tree. For example, `models/constants/` does not make declarations available +under `tests/`. Put a declaration in the top-level `constants/`, `enums/`, or `macros/` directory +when it must be available across different resource trees. Project-root `_constants/`, `_enums/`, +and `_macros/` directories are invalid because there is no owner folder at the project root for +them to be private to. + +Existing scoped declaration roles directly below an owner remain supported. `_sqlbuild/` is the +preferred layout because it keeps all declarations together without changing their visibility. +Only the six public/private declaration-role directories are valid directly under `_sqlbuild/`; +other entries are rejected. `_sqlbuild/` must sit below a concrete owner directory. A project-root +`_sqlbuild/` or authored-root path such as `models/_sqlbuild/` is invalid; use the project-wide +`macros/`, `enums/`, or `constants/` roots at that boundary. + +## Which file controls visibility? + +| SQL being compiled | SQLBuild starts from | +|--------------------|----------------------| +| Model query | The model file | +| Inline SQL hook in a model | The model file | +| Unit test SQL | The test file, plus inferred tested-resource relationships | +| Scenario SQL | The scenario file, plus expected-model enum and constant relationships | +| Named SQL hook | The hook file under `hooks/sql/` | +| SQL function | The function file under `functions/sql/` | +| Audit | The audit file | +| Inline source expression | The source definition | + +This means a reusable named hook does not change meaning depending on which model calls it. The +hook uses declarations available where the hook itself is stored. + +## Tests and expected output + +A test first sees declarations available from its own folder tree. It may also use file-based +macros, enums, and constants available to a model for which it defines expected output. + +```sql +TEST(); + +WITH +__expected__orders AS ( + SELECT + 1 AS order_id, + @normalize_order_status("'completed'") AS normalized_status, + @enum("order_status").COMPLETED AS status +) +SELECT 1 +``` + +Because the test defines `__expected__orders`, the test may use file-based macros, enums, and +constants available to `orders`, including declarations in exact-owner-private roles. +Scope Explorer describes this as **available through expected output for model `orders`**. This is +an additional relationship, not another visibility level. + +This additional access does **not** include: + +- Enums or constants declared privately inside the model's `MODEL()` header +- Declarations from a model merely mentioned by filename or directory layout + +Only an explicit `__expected__model_name` section adds the model's eligible file-based declarations. +When a test checks several models, SQLBuild combines the declarations available through all +expected models and makes that deterministic union available while compiling the entire test. + +Macro-mode tests receive the tested macro and the file-based declarations available from that +macro's production owner. SQLBuild infers tested macros from calls in `__macro_actual__`; the test's +filename or directory does not grant production visibility. + +```text +models/orders/_sqlbuild/_macros/order_policy.py +tests/unit/macros/models/orders/test_order_policy__returns_quantity.sql +``` + +The macro test can call `order_policy` without promoting it to the project-wide `macros/` root. A +test can also use helper macros scoped to its own test directory: + +```text +tests/unit/orders/ +├── _sqlbuild/ +│ └── macros/ +│ └── fixtures.py +└── test_orders__calculates_total.sql +``` + +Those helpers remain unavailable to production resources. Mocked model, source, seed, and dbt +fixture CTEs do not grant declarations from the mocked resource's production scope. + +## Macros importing macros + +A macro file may import another project macro file when the imported macro is available from the +importing file's location. The same directory rules apply as they do to SQL. + +```python +# models/commerce/macros/orders.py +from macros.currency import round_money + +def formatted_total(expression: str) -> str: + return round_money(expression) +``` + +A broadly available macro cannot import a macro from a narrower child or sibling directory. See +[Composition and Context](../macros/composition-and-context.md) for complete examples. + +## Names do not shadow + +Public names must be unique within their feature: + +- One public macro name cannot be defined twice. +- One public enum name cannot be defined twice. +- One public constant name cannot be defined twice. + +Moving a declaration into a narrower directory changes where it is available; it does not create a +second version that overrides another declaration. + +Macros, enums, and constants use separate namespaces, so these three references may coexist: + +```sql +@format_currency() +@const("format_currency") +@enum("format_currency").USD +``` + + Choose the simplest directory that matches where a declaration is used. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md new file mode 100644 index 0000000000..45448fde82 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md @@ -0,0 +1,242 @@ + + +# Data Diffs + +> Compare schemas and data between targets or virtual environments to validate changes before promotion. + +Online: https://docs.sqlbuild.com/concepts/diff + +## Contents + +- Comparison modes +- Deterministic key sampling +- Cursor coverage +- Row matching +- Tolerances +- Excluded columns +- Verbose output +- Structured output +- Invocation safety limits +- Selectors +- Exit codes + +SQLBuild can compare schemas and row-level data between two build contexts. This lets you validate that changes produce the expected results before promoting them. + +`sqb diff FROM:TO` compares: + +- **two targets** (e.g. `prod:dev`) in direct mode, or +- **two virtual environments** (VDEs) when [virtual environments](virtual-environments.md) are enabled. + +The mechanics below are identical for both; only what `FROM` and `TO` refer to changes. + +In direct mode, `FROM` and `TO` resolve the authoritative database/schema namespaces to compare. +SQLBuild uses the `TO` target's named connection for the complete comparison and accesses both +namespaces through fully qualified relations. It does not resolve or open the `FROM` target's +connection. The `TO` connection must therefore be able to read both namespaces; missing credentials +for that execution connection still fail before warehouse inspection. + +```bash +sqb diff prod:dev --full --select customer_status_snapshot +``` + +## Comparison modes + +Every diff requires exactly one mode: + +### Full diff + +Compares both schema and row-level data for the selected models: + +```bash +sqb diff prod:dev --full --select fact_orders +``` + +Rows are joined on the model's `unique_key` and compared column by column. The output shows: +- Row counts for each side +- How many rows are equal, unequal, or only in one side +- Which columns have mismatches with match percentages +- Example values showing what changed + +### Schema-only diff + +Compares column names and types without looking at row data: + +```bash +sqb diff prod:dev --schema-only --select fact_orders +``` + +Useful for quick structural checks or when row comparison would be too expensive. + +### Bounded diff + +Compares only a recent window of data using the model's cursor: + +```bash +sqb diff prod:dev --bounded 14d --select hourly_order_activity +``` + +For timestamp cursors, the bound is a duration (`14d`, `6h`, `30m`). For integer cursors, the bound is an integer value. If the model has no cursor configured, the diff falls back to a full row comparison. + +## Deterministic key sampling + +A cursor bound limits the range of data but does not guarantee a predictable number of rows. Use +deterministic key sampling to cap the wide value comparison for a high-volume model: + +```toml +[defaults] +row_diff_sample_rows = 100000 +row_diff_sample_seed = 0 + +[path_defaults.intermediate] +row_diff_sample_rows = 25000 +``` + +The settings use the normal configuration precedence: project `defaults`, matching +`path_defaults`, the model's `MODEL()` header, and finally CLI overrides. A model can disable an +inherited sample and request exhaustive comparison with `row_diff_sample_rows 0`: + +```sql +MODEL ( + materialized table, + unique_key [order_id, line_id], + row_diff_sample_rows 0, +); +``` + +Override the effective policy for one invocation with `--sample-rows` and `--sample-seed`, or force +an exhaustive comparison with `--exhaustive`: + +```bash +sqb diff prod:dev --bounded 14d --sample-rows 50000 --sample-seed 7 --select order_lines +sqb diff prod:dev --bounded 14d --exhaustive --select order_lines +``` + +SQLBuild applies the complete cursor bound first. It then selects the lowest deterministic hashes +from the union of unique keys found on either side and uses that same key set for both relations. +This avoids false side-only rows caused by independently sampling each side. Composite keys are +encoded with component lengths before hashing, and key columns break hash ties deterministically. + +Schema comparison, bounded row counts, cursor minimum/maximum values, and null/duplicate key checks +remain exhaustive. Only the wide column-by-column comparison is sampled. If the bounded union has +no more keys than the configured limit, SQLBuild reports the comparison as exhaustive. + +Sampled success means that no differences were found in the sampled keys; it is never reported as +complete table equality. The output shows the bounded key population, compared key count, seed, and +percentage evaluated. + +## Cursor coverage + +Before comparing values, SQLBuild reports exact bounded `COUNT`, `MIN(cursor)`, and `MAX(cursor)` +for each side. If cursor extents differ, it warns and continues with the requested comparison. It +does not ask for confirmation, silently narrow to the overlap, or hide rows that exist on only one +side. + +## Row matching + +Rows are matched between the two sides using the model's `unique_key`. Models without a `unique_key` can use schema-only diff but cannot run full or bounded row comparisons. + +The diff output categorises rows as: +- **Equal** - same key, same values on both sides +- **Unequal** - same key, different values (with per-column breakdown) +- **Left only** - exists in the FROM side but not TO +- **Right only** - exists in the TO side but not FROM + +## Tolerances + +Numeric columns can have tolerance rules to avoid false positives from floating-point differences or acceptable variance. Configure tolerances in the model's `MODEL()` header: + +```sql +MODEL ( + materialized incremental, + ... + row_diff_tolerances ( + by_column ( + total_revenue_cents ( + absolute 1, + ), + ), + ), +); +``` + +Tolerance rules support: +- **`absolute`** - maximum allowed absolute difference (e.g. `1` means values differing by 1 or less are treated as equal) +- **`relative`** - maximum allowed relative difference as a decimal (e.g. `0.01` for 1%) + +Tolerances can be set per-column (`by_column`) or per-type (`by_type`). + +## Excluded columns + +Columns that are expected to differ between the two sides (like timestamps or context-specific values) can be excluded from the row comparison: + +```sql +MODEL ( + materialized incremental, + ... + row_diff_exclude_columns [latest_order_status], +); +``` + +Excluded columns are still shown in the schema comparison but skipped during row-level diffing. A column cannot be in both `row_diff_exclude_columns` and `unique_key`. + +## Verbose output + +Add `--verbose` or `-v` to see more example rows for mismatches and side-only rows: + +```bash +sqb diff prod:dev --full --select customer_status_snapshot --verbose +``` + +Default example limits are 3 per category. Verbose mode increases this to 10. You can also set exact limits: + +```bash +sqb diff prod:dev --full --select customer_status_snapshot --max-column-examples 20 --max-row-only-examples 5 +``` + +Example limits only control diagnostic values printed after comparison. They are separate from +`row_diff_sample_rows`, which controls how many unique keys receive the wide value comparison. + +## Structured output + +Use `--json-output PATH` to write a stable structured result while retaining the normal terminal +summary: + +```bash +sqb diff prod:dev --bounded 14d --sample-rows 50000 --select order_lines --json-output diff.json +``` + +Each model records `schema_only`, `exhaustive`, or `sampled` comparison scope; requested and observed +cursor coverage; bounded population and compared key counts; seed and configured limit; row result +counts; and changed-column counts. The top-level status distinguishes `no_differences_found` from +`differences_found`. + +## Invocation safety limits + +Use `--max-models` and `--max-columns` to put explicit hard limits around a broad selector. SQLBuild +fails visibly instead of truncating the selected models or compared columns: + +```bash +sqb diff prod:dev --bounded 14d --sample-rows 50000 --max-models 10 --max-columns 80 --select path:models/intermediate +``` + +These limits are optional and apply to the complete invocation. `--max-columns` checks the larger +observed schema for each model before starting its row comparison. + +## Selectors + +Diff requires `--select` in the current version. You can use any selector syntax: + +```bash +# Diff a single model +sqb diff prod:dev --full --select customer_status_snapshot + +# Diff all models in a path +sqb diff prod:dev --schema-only --select path:models/marts + +# Diff models with a specific tag +sqb diff prod:dev --full --select tag:acceptance +``` + +## Exit codes + +`sqb diff` returns exit code `0` when all selected models have no differences, and `1` when any model has schema or row differences. This makes it usable in CI pipelines as a validation gate. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums-and-constants.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums-and-constants.md new file mode 100644 index 0000000000..2a40b4da12 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums-and-constants.md @@ -0,0 +1,52 @@ + + +# Enums and Constants Have Moved + +> Find the new focused guides for enums, constants, collections, contracts, and private values. + +Online: https://docs.sqlbuild.com/concepts/enums-and-constants + +Enums and constants now have separate guides so each feature is easier to find and learn. This page +remains available for existing links and bookmarks but is not part of the main navigation. + + Define fixed string or integer domains and reference validated members. + Define reusable scalar values and reference them safely from SQL. + Work with lists, sets, objects, value lists, and native arrays. + Keep enums and constants inside one model. + +## Public declarations + +See [Enums](enums.md) and [Constants](constants.md) for project-wide declarations. + +## Constant values + +See [Constants](constants.md#scalar-values) for scalars and +[Collections and Rendering](constants/collections-and-rendering.md) for collections. + +## References and rendering + +See [Collections and Rendering](constants/collections-and-rendering.md). + +## Adapter matrix + +See the [native-array rendering matrix](constants/collections-and-rendering.md#native-array-rendering). + +## Rendering configuration + +See [Project default](constants/collections-and-rendering.md#project-default). + +## Model-local declarations + +See [Model-Private Values](model-private-values.md). + +## Model-private declarations + +See [Model-Private Values](model-private-values.md). + +## Enum validation + +See [Enum validation rules](enums.md#validation-rules). + +## Enum-typed contracts + +See [Enum Model Contracts](enums/model-contracts.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums.md new file mode 100644 index 0000000000..379f6f88fe --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums.md @@ -0,0 +1,91 @@ + + +# Enums + +> Define a fixed set of named string or integer values and use them safely in SQL. + +Online: https://docs.sqlbuild.com/concepts/enums + +Enums give a name to a fixed set of allowed values. SQLBuild checks the enum and every member +reference during compilation, then renders the selected value as a safe SQL literal. + +## Create an enum + +Put project-wide enums under the top-level `enums/` directory. Files are discovered recursively, so +subdirectories can organize a large enum library without changing where the enums are available. + +```text +my_project/ +├── enums/ +│ ├── fulfillment/ +│ │ └── fulfillment_method.sql +│ └── order_status.sql +├── models/ +└── sqlbuild_project.toml +``` + +```sql +-- enums/fulfillment/fulfillment_method.sql +ENUM ( + name fulfillment_method, + members [DELIVERY, PICKUP, SHIPPING], +); +``` + +The shorthand above uses each member name as its string value. Use explicit values when the name +used in SQLBuild should differ from the stored value: + +```sql +ENUM ( + name source, + members ( + WEB "web", + PARTNER "partner", + ), +); +``` + +Integer enums always use explicit values: + +```sql +ENUM ( + name priority, + members (LOW 1, HIGH 3), +); +``` + +## Use an enum member + +Reference one member with `@enum("name").MEMBER`: + +```sql +SELECT * +FROM orders +WHERE fulfillment_method = @enum("fulfillment_method").DELIVERY + AND source = @enum("source").WEB +``` + +SQLBuild validates the enum name and member name before the query runs. The active adapter safely +renders the underlying string or integer value. + +Enum references work in model queries, SQL hooks, SQL functions, audits, unit tests, scenarios, and +inline source expressions. + +## Validation rules + +- An enum must contain at least one member. +- Every member must use the same value type: all strings or all integers. +- Enum names and member names must be SQL identifiers. +- Member names must be uppercase and lookup is case-sensitive. +- Enum names must be unique across all public enums in the project. +- Project-wide names cannot begin with `_`; that prefix is reserved for model-private values. + +Invalid declarations, unknown enums, and unknown members fail compilation. + +## More enum features + + Use an enum as a portable model-column domain and generate accepted-value validation. + Keep an enum inside one model when no other resource should use it. + +To limit an enum to one folder, or to that folder and its child folders, see +[Declarations and Scopes](declaration-scopes.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md new file mode 100644 index 0000000000..ee4d263134 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md @@ -0,0 +1,50 @@ + + +# Enum Model Contracts + +> Use an enum as a portable model-column domain with generated accepted-value validation. + +Online: https://docs.sqlbuild.com/concepts/enums/model-contracts + +An enum can describe the allowed domain of a model column as well as provide individual SQL +literals. + +These are separate uses: + +- `@enum("fulfillment_method").DELIVERY` inserts one validated value into SQL. +- `type fulfillment_method` declares that a model column uses the complete enum domain. + +## Declare an enum-typed column + +Use the enum name as the column type: + +```sql +MODEL ( + contract enforced, + columns ( + fulfillment_method (type fulfillment_method), + ), +); +``` + +SQLBuild does not send `fulfillment_method` to the warehouse as a physical type and does not create a +warehouse-native enum. It translates the enum into portable column metadata: + +| Enum values | Physical column type | Generated validation | +|-------------|----------------------|----------------------| +| Strings such as `DELIVERY` and `PICKUP` | `VARCHAR` | `accepted_values` for the enum strings | +| Integers such as `1` and `3` | `INTEGER` | `accepted_values` for the enum integers | + +## Contract behavior + +With `contract enforced`, SQLBuild runs the generated `accepted_values` audit with the model's other +audits. Its severity and timing follow the model's ordinary audit and materialization settings. + +With `contract none`, SQLBuild still resolves the enum to its portable scalar column type but does +not generate the domain audit. + +This behavior is the same on adapters with and without native enum support. The physical warehouse +column remains an ordinary string or integer column. + +Changing the members of an enum-typed contract changes the model's contract identity and generated +validation. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md new file mode 100644 index 0000000000..4b8fb79242 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -0,0 +1,182 @@ + + +# Functions + +> User-defined functions and table functions managed as part of your project. + +Online: https://docs.sqlbuild.com/concepts/functions + +## Contents + +- Scalar UDFs +- Table functions +- References inside functions +- Change propagation +- Project layout +- Python UDF options +- Adapter support + +Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG - if a function definition changes, every model that uses it is rebuilt. + +## Scalar UDFs + +Scalar UDFs return a single value per row. They can be written in SQL or Python. + +### SQL UDFs + +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body: + +```sql +-- functions/sql/udf__is_completed_order.sql +FUNCTION ( + arguments (order_status STRING), + returns BOOLEAN, +); + +order_status = 'completed' +``` + +### Python UDFs + +Place Python function files under `functions/python/`. Each file has exactly one function decorated with `@udf(...)`: + +```python +# functions/python/is_completed_order_py.py +from sqlbuild.functions import udf + +@udf( + arguments={"order_status": "STRING"}, + returns="BOOLEAN", + runtime_version="3.11", +) +def main(order_status: str | None) -> bool: + return order_status == "completed" +``` + +Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. + +### Using scalar UDFs + +Reference scalar UDFs in models with `__udf("name")`: + +```sql +SELECT + order_id, + __udf("udf__is_completed_order")(status) AS is_completed +FROM __ref("stg_orders") +``` + +The function name is the file stem (filename without extension). `__udf()` resolves to the adapter-native function call at compile time. + +## Table functions + +Table functions return multiple rows and columns. They are written in SQL under `functions/sql/` with a `returns table(...)` declaration: + +```sql +-- functions/sql/table_fn__customer_orders.sql +FUNCTION ( + arguments (p_customer_id INTEGER), + returns table ( + order_id INTEGER, + ordered_at TIMESTAMP, + waffle_name VARCHAR, + line_total_cents INTEGER, + order_status VARCHAR, + is_completed_order BOOLEAN + ) +); + +SELECT + order_id, + ordered_at, + waffle_name, + line_total_cents, + order_status, + is_completed_order +FROM __ref("fact_orders") +WHERE customer_id = p_customer_id +``` + +### Why table functions exist + +Table functions are designed as an alternative to final-layer views for cases where views don't push predicates efficiently. A view over `fact_orders` with a `WHERE customer_id = ?` filter may scan the entire table if the engine doesn't push the predicate down. A table function accepts the filter as an argument and guarantees the predicate is applied at execution time. + +### Managed table function dependencies + +Models call managed table functions with `__table_fn("name")(arguments...)`: + +```sql +SELECT customer_id, order_id +FROM __table_fn("table_fn__customer_orders")(42) +``` + +The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. + +The reference creates a typed dependency in the project DAG. Selecting a consuming model also selects the required function, function changes propagate to consumers, and cycles through models and functions are rejected. SQLBuild treats the call as an opaque relation for SQL analysis and uses the function's declared `returns table(...)` columns for star expansion and terminal lineage. + +Incremental state still belongs to the consuming model. A table function does not own a cursor interval or retain execution state. + +### Using table functions + +Applications and analysts can call the deployed warehouse function directly: + +```sql +SELECT * FROM table_fn__customer_orders(42) +``` + +## References inside functions + +SQL functions can reference the same resources as models: + +| Reference | Syntax | +|-----------|--------| +| Model | `__ref("model_name")` | +| Seed | `__seed("seed_name")` | +| Source | `__source("source_name")` | +| Scalar UDF | `__udf("function_name")` | +| Table function | `__table_fn("function_name")(arguments...)` | + +These references are resolved at compile time and create DAG edges. If a referenced model changes, the function is redeployed. + +A SQL function uses macros, constants, and enums available from its file under `functions/sql/`. +See [How Visibility Works](declaration-scopes/visibility.md) to limit declarations to one +function folder or to that folder and its children. + +## Change propagation + +Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it and marks dependent models as changed. + +## Project layout + +``` +functions/ + sql/ + udf__is_completed_order.sql # scalar SQL UDF + table_fn__customer_orders.sql # table function (returns table) + python/ + is_completed_order_py.py # scalar Python UDF +``` + +## Python UDF options + +The `@udf` decorator accepts these keyword arguments: + +| Argument | Required | Description | +|----------|----------|-------------| +| `arguments` | Yes | Dict mapping argument names to SQL types | +| `returns` | Yes | SQL return type string | +| `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | +| `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | +| `packages` | No | List of Python packages required at runtime | + +## Adapter support + +All four supported adapters implement SQL UDFs, Python UDFs, and table functions: + +| Feature | DuckDB | Snowflake | BigQuery | Databricks | +|---------|--------|-----------|----------|------------| +| SQL UDFs | Yes | Yes | Yes | Yes | +| Python UDFs | Yes | Yes | Yes | Yes | +| Table functions | Yes | Yes | Yes | Yes | + +Future adapters may not support all function types. SQLBuild raises a clear error if a function type is unsupported by the configured adapter. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md new file mode 100644 index 0000000000..5e2211f92d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -0,0 +1,395 @@ + + +# Incremental Models + +> Cursor-based incremental strategies, microbatch execution, and backfill policies. + +Online: https://docs.sqlbuild.com/concepts/incremental + +## Contents + +- Strategies +- Full-refresh overrides +- Cursors +- Microbatch execution +- Replay on change + +Incremental models process only new or changed data instead of rebuilding the entire table. In the +default sequential path, SQLBuild works out where to resume from the current target and input +relations with no separate checkpoint store. A retry recomputes its interval from current warehouse +state rather than reusing the exact interval of a failed attempt. Opt-in concurrent microbatching +adds immutable coordination facts as described below. + +## Strategies + +### append + +Inserts new rows without modifying existing data. Optionally uses a cursor (read from the target table's highest value) to avoid reprocessing the full source on every run. + +```sql +MODEL ( + materialized incremental, + incremental_strategy append, + cursor created_at, + cursor_type timestamp, + cursor_grain second, + append_cursor_inclusive true, +); + +SELECT id, customer_id, created_at +FROM __source("raw_events") +``` + +When `append_cursor_inclusive` is `true` (the default), the lower bound uses `>=`, which may duplicate the boundary row but avoids missing late-arriving data with the same cursor value. Set to `false` for an exclusive (`>`) lower bound if your cursor values are guaranteed unique. + +Append without a cursor is also valid. The model simply inserts all rows from the source query on every run. + +### delete_insert + +Deletes rows in the cursor range, then inserts the new delta. Requires either `cursor` or `unique_key`. + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + unique_key [order_id], + cursor order_id, + cursor_type integer, +); + +SELECT order_id, customer_id, order_status, ordered_at, line_total_cents +FROM __ref("fact_orders") +``` + +With a cursor, `delete_insert` removes rows where the cursor column falls within the replay window, then inserts the new delta. With a `unique_key` only, it deletes matching rows by key before inserting. + +### merge + +Upserts rows using a unique key. Matched rows are updated; unmatched rows are inserted. + +```sql +MODEL ( + materialized incremental, + incremental_strategy merge, + unique_key [customer_id], + cursor last_ordered_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + fact_orders ordered_at, + ), +); + +SELECT + customer_id, + MAX(ordered_at) AS last_ordered_at, + COUNT(*) AS total_orders, + SUM(line_total_cents) AS total_revenue_cents +FROM __ref("fact_orders") +GROUP BY customer_id +``` + +`merge` always requires `unique_key`. The cursor controls which upstream rows are scanned; the unique key determines how they're matched against the target. + +## Full-refresh overrides + +Incremental models can override command-level full-refresh behavior with `full_refresh`: + +| Model setting | Normal command | Command with `--full-refresh` | +|---------------|----------------|-------------------------------| +| omitted | incremental | full refresh | +| `full_refresh false` | incremental | incremental | +| `full_refresh true` | full refresh | full refresh | + +Use `full_refresh false` for models that must retain their normal cursor or microbatch execution even when a broader job requests full refresh: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + full_refresh false, + cursor event_date, + cursor_type timestamp, + cursor_grain day, +); +``` + +The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. It controls execution mode rather than acting as a safety rejection: an opted-out model does not abort the rest of the build. + +This does not skip initial loading. If the destination relation does not exist, an incremental or microbatch model still builds the history required by its cursor policy. Cloning an existing destination before the build can provide a current watermark and avoid a first-run historical replay. + +## Cursors + +Cursors define the incremental replay boundary. SQLBuild queries `MAX(cursor)` from the target table and `MIN/MAX` from upstream inputs to compute the replay window automatically. Observed maxima are inclusive warehouse values; SQLBuild advances them once to produce an effective exclusive end bound. + +| Field | Description | +|-------|-------------| +| `cursor` | Output column used to track incremental position | +| `cursor_type` | `timestamp` or `integer` | +| `cursor_grain` | Time grain for timestamp cursors: `second`, `minute`, `hour`, `day`, `month`, `year` | +| `cursor_start` | Lower bound floor; the cursor will never replay before this value | +| `cursor_inputs` | Map of upstream ref/source names to their cursor columns | + +### cursor_inputs + +When a model references multiple upstream inputs, `cursor_inputs` is required to tell SQLBuild which column on each input carries the cursor: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs ( + fact_orders ordered_at, + ), +); +``` + +SQLBuild uses these to determine the replay window. With multiple listed inputs, the end is the conservative common watermark: the minimum of their maxima. This prevents a faster input from advancing the model beyond data available from a slower input. + +On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. If it cannot establish a valid interval, the build fails before mutating the destination. + +### Cursor bounds in model SQL + +Cursor-based incremental models can read their effective interval with zero-argument intrinsics: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs ( + fact_orders ordered_at, + ), +); + +SELECT customer_id, ordered_at AS activity_hour +FROM __ref("fact_orders") +WHERE ordered_at >= __cursor_start() + AND ordered_at < __cursor_end() +``` + +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. + +In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. + +#### Listed inputs bound the window; unlisted inputs do not + +Only the inputs you list in `cursor_inputs` bound the replay window. This is an explicit choice, and it has two consequences worth understanding: + +- **Listed inputs** drive the window. Their new data advances the `MAX`, which is what tells SQLBuild how far to reprocess and which rows of the target to rewrite. +- **Unlisted inputs are read in full.** SQLBuild does not add a cursor filter to them, and they do not bound the window. This is correct for lookup or dimension tables that have no meaningful cursor column: you do not list them, and SQLBuild reads them whole rather than trying to filter on a column that may not exist. + +The implication for `delete_insert` and `merge`: the target rows that get rewritten are the ones whose cursor falls inside the window derived from the listed inputs. If an unlisted input changes in a way that should affect target rows outside that window, those rows are not rewritten on a normal incremental run. List every input whose new data should drive reprocessing; leave unlisted only the inputs you intend to read in full. + +To capture changes that fall outside the normal forward window, see [Lookback](#lookback) for late-arriving data and [Replay on change](#replay-on-change) for model changes. + +### Lookback + +Lookback extends the start of the replay window backwards to re-process recent data. The cursor is forward-moving, so use lookback to capture late-arriving or backfilled records that land just behind the current position: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor event_date, + cursor_type timestamp, + cursor_grain day, + lookback 3d, +); +``` + +With `lookback 3d`, the replay window starts 3 days before the normal cursor position, ensuring that any late-arriving data within that window is picked up. + +## Microbatch execution + +For large incremental ranges, microbatch mode splits the replay window into configurable batches. +Choose an explicit strategy: + +| Strategy | Window source | +|----------|---------------| +| `watermark` | Declared input availability; supports timestamp and integer cursors | +| `rolling_window` | A timestamp window relative to the current run rather than input maxima | + +Each batch has its own audit cycle: create delta, run delta audits, apply DML, and clean up. + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor_inputs ( + fact_orders (column ordered_at, roles [filter, watermark]), + ), + batch_size 1d, +); +``` + +Without microbatch mode, the entire replay range is processed in one pass. + +### Batch size + +`batch_size` controls the window size for each batch. For timestamp cursors, use duration strings like `1d`, `6h`, `1mo`. For integer cursors, use an integer value. + +For watermark models, each `cursor_inputs` entry declares whether the input filters model SQL, +contributes an availability watermark, or does both. `cursor_watermark_mode all` uses the +conservative common watermark; `any` permits progress from any watermark input. A filter-only input +does not claim that its intervals are available. + +### Concurrent batches + +Sequential execution is the default and does not read or write `_sqlbuild_microbatches`. To opt into +concurrent `delete_insert` batches, enable the project gate and choose a model limit: + +```toml +[settings] +microbatch_concurrency = true +``` + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + batch_concurrency 4, + -- cursor configuration omitted +); +``` + +Concurrent execution uses immutable requirement and completion facts to coordinate dependencies and +reconcile retries. It does not update one mutable status row through planned/running/complete +states. Increase concurrency deliberately because every active batch can consume warehouse work. + +### Watermark batch limits + +Watermark microbatch models can declare what to do when their resolved range contains more batches than an ordinary run should process: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor event_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs ( + raw_events (column event_date, roles [filter, watermark]), + ), + batch_size 1d, + microbatch_limit ( + max_batches 7, + action cap_from_end, + ), +); +``` + +| Action | Behavior when the range exceeds `max_batches` | +|--------|------------------------------------------------| +| `error` | Fail before hooks or model mutation | +| `warn` | Emit a prominent warning and execute the full range | +| `cap_from_start` | Execute the earliest N aligned batches and defer later work | +| `cap_from_end` | Execute the latest N aligned batches ending at the resolved watermark and defer older work | + +A cap changes only the work selected for that invocation. Deferred batches are not recorded as complete. `cap_from_end` is useful for feeds where keeping the latest projection current is more important than catching up oldest-first; `cap_from_start` is the oldest-first catch-up policy. + +Downstream watermark models consume durable completion facts from capped upstream models. A target +table's physical minimum/maximum does not prove that deferred or intervening intervals were built; +if completion evidence is unavailable, SQLBuild fails closed rather than inventing availability. + +The project can also set an outer safety policy: + +```toml +[microbatches.limits] +max_batches = 100 +action = "error" # or "warn" +``` + +Project limits support only `error` and `warn`; they never silently cap work. When a model has a nested `microbatch_limit`, the project limit checks the full resolved range first and the model policy then applies. + +`--max-microbatches N` is an invocation-wide, hard `error` ceiling and an explicit one-run authorization. It takes precedence over project and model limits, applies to models without a declared limit, and never inherits `cap_from_start` or `cap_from_end`. For example, passing a value large enough for an intentional backfill authorizes the full range instead of retaining the model's ordinary-run cap. + +The legacy scalar `max_microbatches` model field remains supported as a fail/warn guard. New models should use the nested form when the action is part of the model's execution policy. + +### Mixed-grain chains + +When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` cascade behavior described below. + +Alignment does two things: it floors the replay window edges to the coarsest grain, and it coarsens the batch size to that grain. For example, an hourly model downstream of a daily model processes in day-sized batches, so each batch lines up with a unit of upstream data that actually advances instead of producing empty or boundary-straddling windows: + +```sql +-- Upstream: daily grain, 2d batches +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_day, + cursor_type timestamp, + cursor_grain day, + cursor_inputs ( + hourly_order_activity activity_hour, + ), + incremental_mode microbatch, + batch_size 2d, +); + +-- Downstream: hourly grain, but aligns to day automatically +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs ( + daily_activity_rollup activity_day, + ), + incremental_mode microbatch, + batch_size 6h, +); +``` + +## Replay on change + +When a model's version identity changes (query, config, upstream cascade, or any other change reason), `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large downstream tables. You choose the cost per model: + +| Value | Effect | +|-------|--------| +| `forward` | Run the normal incremental delta from the cursor (default) | +| `full` | Full table rebuild | +| `bounded-14d` | Replay the last 14 days of data | + +The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (seconds). For example: `bounded-7d`, `bounded-24h`, `bounded-30m`. + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + replay_on_change full, +); +``` + +See [Cascade propagation](planning/cascade-propagation.md) for how replay policies propagate through the DAG and how downstream models can override inherited replay behavior. + +### on_schema_change + +Controls how schema differences are handled at execution time when the incremental delta has different columns than the target table: + +| Value | Effect | +|-------|--------| +| `append_new_columns` | Add new columns to the target table (default) | +| `sync_all_columns` | Add, drop, and alter columns to match the delta | +| `ignore` | Log and continue without schema changes | +| `fail` | Reject the build with an error | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md new file mode 100644 index 0000000000..d34eca9790 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md @@ -0,0 +1,210 @@ + + +# Interpolation + +> How SQLBuild processes variables, context, and dynamic content in SQL and config. + +Online: https://docs.sqlbuild.com/concepts/interpolation + +## Contents + +- Syntax reference +- Project variables +- Environment variables +- Context variables +- Deferred placeholders +- Audit parameters +- Named SQL hook parameters +- Compilation order + +SQLBuild uses two syntax layers for dynamic content: + +- **`@` syntax** is for any executable SQL - model queries, SQL hooks, tests, audits, and inline source expressions +- **`${...}` syntax** is for config values - project TOML config, MODEL() header fields (excluding SQL hooks), and source/seed YAML declarations + +The rule is simple: if it's any SQL that will be executed, it uses `@`. If it's a config value, it uses `${...}`. These layers never mix. + +## Syntax reference + +| Syntax | Where | Resolved | +|--------|-------|----------| +| `@enum("name").MEMBER` | Executable SQL | Compile time - expands to the validated enum member value | +| `@const("name")` | Executable SQL | Compile time - expands to the named typed scalar, value list, native array, or object expression | +| `@macro(args)` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - expands to macro return value | +| `@@name` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - project variable substitution | +| `@@ENV:NAME` | Model SQL, SQL hooks, tests, audits, inline source expressions | Compile time - environment variable | +| `@@CTX:name` | SQL hooks only | Compile time - destination relation, target, run ID | +| `@@@name` | Model SQL | Preserved for runtime (custom materializations) | +| `@name` / `@'name'` | Named SQL hook bodies | Raw / SQL-literal invocation argument, resolved at compile time | +| `@name` / `@'name'` | Generic audit SQL | Raw / SQL-literal audit argument | +| `${CTX:...}` | TOML/YAML config values | Config compilation | +| `${ENV:...}` | TOML/YAML config values | Config compilation | + +`@@CTX:` is intentionally SQL-hook-only. Model SQL describes a relation's data and should not reference its own destination identity. SQL hooks are the operational SQL layer where destination context is useful - grants, logging, post-materialization DDL. Python hooks access the same information through `ctx.destination` on the [`HookContext`](models/hooks/python.md#hook-context) object. + +See [Enums](enums.md) and [Constants](constants.md) for reusable validated values. See +[Collections and Rendering](constants/collections-and-rendering.md) for lists, sets, objects, +and adapter-specific rendering. + +SQLBuild uses the location of a SQL file to determine which macros, enums, and constants it can +use. Named SQL hooks use their own file location rather than the location of the calling model. See +[How Visibility Works](declaration-scopes/visibility.md) for the complete directory rules. + +## Project variables + +Project variables use `@@name` syntax in SQL and are defined in `sqlbuild_project.toml` or per-target: + +```toml +# sqlbuild_project.toml +[vars] +schema_prefix = "analytics" + +[targets.prod.vars] +schema_prefix = "prod_analytics" +``` + +```sql +SELECT * FROM @@schema_prefix.customers +``` + +Variables can also be set in `sqlbuild_local.toml` for developer-specific overrides, or passed via the CLI using `--vars` with a JSON object: + +```bash +sqb build --vars '{"schema_prefix": "staging_analytics"}' +``` + +CLI vars take precedence over local and project config vars. + +### JSON vars and nested values + +`--vars` accepts full JSON objects. Values can be strings, numbers, booleans, null, arrays, or nested objects: + +```bash +sqb build --vars '{"schema_prefix": "staging", "grants": {"primary_role": "analyst"}, "enabled": true}' +``` + +In SQL interpolation (`@@name`), only top-level scalar values can be used directly. `null` renders as an empty string. If a variable resolves to an array or object, SQLBuild raises a clear error suggesting you use a macro instead. + +For nested or complex values, use `ctx.vars` in a macro: + +```python +def grant_role(ctx): + return ctx.vars["grants"]["primary_role"] +``` + +The macro context receives the full native JSON structure including nested dicts, lists, booleans, numbers, and `None`. + +## Environment variables + +Environment variables use `@@ENV:NAME` syntax to inject values from the shell environment: + +```sql +SELECT * +FROM @@schema_prefix.customers +WHERE source_system = '@@ENV:SOURCE_SYSTEM' +``` + +If the environment variable is not set, SQLBuild raises a compile error. + +## Context variables + +Context variables provide access to the current model's destination relation, active target, and run metadata. + +**In SQL hooks** (`@@CTX:` syntax): + +```sql +post_hooks [inline_sql('GRANT SELECT ON @@CTX:destination.qualified TO analyst_role')], +``` + +Context components can appear directly beside qualification dots. SQLBuild resolves the longest +available context key before treating the dot as SQL punctuation: + +```sql +CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() +RETURNS INTEGER +LANGUAGE SQL +AS 'SELECT 1' +``` + +**In TOML/YAML config values** (`${CTX:...}` syntax): + +```toml +[targets.prod] +schema = "${CTX:destination.schema}" +``` + +Available context variables: + +| Variable | Value | +|----------|-------| +| `destination.qualified` | Fully qualified destination relation name | +| `destination.schema` | Destination schema | +| `destination.database` | Destination database | +| `destination.table` | Destination relation name | +| `model.name` | Model name | +| `model.database` | Model database | +| `model.schema` | Model schema | +| `model.alias` | Model alias | +| `run.target` | Active target name | +| `run.id` | Current run ID | + +**In macros**, the `MacroContext` object is passed as the first argument when a macro function accepts a `ctx` parameter: + +```python +def timestamp_trunc(ctx, grain: str, expr: str) -> str: + if ctx.adapter_name == "bigquery": + return f"TIMESTAMP_TRUNC({expr}, {grain.upper()})" + return f"DATE_TRUNC('{grain}', {expr})" +``` + +The macro context provides `adapter_name`, `sql_analysis_enabled`, `target_name`, and `vars`. + +## Deferred placeholders + +Custom materializations can define runtime placeholders using `@@@name` syntax. These are preserved through compilation and resolved by the materialization at execution time: + +```sql +WHERE CAST(ordered_at AS DATE) >= CAST(@@@partition_start AS DATE) + AND CAST(ordered_at AS DATE) < CAST(@@@partition_end AS DATE) +``` + +## Audit parameters + +Generic audit SQL uses `@name` (single `@`, no parentheses) for raw SQL placeholders and `@'name'` for escaped SQL-literal placeholders. These are resolved when the audit is attached: + +```sql +SELECT @column +FROM @relation +WHERE @column IS NULL +``` + +Use the quoted form for values rather than SQL identifiers or expressions: + +```sql +WHERE status = @'expected_status' +``` + +This is distinct from `@@name` (project variables) and `@macro()` (macro calls), so there is no ambiguity. See [Audits](audits.md) for details on generic audit parameters. + +## Named SQL hook parameters + +Reusable hooks under `hooks/sql/` use `@name` for raw SQL substitution and `@'name'` for escaped SQL-literal substitution. These parameters are supplied by `sql("name", args...)` and resolved before the rest of the hook SQL is compiled. See [SQL hooks](models/hooks/sql.md#sql-hook-arguments) for value rendering and validation rules. + +## Compilation order + +SQLBuild processes authored SQL in this order: + +1. **Config templates** (`${CTX:...}`, `${ENV:...}`) in TOML/YAML config values are resolved during config compilation +2. **Named SQL hook arguments** (`@name` and `@'name'`) are substituted into reusable hook bodies +3. **Project variables** (`@@name`), **environment variables** (`@@ENV:NAME`), and **context variables** (`@@CTX:name` in SQL hooks) are substituted +4. **Enum and constant references** are validated, normalized, and expanded through the active adapter +5. **Macro calls** (`@name(args)`) are expanded +6. **SQL analysis validation** runs against the fully expanded SQL + +This means: +- Config templates resolve first, before any SQL processing +- Named SQL hook arguments can contain `@@CTX:...`, `@@ENV:...`, project-variable, declaration, or macro text; model context values describe the invoking model, while declarations and macros in the resulting named hook body resolve from the hook definition path +- Macros see already-substituted variable values in the SQL +- `@@CTX:destination.qualified` in SQL hooks sees the final target-overridden destination name because hooks are expanded after destination naming is fully resolved +- SQL analysis validates the final expanded SQL, catching syntax errors from both vars and macros +- Python hook arguments are not SQL and remain unexpanded diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md new file mode 100644 index 0000000000..b5c4227799 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -0,0 +1,178 @@ + + +# Writing Macros + +> Write Python functions that generate reusable SQL fragments at compile time. + +Online: https://docs.sqlbuild.com/concepts/macros + +## Contents + +- Create a macro +- Call a macro from SQL +- Arguments +- Keep implementation details private +- Use macros in tests +- Use macros in hooks +- Where macros work +- Next steps + +Macros are Python functions that generate SQL during compilation. They provide reusable, +parameterized SQL without introducing a separate template language. + +## Create a macro + +Put project-wide macros in Python files under the top-level `macros/` directory: + +```text +my_project/ +├── macros/ +│ ├── currency.py +│ └── test_helpers.py +├── models/ +└── sqlbuild_project.toml +``` + +Every public function defined by a macro file becomes callable from SQL: + +```python +# macros/currency.py +def cents_to_dollars(expression: str) -> str: + """Convert a cents expression to dollars.""" + return f"ROUND(CAST({expression} AS DOUBLE) / 100, 2)" +``` + +## Call a macro from SQL + +Use `@macro_name(...)` in a model query: + +```sql +MODEL ( + materialized table, + tags [marts], +); + +SELECT + order_id, + @cents_to_dollars("amount_cents") AS amount_dollars +FROM __ref("stg_orders") +``` + +During compilation, SQLBuild replaces the call with the function's returned SQL: + +```sql +ROUND(CAST(amount_cents AS DOUBLE) / 100, 2) +``` + +A macro used directly in SQL must return a string. + +## Arguments + +Macro calls accept Python literal values: + +- Strings: `"value"` or `'value'` +- Numbers: `42`, `3.14`, `-1` +- Booleans: `True`, `False` +- Lists: `[1, 2, 3]` +- Dictionaries: `{"key": "value"}` +- `None` +- The result of another macro call + +Positional and keyword arguments are supported: + +```sql +@mock_orders(count=5, status="completed") +``` + +Use quoted strings when passing SQL expressions such as column names. The macro decides how to +place that text into its returned SQL. + +## Keep implementation details private + +Only public functions owned by the file are exported as macros. Prefix helpers, constants, classes, +and type aliases with `_`: + +```python +# macros/orders.py +_DEFAULT_STATUS = "completed" + +def _status_filter(status: str) -> str: + return f"status = '{status}'" + +def completed_orders() -> str: + return _status_filter(_DEFAULT_STATUS) +``` + +Here, SQL may call `@completed_orders()`. `_status_filter` and `_DEFAULT_STATUS` remain ordinary +Python implementation details. + +Imported functions are not re-exported as new macros from the importing file. + +## Use macros in tests + +Tests are SQL, so they can use macros as reusable fixture generators: + +```python +# macros/test_helpers.py +def mock_orders(count: int = 1) -> str: + rows = [ + f"SELECT {i} AS id, {i * 100} AS customer_id, 'completed' AS status" + for i in range(1, count + 1) + ] + return " UNION ALL ".join(rows) +``` + +```sql +TEST(); + +WITH +__source__raw__orders AS ( + @mock_orders(3) +), +__expected__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status + UNION ALL + SELECT 2 AS order_id, 200 AS customer_id, 'completed' AS status + UNION ALL + SELECT 3 AS order_id, 300 AS customer_id, 'completed' AS status +) +SELECT 1 +``` + +## Use macros in hooks + +Macros work inside inline and named SQL hooks: + +```python +# macros/permissions.py +def grant_target(target: str) -> str: + return f"GRANT SELECT ON {target} TO analyst_role" +``` + +```sql +MODEL ( + post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], +); + +SELECT 1 AS id +``` + +See [SQL Hooks](models/hooks/sql.md) for hook lifecycle and context syntax. + +## Where macros work + +Macros are supported in: + +- Model query SQL +- Inline and named SQL hooks +- Unit tests and scenarios +- Standalone audit SQL +- SQL functions and supported inline source expressions + +Macros are not accepted in ordinary `MODEL()` configuration fields. SQL hook entries are the +exception because their contents are SQL. + +## Next steps + + Compose macros through Python imports and use adapter or target context. + Limit a macro to one folder, or to that folder and its child folders. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/composition-and-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/composition-and-context.md new file mode 100644 index 0000000000..5b667d86f8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/composition-and-context.md @@ -0,0 +1,139 @@ + + +# Composition and Context + +> Compose macros through Python, use compile context, and understand scoped imports. + +Online: https://docs.sqlbuild.com/concepts/macros/composition-and-context + +## Contents + +- Compose helpers in one file +- Compose macros from different files +- Macro output is final SQL +- Nested calls written in SQL +- Macro context + +Macros are Python functions, so reusable macros should compose through ordinary Python calls. A +macro returns final SQL; SQLBuild does not treat its output as another layer of macro source. + +## Compose helpers in one file + +Use underscore-prefixed helpers for implementation details that should not be callable from SQL: + +```python +# macros/currency.py +def _divide_by_100(expression: str) -> str: + return f"({expression} / 100.0)" + +def cents_to_dollars(expression: str) -> str: + return f"ROUND({_divide_by_100(expression)}, 2)" +``` + +Only `cents_to_dollars` is exported as a SQLBuild macro. + +Public macros in the same file are also ordinary Python functions and may call one another: + +```python +def add_tax(expression: str) -> str: + return f"({expression} * 1.2)" + +def round_money(expression: str) -> str: + return f"ROUND({expression}, 2)" + +def formatted_total(expression: str) -> str: + return round_money(add_tax(expression)) +``` + +## Compose macros from different files + +Import another project macro when it is visible from the importing macro file: + +```python +# macros/orders.py +from macros.currency import add_tax, round_money + +def formatted_order_total(expression: str) -> str: + return round_money(add_tax(expression)) +``` + +SQLBuild records the imported macro files as dependencies. It rejects an import when the target +macro is outside the importing file's declaration scope or when imports form a cycle. + +Imported functions do not become duplicate exports from the importing file. In the example above, +`add_tax` and `round_money` retain their original identities; only `formatted_order_total` is newly +exported by `orders.py`. + +The same visibility direction applies to scoped macros: + +- A scoped macro may import a project-wide macro. +- A scoped macro may import a macro available from its own or an ancestor directory. +- A project-wide macro cannot import a narrower macro. +- A macro cannot import from a sibling or unrelated scope. + +See [Declarations and Scopes](../declaration-scopes.md) when macros are stored under `macros/` +or `_macros/`. + +## Macro output is final SQL + +Do not return SQL containing another `@macro()` call: + +```python +# Invalid: creates another macro-expansion layer. +def formatted_order_total(expression: str) -> str: + return f"@round_money(@add_tax({expression!r}))" +``` + +SQLBuild rejects this output. Import and call the Python functions instead: + +```python +from macros.currency import add_tax, round_money + +def formatted_order_total(expression: str) -> str: + return round_money(add_tax(expression)) +``` + +This keeps macro behavior readable in Python and ensures one expansion produces final SQL. + +## Nested calls written in SQL + +The SQL author may explicitly pass one macro's result to another: + +```sql +SELECT @round_money(@add_tax("subtotal")) AS order_total +``` + +SQLBuild evaluates `add_tax` first and passes its returned string to `round_money`. This is not a +second expansion of generated output: both calls are visible in the SQL source. + +Inner macros may return any Python value accepted by the outer macro. A macro used directly in SQL +must return a string. + +## Macro context + +When the first parameter is named `ctx`, SQLBuild passes a `MacroContext` with adapter, target, and +project-variable information: + +```python +# macros/datetime.py +def timestamp_trunc(ctx, grain: str, expression: str) -> str: + if ctx.adapter_name == "bigquery": + return f"TIMESTAMP_TRUNC({expression}, {grain.upper()})" + return f"DATE_TRUNC('{grain}', {expression})" +``` + +| Field | Description | +|-------|-------------| +| `adapter_name` | Active adapter, such as `duckdb` or `snowflake` | +| `sql_analysis_enabled` | Whether SQL analysis is enabled | +| `target_name` | Active target name, when selected | +| `vars` | Effective project variables after project, target, local, and CLI merging | + +```python +def schema_qualified(ctx, table: str) -> str: + schema = ctx.vars.get("schema_prefix", "public") + return f"{schema}.{table}" +``` + +Use context when generated SQL genuinely differs by adapter or target. Prefer ordinary parameters +for values that the SQL caller should choose explicitly. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md new file mode 100644 index 0000000000..cea0bbd4a5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md @@ -0,0 +1,71 @@ + + +# Model-Private Values + +> Keep enums and constants inside one model when no other resource should use them. + +Online: https://docs.sqlbuild.com/concepts/model-private-values + +Put an enum or constant directly in `MODEL()` when it belongs to one model and should not enter the +project-wide namespace. + +```sql +MODEL ( + enums ( + _state [OPEN, CLOSED], + ), + constants ( + _min_items 2, + _supported_countries ["GB", "FR", "DE"], + ), +); + +SELECT * +FROM order_batches +WHERE state = @enum("_state").OPEN + AND item_count > @const("_min_items") + AND country_code IN @const("_supported_countries") +``` + +## Where private values work + +A model-private value is available in: + +- The owning model's query +- Inline SQL hooks written in that model + +It is not available in: + +- Another model +- A child or sibling model directory +- A unit test or scenario +- A named SQL hook stored under `hooks/sql/` + +Private names begin with exactly one `_`. Names beginning with `__` are reserved for SQLBuild. +Because the model owns the name, different models may each define `_state` without creating a +collision. + +## Explicit types and rendering + +Use `constant(...)` when a private constant needs an exact type or rendering choice: + +```sql +MODEL ( + constants ( + _usd_rate constant( + type decimal, + value "2.4700", + ), + _supported_countries constant( + value ["GB", "FR", "HK"], + render_as array, + ), + ), +); +``` + +## When a value becomes shared + +Move the declaration out of `MODEL()` when another resource needs it. Remove the `_` prefix and put +it in the narrowest suitable enum or constant directory. See +[Declarations and Scopes](declaration-scopes.md) for those advanced placement options. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models.md new file mode 100644 index 0000000000..6d136060f7 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models.md @@ -0,0 +1,57 @@ + + +# Overview + +> SQL model anatomy, references, dependencies, and the model documentation guide. + +Online: https://docs.sqlbuild.com/concepts/models + +A model is a SQL file that defines one transformation step and produces a table or view in the warehouse. + +## Model anatomy + +Every model starts with a `MODEL()` header followed by its query: + +```sql +MODEL ( + materialized table, + tags [marts], + description "Order fact table", + columns ( + order_id (type INTEGER, audits [not_null]), + ), +); + +SELECT + o.order_id, + o.customer_id, + p.amount_cents +FROM __ref("stg_orders") o +JOIN __ref("stg_payments") p USING (order_id) +``` + +The header controls how SQLBuild builds, validates, and documents the model. The query remains ordinary SQL apart from SQLBuild reference and macro calls. + +## References + +| Reference | Syntax | Resolves to | +|-----------|--------|-------------| +| Model | `__ref("name")` | Another model | +| Seed | `__seed("name")` | A seed CSV table | +| Source | `__source("name")` | An external source | +| Scalar UDF | `__udf("name")` | A user-defined function | + +SQLBuild discovers the dependency graph from these calls and orders selected work topologically. Among selected models, upstream models run before downstream dependents. An unselected upstream is read from its existing warehouse relation; use an upstream-expanding selector such as `+fact_orders` when it should also be built. Seeds use `__seed()`, not `__ref()`. + +See [Functions](functions.md) for scalar UDF and table-function references. + +## Model guide + +- [Materializations](models/materializations.md): views, tables, incrementals, snapshots, and custom materializations. +- [Schemas](models/schemas.md): inline columns, reusable schemas, inheritance, audits, and model-local extensions. +- [Type Enforcement](models/type-enforcement.md): static type checks and runtime cast behavior by materialization. +- [Contracts](models/contracts.md): exact and open output validation, runtime guarantees, nullability, and enum-backed columns. +- [Hooks](models/hooks.md): SQL and Python lifecycle hooks. +- [Configuration](models/configuration.md): `MODEL()` field reference and SQL-validation controls. + +For deeper execution behavior, see [Incremental](incremental.md), [Snapshots](snapshots.md), and [Audits](audits.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md new file mode 100644 index 0000000000..f7c37aa2d2 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -0,0 +1,122 @@ + + +# Configuration + +> MODEL() header fields and SQL-analysis controls. + +Online: https://docs.sqlbuild.com/concepts/models/configuration + +## Contents + +- Common fields +- SQL analysis +- Table fields +- Incremental fields +- Snapshot fields +- Custom materialization fields +- Diff fields + +## Common fields + +| Field | Description | +|-------|-------------| +| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | +| `tags` | Tags used by selectors | +| `description` | Human-readable model description | +| `columns` | Model-local column declarations or inherited-column audit augmentation | +| `model_schema` | Reusable column schema name | +| `audits` | Model-level audit instances | +| `enums` | Model-local enum declarations; names must begin with `_` | +| `constants` | Model-local scalar, list, set, or object constants; names must begin with `_`. Use `constant(...)` for an explicit decimal type or collection rendering override. | +| `schema` | Destination warehouse schema override | +| `database` | Destination database override | +| `alias` | Destination relation-name override | +| `pre_hooks` | Ordered `inline_sql(...)`, `sql("name", ...)`, or `python("name", ...)` hooks before materialization | +| `post_hooks` | Ordered `inline_sql(...)`, `sql("name", ...)`, or `python("name", ...)` hooks after materialization | +| `enabled` | Set to `false` to disable the model | +| `contract` | `none` for an open statically checked declaration, or `enforced` for an exact declaration | +| `sql_analysis` | Per-model SQL-analysis override | + +## SQL analysis + +SQL analysis has project, invocation, and model gates: + +1. `settings.sql_analysis` must be enabled. +2. `--no-sql-analysis` must not be present. +3. `MODEL (sql_analysis false)` can disable analysis for that model. If omitted, the project setting + is used. + +A model cannot re-enable analysis when either broader gate disables it. The older `sql_validation` +and `--no-sql-validation` spellings remain compatibility aliases; use `sql_analysis` for new +configuration. + +## Table fields + +| Field | Description | +|-------|-------------| +| `run_despite_unchanged` | Table-only change-aware policy. `always` rebuilds whenever selected. A duration such as `30d`, `12h`, or `90m` rebuilds while the newest timestamp-based upstream source observation is within that age; it is not a periodic schedule. | + +Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](materializations.md#table). + +## Incremental fields + +| Field | Description | +|-------|-------------| +| `incremental_strategy` | `append`, `delete_insert`, or `merge` | +| `cursor` | Output column used to track incremental position | +| `cursor_type` | `timestamp` or `integer` | +| `cursor_grain` | Timestamp grain such as `second`, `hour`, or `day` | +| `cursor_start` | Lower cursor bound | +| `cursor_inputs` | Upstream names mapped to cursor columns | +| `unique_key` | Merge or delete/insert matching columns | +| `incremental_mode` | Set to `microbatch` for batched execution | +| `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | +| `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | +| `batch_size` | Timestamp duration string such as `1d` or `1h`; use a numeric string such as `"1000"` for an integer cursor | +| `batch_concurrency` | Concurrent batch workers; values above `1` require `delete_insert` and the project concurrency gate | +| `microbatch_limit` | Nested `max_batches` and `action` policy for watermark execution | +| `lookback` | Backward replay extension | +| `append_cursor_inclusive` | Include (`true`, default) or exclude (`false`) the current append-cursor boundary | +| `merge_exclude_columns` | Columns left unchanged by matched-row merge updates | +| `full_refresh` | Optional model execution override: `false` always runs incrementally, `true` always full-refreshes, and omission follows the command | +| `on_schema_change` | `append_new_columns`, `sync_all_columns`, `ignore`, or `fail` | +| `replay_on_change` | `forward`, `full`, or `bounded-` | + +See [Incremental](../incremental.md) for full semantics. + +Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. + +## Snapshot fields + +| Field | Description | +|-------|-------------| +| `unique_key` | Columns identifying one logical record | +| `snapshot_strategy` | `timestamp` or `check` | +| `updated_at` | Source update timestamp for the timestamp strategy | +| `check_columns` | Columns compared by the check strategy, or `[*]` | +| `observed_at` | Observation timestamp for historical inputs | +| `historical_input` | `snapshot` or `changes` | +| `initial_valid_from` | Initial validity policy for first-seen rows | +| `invalidate_hard_deletes` | Close records that disappear from current-state input | +| `valid_from_column` | Override the generated valid-from column name | +| `valid_to_column` | Override the generated valid-to column name | +| `snapshot_full_refresh` | Snapshot full-refresh safety policy | +| `snapshot_schema_change` | Snapshot schema-change policy | + +See [Snapshots](../snapshots.md) for strategy combinations, defaults, and safety behavior. + +## Custom materialization fields + +| Field | Description | +|-------|-------------| +| `config` | Arbitrary values passed to `ctx.config` | +| `placeholders` | Defaults for runtime `@@@placeholder` tokens | + +## Diff fields + +| Field | Description | +|-------|-------------| +| `row_diff_exclude_columns` | Columns excluded from row-level comparison | +| `row_diff_tolerances` | Numeric comparison tolerances | +| `row_diff_sample_rows` | Deterministic unique-key sample size; `0` disables inherited sampling | +| `row_diff_sample_seed` | Integer seed used for deterministic key hashing | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md new file mode 100644 index 0000000000..bd49cdf51c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md @@ -0,0 +1,171 @@ + + +# Contracts + +> Validate required or exact model output schemas. + +Online: https://docs.sqlbuild.com/concepts/models/contracts + +## Contents + +- Validation +- Type enforcement +- Reusable schemas +- Enum columns +- Adopting an existing schema +- Related policies + +A model's declared columns provide output metadata, type enforcement, and column audit attachment. The project `column_contract_mode` and model `contract` policy determine whether those declarations also define a statically validated output shape. + +| Model declaration | Behavior | +|-------------------|----------| +| Unspecified | Follow `settings.column_contract_mode`: `implicit` validates declared columns as an open shape; `explicit` treats them as metadata and audit attachment only. | +| `none` | Do not activate declared-shape or nullability contract validation for this model. Explicit type enforcement remains active. | +| `enforced` | Treat declared columns as the complete authoritative output shape and enable runtime exact-schema checks on supported materializations. | + +The project default is `column_contract_mode = "implicit"`, which preserves SQLBuild's original behavior. Projects that use column declarations primarily for metadata and audits, including migrations from dbt, can opt into explicit contracts: + +```toml +[settings] +column_contract_mode = "explicit" +``` + +Under explicit mode, a model must declare `contract enforced` to activate shape validation. A model-level `contract enforced` or `contract none` always overrides the project mode. + +```sql +MODEL ( + materialized table, + contract enforced, + columns ( + order_id (type INTEGER, nullable false), + customer_id (type INTEGER, nullable false), + amount_cents (type INTEGER), + status (type VARCHAR), + ), +); +``` + +## Validation + +### Compile time + +When declared-shape validation is active and SQL analysis can infer the model's output: + +- A missing declared column fails. +- A proven type mismatch fails when type enforcement or an exact contract is active. +- A declared non-null column fails when its expression is proven nullable. +- Additional inferred columns fail only for `contract enforced`. + +If SQLBuild cannot infer the output shape, these static shape checks cannot run. An enforced contract additionally requires a non-empty column declaration and adds runtime requirements on supported materializations. + +Type enforcement remains independent. Typed columns continue to receive static type checks and supported runtime casts under explicit mode and `contract none`. + +Configuration fields that reference output columns, including `unique_key`, `cursor`, `updated_at`, and `check_columns`, are checked against an enforced declaration. + +### Runtime + +For supported materialization paths, `contract enforced` inspects the staged relation and rejects: + +- Missing declared columns. +- Additional undeclared columns. +- Warehouse types that differ from declared types. + +Runtime contract validation does not scan data for nulls and does not inspect warehouse nullability metadata. `nullable false` participates in static nullability analysis; add a `not_null` audit when null values must be checked at runtime. + +Runtime exact-schema validation currently runs for: + +- Staged full-table builds. +- Non-microbatch incremental deltas. +- Snapshot deltas. + +Views and custom materializations rely on compile-time contract analysis. Microbatch incrementals currently apply type enforcement and audits but do not perform the framework runtime exact-schema validation step. + +Staged table validation happens before promotion, so a failure leaves the existing destination untouched. Immediate table promotion is incompatible with `contract enforced` because SQLBuild cannot validate output before replacing the destination. + +## Type enforcement + +Declaring a model column type enables type enforcement automatically, independently of the contract policy. Type enforcement controls static type compatibility and runtime casts on supported table and incremental paths; contracts control output shape. See [Type Enforcement](type-enforcement.md) for the materialization matrix. + +## Reusable schemas + +Contracts apply to the effective declaration, not only the reusable base: + +```sql +MODEL ( + model_schema order, + columns ( + ingestion_batch_id (type VARCHAR, nullable false), + ), + contract enforced, +); +``` + +With `contract enforced`, this requires exactly the resolved `order` columns plus `ingestion_batch_id`. With `contract none`, the resolved columns remain metadata, audit attachment points, and inputs to explicit type enforcement without activating shape validation. See [Schemas](schemas.md). + +## Enum columns + +A column may use a declared enum as a portable logical domain type: + +```sql +MODEL ( + contract enforced, + columns ( + fulfillment_method (type fulfillment_method), + ), +); +``` + +This does not require, create, or reference a warehouse-native enum type. SQLBuild resolves string-valued enums to `VARCHAR` and integer-valued enums to `INTEGER`. Under `contract enforced`, it also generates an `accepted_values` audit for the declared members. Audit severity and timing follow normal audit configuration and materialization behavior. With the default error severity, it gates staged-table promotion and pre-DML delta paths; views and custom materializations may already have changed their relation when the audit runs. + +Enum member references such as `@enum("fulfillment_method").DELIVERY` are a separate feature that render one validated SQL literal. See [Enum Model Contracts](../enums/model-contracts.md) for the complete distinction and lowering behavior. + +## Adopting an existing schema + +Ordinary `sqb compile` validates authored contracts offline. To compare declarations with existing +warehouse relations, use the explicitly online, read-only contract commands: + +```bash +sqb contract diff --from prod --select tag:commerce +sqb contract generate --from prod --select tag:commerce +``` + +The target named by `--from` supplies the database and schema namespace to inspect. SQLBuild uses +the active project connection and never resolves a second set of origin credentials. + +### Preview and write + +`contract diff` returns exit code `1` when declarations and physical relations disagree. +`contract generate` prints proposed declaration changes without changing files: + +```bash +# Fill missing types and append physical columns missing from code +sqb contract generate --from prod --select tag:commerce --write + +# Replace conflicting types/names and remove declarations absent physically +sqb contract generate --from prod --select tag:commerce --write --overwrite +``` + +Contract commands never mutate warehouse relations, lifecycle state, or fingerprints. + +### Safety rules + +- Additive generation preserves existing types on conflict, repository-only columns, comments, + descriptions, audits, nullability, enums, tags, freshness, and loader configuration. +- `--overwrite` is an explicit repository replacement policy, not a warehouse write. +- Generation does not add `contract enforced`; activation remains an authored decision. +- A shared `SCHEMA()` is not changed from one model's evidence. Ownership conflicts are reported. +- Writes are atomic and SQLBuild recompiles the project, restoring prior contents if validation + fails. +- Sources are first-class selectable resources, for example `--select source:raw_orders`. + +Enforced upstream contracts are authoritative compile-time interfaces. When input evidence is +complete, missing or ambiguous columns fail in projections, joins, filters, grouping, windows, and +ordering. Partial or opaque schemas remain open; SQLBuild does not turn missing metadata into a +false error. Use `sqb lineage --include-uses --format json` to inspect direct +non-projection uses such as `join_on`, `where`, `group_by`, and `window_order_by`. + +## Related policies + +The project default is implicit open-shape validation for models that omit `contract`. A repository can select explicit opt-in contracts with `settings.column_contract_mode = "explicit"` or configure a custom [Rule](../rules/custom-rules/overview.md) that requires enforced contracts for selected model families. + +Contracts also constrain schema-change behavior. For example, `snapshot_schema_change append_new_columns` is incompatible with `contract enforced` because an unannounced appended column would violate the exact declaration. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md new file mode 100644 index 0000000000..e1055781f6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md @@ -0,0 +1,119 @@ + + +# Hooks + +> Run SQL or Python lifecycle hooks around model materialization. + +Online: https://docs.sqlbuild.com/concepts/models/hooks + +## Contents + +- Project layout +- Choose a hook type +- Configure model hooks +- Lifecycle and failure timing +- Names and identity +- Diagnostics + +Pre-hooks and post-hooks run before and after model materialization. Use SQL hooks for warehouse statements and Python hooks for runtime control flow, providers, queries, and skip decisions. + +## Project layout + +SQL and Python hooks live in language-specific directories: + +```text +my_project/ + models/ + marts/ + orders.sql + hooks/ + sql/ + permissions/ + grant_access.sql + record_access.sql + python/ + notifications.py +``` + +Directories below `hooks/sql/` and `hooks/python/` are organizational. They do not namespace resource names: `hooks/sql/permissions/grant_access.sql` is still invoked as `sql("grant_access")`. + +## Choose a hook type + + + + + + + + + + +Hook lists use three explicit entry types: + +| Entry | Definition | Use | +|-------|------------|-----| +| [`inline_sql("...")`](hooks/sql.md#inline-sql-hooks) | SQL written directly in the model header | Short, model-specific SQL | +| [`sql("name", args...)`](hooks/sql.md) | A reusable resource under `hooks/sql/` | Shared, parameterized SQL | +| [`python("name", args...)`](hooks/python.md) | A decorated function under `hooks/python/` | Control flow, providers, queries, or skips | + +Bare SQL strings are not hook entries. The singular `pre_hook` and `post_hook` fields are also invalid; use `pre_hooks` and `post_hooks` with one of the three typed forms above. `sql("...")` always means a named SQL hook and never falls back to inline SQL. + +## Configure model hooks + +Models may mix SQL and Python entries in one ordered hook list: + +```sql +MODEL ( + materialized table, + pre_hooks [ + inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), + ], + post_hooks [ + sql( + "grant_access", + relation: "@@CTX:destination.qualified", + role: "analyst_role", + ), + python( + "notify_complete", + channel: "#data-builds", + ), + ], +); + +SELECT 1 AS id +``` + +Entries execute in the order authored, including when SQL and Python entries are mixed. + + + +## Lifecycle and failure timing + +Pre-hooks run before materialization. A failed pre-hook prevents materialization, and a Python pre-hook that returns `ctx.skip(...)` stops the remaining pre-hooks and prevents the model from running. + +Post-hooks run after warehouse mutation and audits. They are not promotion gates: a failed post-hook marks the model run as failed, but the relation has already been created, promoted, or incrementally updated. Put logic that must prevent materialization in a pre-hook, contract, pre-promotion audit, or the materialization itself. + +A Python post-hook that returns `ctx.skip(...)` stops the remaining post-hooks and changes the reported model result. `mode="hard"` blocks downstream nodes. A soft skip does not automatically block every downstream node; scheduler propagation also depends on the other upstream results. + +## Names and identity + +SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. + +The ordered hook list participates in each model's version identity: + +- Inline and named SQL hooks contribute their fully rendered statements. Named hooks also retain their resource name, definition, arguments, description, and source path in identity metadata. +- Python hooks contribute the invocation name, configured arguments, and a version hash derived from the decorated function, its transitive first-party dependencies, and decorator configuration. +- Changing a hook body, arguments, or order changes the consuming model's identity and makes it stale for change-aware planning. +- Executed Python hooks record their own hook fingerprints after successful completion or an explicit skip. + +Named SQL hooks compile into their consuming models rather than becoming independently scheduled nodes. Python hooks also run as part of their model's lifecycle phase rather than as independently selected Python nodes. + +## Diagnostics + +Discovery and compilation fail early for malformed definitions and invocations. Diagnostics include the resource path and, for model entries, the model name and indexed label such as `post_hooks[1] sql("grant_access")`. + +See the language-specific pages for definition, argument, validation, and runtime diagnostics: + +- [SQL hooks](hooks/sql.md) +- [Python hooks](hooks/python.md) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md new file mode 100644 index 0000000000..92bf8b2198 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -0,0 +1,105 @@ + + +# Python Hooks + +> Define Python lifecycle hooks with runtime context, providers, SQL access, and skips. + +Online: https://docs.sqlbuild.com/concepts/models/hooks/python + +Python hooks run model lifecycle logic that needs Python control flow, providers, warehouse queries, logging, or explicit skip decisions. + +For shared lifecycle ordering, failure timing, naming, and identity rules, see the [Hooks overview](../hooks.md). + +## Define a Python hook + +SQLBuild discovers decorated functions recursively from `.py` files under `hooks/python/`: + +**`hooks/python/notifications.py`** + +```python +from sqlbuild.hooks import hook + +@hook +def notify_complete(ctx, channel="#data-builds"): + ctx.log( + f"Notify {channel}: " + f"{ctx.model_name} completed during {ctx.phase}" + ) +``` + +By default, the hook name is the function name. The decorator accepts optional `name` and `description` arguments. A Python file may define multiple decorated hooks. + +Files named `__init__.py` or beginning with `_` are skipped. Imported decorated functions are not registered again from the importing module. + +## Invoke a Python hook + +Reference the hook by name and optionally pass keyword arguments: + +```sql +MODEL ( + materialized table, + post_hooks [ + python( + "notify_complete", + channel: "#model-alerts", + ), + ], +); + +SELECT 1 AS id +``` + +Python hook arguments are ordinary configuration values and are not SQL-expanded. An argument containing `@@CTX:...` or `@macro()` reaches the function unchanged. + +## Signature validation + +Unknown hooks, unknown keyword arguments, missing required arguments, required positional-only arguments, and arguments that conflict with context or provider injection fail compilation. A function with `**kwargs` can accept additional configured arguments. + +Python hooks must return `None` or `ctx.skip(...)`. Any other return value fails execution. + +## Hook context + +Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, or `hook_context`. It need not be the first parameter when providers or configured arguments are also present: + +| Field | Description | +|-------|-------------| +| `ctx.model_name` | Model being built | +| `ctx.phase` | `pre_hooks` or `post_hooks` | +| `ctx.hook_name` | Invoked hook name | +| `ctx.hook_index` | Zero-based position in the authored hook list | +| `ctx.run_id` | Current run ID | +| `ctx.target` | Active target | +| `ctx.vars` | Effective project variables | +| `ctx.destination` | Destination relation metadata | +| `ctx.adapter_name` | Active adapter name | +| `ctx.adapter` | Adapter instance | +| `ctx.connection` | Live connection | +| `ctx.execute_sql(sql)` | Execute SQL | +| `ctx.query(sql)` | Execute SQL and return rows | +| `ctx.log(message)` | Write run output | +| `ctx.skip(reason="...", mode="soft")` | Return a soft or hard skip result; arguments are keyword-only | +| `ctx.providers` | Access discovered [providers](../../python-nodes/providers.md) | + +## Providers + +Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, tasks, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. + +```python +from sqlbuild.hooks import hook + +@hook +def notify_complete(ctx, slack_notifier): + slack_notifier.send( + f"Model {ctx.model_name} built successfully" + ) +``` + +## Skips and failures + +Returning `ctx.skip(...)` stops the remaining hooks in the current phase. Runtime exceptions fail that lifecycle phase. See [Lifecycle and failure timing](../hooks.md#lifecycle-and-failure-timing) for the effects of pre-hook and post-hook skips, soft and hard modes, and failures after warehouse mutation. + +## Identity and diagnostics + +Python hook invocations and version hashes participate in model identity as described in [Names and identity](../hooks.md#names-and-identity). Executed hooks also record their own fingerprints after successful completion or an explicit skip. + +Compilation reports unknown hooks and invalid signatures with the model name and indexed invocation label, such as `post_hooks[1] python("notify_complete")`. Runtime output preserves the authored index and hook name. Exceptions and unsupported return values fail with the Python hook identity attached. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md new file mode 100644 index 0000000000..a1c7ac417c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md @@ -0,0 +1,177 @@ + + +# SQL Hooks + +> Define, parameterize, compile, and invoke reusable or inline SQL lifecycle hooks. + +Online: https://docs.sqlbuild.com/concepts/models/hooks/sql + +## Contents + +- Reusable SQL hooks +- Invoke a named hook +- SQL hook arguments +- Inline SQL hooks +- Compile-time context +- Validation and diagnostics + +SQL hooks submit one rendered SQL payload to the adapter before or after model materialization. Use a named SQL hook for reusable behavior and `inline_sql(...)` for short model-specific payloads. + +For shared lifecycle ordering, failure timing, naming, and identity rules, see the [Hooks overview](../hooks.md). + +## Reusable SQL hooks + +SQLBuild discovers `.sql` files recursively under `hooks/sql/`. Each file defines exactly one hook and must start with a `HOOK(...)` header as its first non-whitespace content. + +**`hooks/sql/permissions/grant_access.sql`** + +```sql +HOOK ( + description "Grant a warehouse role access to the model relation" +); + +GRANT SELECT ON @relation TO @role +``` + +The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. + +`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. + +Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. + +## Invoke a named hook + +Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` list: + +**`models/marts/orders.sql`** + +```sql +MODEL ( + materialized table, + post_hooks [ + sql( + "grant_access", + relation: "@@CTX:destination.qualified", + role: "analyst_role", + ), + ], +); + +SELECT 1 AS id +``` + +SQLBuild resource-header fields use native `key value` syntax, as in `TEST (mode macro)`, `SCENARIO (tags ["revenue"])`, `AUDIT (severity error)`, and `HOOK (description "...")`. The `relation: ...` and `role: ...` entries above intentionally retain `key: value` syntax because they are nested named arguments to the `sql(...)` hook call, not resource-header fields. The same distinction applies to named arguments passed to `python(...)`. + +## SQL hook arguments + +Named SQL hooks declare parameters by using them in the SQL body. Arguments are supplied as named values in `sql("name", args...)`: + +| Syntax | Behavior | +|--------|----------| +| `@name` | Raw substitution. Strings are inserted verbatim for relations, identifiers, keywords, or SQL fragments. | +| `@'name'` | SQL-literal substitution. Strings are single-quoted and embedded quotes are escaped. | + +**`hooks/sql/record_access.sql`** + +```sql +HOOK ( + description "Record access configuration" +); + +INSERT INTO audit.access_log (relation_name, role_name) +VALUES (@'relation', @'role') +``` + +```sql +MODEL ( + post_hooks [ + sql( + "record_access", + relation: "@@CTX:destination.qualified", + role: "O'Brien", + ), + ], +); + +SELECT 1 AS id +``` + +For both forms, booleans render as `TRUE` or `FALSE`, numbers render directly, and `null` renders as `NULL`. Lists render as comma-separated values, applying raw or quoted behavior to each item. For example, `@'roles'` with `roles: ["reader", "writer"]` renders as `'reader', 'writer'`. + +Every referenced argument is required and every supplied argument must be used. Missing arguments, unused arguments, and unsupported values such as maps fail compilation. Raw string arguments are not escaped; use `@'name'` for data values and reserve `@name` for trusted SQL structure. + +## Inline SQL hooks + +Use `inline_sql("...")` for SQL that is specific to one model: + +```sql +MODEL ( + materialized table, + post_hooks [ + inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), + ], +); + +SELECT 1 AS id +``` + +An inline hook accepts exactly one quoted SQL string and no additional arguments. That string becomes one adapter execution payload. + +## Compile-time context + +Both named and inline SQL hooks receive the invoking model's runtime context. They support: + +- Project variables such as `@@audit_schema` +- Environment variables such as `@@ENV:DEPLOY_ROLE` +- Hook context variables such as `@@CTX:destination.qualified` +- Enums and constants such as `@enum("role").ANALYST` and `@const("retention_days")` +- Python macros such as `@grant_target("@@CTX:destination.qualified")` + +For named hooks, SQLBuild first substitutes `@name` and `@'name'` arguments into the hook body. A supplied argument such as `relation: "@@CTX:destination.qualified"` therefore resolves to the invoking model's final target-overridden destination. + +An inline hook uses macros, constants, and enums available to its model file. A named hook uses those +available to its own file under `hooks/sql/`. See +[How Visibility Works](../../declaration-scopes/visibility.md#which-file-controls-visibility). + +`${...}` config-template syntax is not valid in SQL hooks. + +| Variable | Value | +|----------|-------| +| `@@CTX:destination.qualified` | Fully qualified destination relation | +| `@@CTX:destination.schema` | Destination schema | +| `@@CTX:destination.database` | Destination database | +| `@@CTX:destination.table` | Destination relation name | +| `@@CTX:model.name` | Model name | +| `@@CTX:model.database` | Model database | +| `@@CTX:model.schema` | Model schema | +| `@@CTX:model.alias` | Model alias | +| `@@CTX:run.target` | Active target name | +| `@@CTX:run.id` | Current run ID | + +Context components can be combined into qualified identifiers without whitespace or a Python +wrapper: + +```sql +HOOK (); + +CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() +RETURNS INTEGER +LANGUAGE SQL +AS 'SELECT 1' +``` + +## Validation and diagnostics + +SQLBuild strictly validates hook resource syntax, arguments, interpolation, macros, metadata, and non-empty payloads. It does not classify executable statement kinds or implement vendor SQL grammar. Each rendered hook payload is passed to the adapter in one `execute` call; whether a driver accepts multiple statements or client-side batch separators is adapter and warehouse behavior. Use separate hook entries when portable ordering between statements matters. + +After expansion, Polyglot validates the complete payload using the active adapter's analysis dialect when the model's effective `sql_analysis` setting is enabled, which it is by default. The `--no-sql-analysis` flag disables this analysis for the invocation. Polyglot does not understand every administrative or procedural command supported by every warehouse. If it cannot parse valid vendor-specific hook SQL, disable SQL analysis for that model or invocation and let the adapter and warehouse provide the authoritative result. SQLBuild does not compensate with keyword allowlists or handwritten parser fallbacks. + +Common errors include: + +- Missing or malformed leading `HOOK(...)` headers +- Unsupported header keys, empty descriptions, and missing SQL bodies +- Missing or unused arguments and unsupported argument values +- Unknown named hooks, unquoted hook names, and bare hook strings +- SQL rejected by Polyglot while optional SQL validation is enabled + +Definition errors point to the hook file. Invocation and argument errors also identify the consuming model entry, such as `post_hooks[1] sql("grant_access")`. Runtime output preserves the authored hook index and identifies named and inline SQL entries. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md new file mode 100644 index 0000000000..52aefa9e51 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -0,0 +1,100 @@ + + +# Materializations + +> Choose how SQLBuild persists model output. + +Online: https://docs.sqlbuild.com/concepts/models/materializations + +The `materialized` field selects how a model becomes a warehouse relation. + +## View + +Creates or replaces a database view on each build: + +```sql +MODEL (materialized view); + +SELECT id AS order_id, customer_id, status +FROM __source("raw__orders") +``` + +View audits run after the view has been replaced. A failing audit marks the build as failed but cannot preserve the previous view. Views use compile-time contract and type analysis; SQLBuild does not rewrite the view with runtime casts or run the staged runtime-contract step. + +## Table + +By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. + +```sql +MODEL (materialized table); + +SELECT customer_id, COUNT(*) AS total_orders +FROM __ref("stg_orders") +GROUP BY customer_id +``` + +Projects may opt into `settings.table_promotion_mode = "immediate"`. Immediate promotion replaces the destination before audits, so a failed audit does not restore the old table. Immediate promotion rejects models that require declared-type enforcement or `contract enforced`; use staged promotion for those guarantees. + +## Incremental + +On a normal incremental run, applies append, delete/insert, or merge DML from a staged delta. Delete/insert removes matching keys or cursor ranges before inserting replacement rows. Cursor configuration controls replay bounds and may split work into microbatches. + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + unique_key [activity_hour], +); +``` + +For non-microbatch models, the first run, `--full-refresh`, and a full replay-on-change rebuild use the full-table path rather than incremental DML. Microbatch models retain batched execution: a full rebuild drops the existing target, creates its replacement from the first batch, and applies later batches with incremental DML. A failed full-refresh microbatch does not preserve the previous target. + +See [Incremental](../incremental.md) for cursor semantics, replay, schema changes, and microbatch execution. + +## Snapshot + +Maintains historical row versions with SCD Type 2 semantics: + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy timestamp, + updated_at updated_at, +); + +SELECT customer_id, name, plan, status, updated_at +FROM __source("customers") +``` + +See [Snapshots](../snapshots.md) for timestamp and check strategies, historical inputs, and full-refresh policies. + +## Custom + +A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. + +```sql +MODEL ( + materialized partition_tracked, + placeholders ( + partition_start "'2026-04-01'", + partition_end "'2026-04-05'", + ), + config ( + tracking_table partition_state, + partition_column order_date, + ), +); + +SELECT * +FROM __ref("stg_orders") +WHERE ordered_at >= @@@partition_start + AND ordered_at < @@@partition_end +``` + +The `config` block is passed to the Python function through `ctx.config`. Runtime-owned `@@@placeholder` values remain unresolved until materialization execution. + +The custom function owns staging, runtime type or contract enforcement, audit timing, promotion, and rollback. Framework final audits run after the function returns unless `MaterializationResult.audit_results` is populated. A custom materialization can call `ctx.run_audits` against a staging relation before applying changes and return those results through `MaterializationResult.audit_results`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md new file mode 100644 index 0000000000..4d3b9b25e6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md @@ -0,0 +1,137 @@ + + +# Schemas + +> Declare model columns inline or reuse canonical inherited schemas. + +Online: https://docs.sqlbuild.com/concepts/models/schemas + +## Contents + +- Inline columns +- Reusable schemas +- Model-local columns +- Model-specific column audits +- Inheritance +- Contracts and planning +- Limitations + +Schema metadata defines column names, types, nullability, descriptions, and column audits. It can live inline in one `MODEL()` header or in a reusable `SCHEMA()` declaration. + +## Inline columns + +Use inline columns for metadata owned by one model: + +```sql +MODEL ( + materialized view, + description "Cleaned order records", + columns ( + order_id (type INTEGER, nullable false, audits [not_null, unique]), + customer_id (type INTEGER, nullable false, audits [not_null]), + status ( + type VARCHAR, + audits [accepted_values (values ["placed", "completed", "cancelled"])], + ), + ), +); +``` + +See [Audits](../audits.md) for built-in audits, custom audits, arguments, severity, and incremental run scope. + +## Reusable schemas + +When multiple models implement the same relation shape, declare it once under `schemas/`. SQLBuild discovers schema files recursively and makes public names available throughout the project. + +```sql +-- schemas/orders/order.sql +SCHEMA ( + name order, + description "Canonical staged order shape", + columns ( + order_id (type INTEGER, nullable false, audits [not_null]), + customer_id (type INTEGER, nullable false, audits [not_null]), + status (type VARCHAR), + ), +); +``` + +Bind a model with `model_schema`: + +```sql +MODEL ( + materialized view, + schema staging, + model_schema order, + contract enforced, +); +``` + +`schema staging` selects the warehouse destination schema. `model_schema order` selects reusable column metadata. + +The reusable description becomes the model description when the model does not declare one. A model-owned description takes precedence. + +## Model-local columns + +A bound model may add output columns that are not part of the reusable shape: + +```sql +MODEL ( + model_schema order, + columns ( + ingestion_batch_id (type VARCHAR, nullable false), + ), + contract enforced, +); +``` + +Resolved schema columns retain their order and new model-local columns follow them. Use a named child schema when an extension is reusable; use inline columns for an extension owned by one model. + +## Model-specific column audits + +Audits in a reusable schema apply to every bound model. A model can add stricter audits to an inherited column by naming that column and declaring only `audits`: + +```sql +MODEL ( + model_schema order, + columns ( + order_id (audits [unique]), + ), +); +``` + +The effective `order_id` keeps the reusable type, nullability, description, and `not_null` audit, then adds `unique`. A model cannot remove reusable audits or override inherited metadata. An inherited-column entry containing `type`, `nullable`, or `description` fails compilation. Identical audit instances are deduplicated. + +## Inheritance + +A reusable schema may extend one parent with additional columns: + +```sql +SCHEMA ( + name sourced_order, + extends order, + columns ( + source (type VARCHAR, nullable false), + ), +); +``` + +Inheritance may be transitive. Parent columns resolve before child columns. An inherited column cannot be redeclared or overridden in a child schema. SQLBuild rejects unknown parents, cycles, case-insensitive duplicates, and multiple parents. + +Physical SQL output order is not currently enforced. Static contract analysis matches names and checks declared types and proven non-nullability. Runtime exact-contract validation matches names and types but does not validate nullability. + +## Contracts and planning + +`contract enforced` treats the complete named-plus-local declaration as the exact output shape. An unspecified model follows the project's `column_contract_mode`; `contract none` explicitly keeps the resolved columns as metadata, audit attachment, and type-enforcement inputs without activating shape validation. See [Contracts](contracts.md). + +For models bound to a reusable schema, effective column names, types, nullability, and enum members participate in model version identity. Changing those fields on a parent therefore affects models bound through descendants. Descriptions do not change model identity. Audits have their own audit-gate identities, so audit changes invalidate reusable audit results without changing the model version itself. + +## Limitations + +Reusable schemas intentionally support a narrow ownership model: + +- One optional parent, with transitive inheritance. +- Additive child and model-local output columns. +- Audit-only model augmentation of inherited columns. +- No general column overrides, multiple inheritance, composition, mixins, parameters, or generated projections. +- No physical output ordinal enforcement. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md new file mode 100644 index 0000000000..c86f72603f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md @@ -0,0 +1,74 @@ + + +# Type Enforcement + +> Understand declared model types, static checks, and runtime casting. + +Online: https://docs.sqlbuild.com/concepts/models/type-enforcement + +Type enforcement makes authored column types operational. It is a model feature as well as a related source feature, and it is separate from whether a model uses an exact contract. + +## Enabling it + +For a SQL model, declaring a type on any inline or reusable-schema column enables type enforcement automatically: + +```sql +MODEL ( + materialized table, + columns ( + order_id (type INTEGER), + ordered_at (type TIMESTAMP), + source_system (), + ), +); +``` + +There is no `MODEL(type_enforcement ...)` switch. In this example, `order_id` and `ordered_at` are typed; `source_system` remains untyped and passes through unchanged. + +Sources have related behavior but different configuration. A source can explicitly set `type_enforcement: false`; see [Sources](../sources.md#type-enforcement). + +## Compile-time checks + +When SQL analysis can infer an output type, SQLBuild compares it with the declared type using adapter-aware normalization. A proven mismatch is a compile error when type enforcement is active. If SQLBuild identifies an output column but cannot prove its expression type, it reports an unproven-type warning rather than guessing. If the entire output shape cannot be inferred, static type checks and warnings cannot run. + +Static analysis does not rewrite the authored query. It determines whether the inferred output is compatible before warehouse execution. + +## Runtime casting + +Runtime enforcement inspects a staged relation. If a typed output column does not already have its declared type, SQLBuild rebuilds the staging or delta relation with an explicit cast: + +```sql +CAST(order_id AS INTEGER) AS order_id +``` + +Only columns with declared types are cast. Untyped output columns are preserved. If a cast fails, the model fails during the type-enforcement phase before the staged table is promoted or the incremental delta is applied. + +Runtime cast support currently depends on materialization: + +| Materialization path | Framework runtime casts | +|----------------------|-------------------------| +| Full table with staged promotion | Yes, on the staging table | +| Non-microbatch incremental | Yes, on the staged delta | +| Microbatch incremental | Yes, on each staged batch | +| View | No; the view query defines the warehouse output type | +| Snapshot | No framework cast reconstruction | +| Custom materialization | Owned by the custom materialization | +| Full table with immediate promotion | Rejected when type enforcement is required | + +Staged promotion is the default table mode. If a project selects immediate table promotion, a typed model fails with guidance to use staged promotion because SQLBuild cannot inspect and reconstruct output before mutating the destination. + +## Type enforcement versus contracts + +These controls answer different questions: + +| Feature | Question | +|---------|----------| +| Type enforcement | Should typed columns be checked and, on supported paths, cast to their declared physical types? | +| `contract none` | Which declared columns can SQLBuild verify statically while still allowing additional output columns? | +| `contract enforced` | Must the output have exactly the complete declared shape? | + +Type enforcement can be active with `contract none`. Conversely, an enforced contract may contain untyped columns: exact shape validation still checks their names, but there is no declared physical type to cast or compare. + +## Schema changes + +Declared types are used in schema-change detection. With type enforcement active, the declared type is authoritative for typed columns when SQLBuild compares a planned model with the warehouse relation. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md new file mode 100644 index 0000000000..9771a28f5a --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md @@ -0,0 +1,86 @@ + + +# Execution Observability + +> Choose authoritative lifecycle facts, readable logs, or command-output records. + +Online: https://docs.sqlbuild.com/concepts/observability + +SQLBuild keeps execution facts separate from human output. Do not parse terminal text to determine +whether work started, completed, failed, skipped, or retried. + +| Record | Purpose | Authority | +|--------|---------|-----------| +| `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | +| `DiagnosticLog` | Structured framework diagnostics | Explanatory only | +| `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | +| `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | +| Final JSON output | End-of-command aggregate | Result projection, not event history | + +Human-readable CLI output remains the normal operator interface. Structured records are a separate +contract for integrations and durable consumers. + +## Lifecycle envelope + +New events use lifecycle schema version 2: + +```json +{ + "event_id": "9fb6d899754d4bd79c703067a8d5046c", + "event_type": "statement_completed", + "schema_version": 2, + "producer": "sqlbuild", + "producer_version": "0.91.0", + "occurred_at": "2026-09-07T18:15:30.125000Z", + "invocation_id": "4cc757dd93fe466aad2a220f1e76625e", + "invocation_sequence": 17, + "run_id": "run-20260907-181500", + "resource_id": "model:orders", + "resource_attempt_id": "97933c4aa29441cda133641345515e20", + "operation_id": "c978e47367494051bdde68ad03e56db4", + "statement_id": "5113ebc5afe247eb9c41dc0b27ef1a9a", + "external_context": { + "integration": { + "name": "dagster", + "run_id": "dagster-run-1", + "job_name": "daily_models", + "step_key": "sqlbuild_assets", + "retry_number": 0, + "partition_key": "2026-09-07" + } + }, + "payload": { + "adapter": "snowflake", + "duration_ms": 318.4, + "query_id": "01b6f1c2-0000-0000-0000-000000000000", + "sql_digest": "881b77b00a75e07936d7f20ea308a55e9ef98f2f1fd42c144a3a998d32094c1b" + } +} +``` + +`event_id` is the immutable deduplication identity. `invocation_sequence` is unique and increasing +in synchronous publication order within one invocation, including concurrent workers. A sink may +prioritize delivery, so consumers reconstruct invocation chronology by sorting this field rather +than relying on arrival timestamps. + +Schema-version-1 events remain readable. Unknown event names and future schema versions are retained +as opaque envelopes instead of being assigned invented semantics. + +## Correlation and privacy + +Every fact has an invocation ID. Nested IDs identify a run, logical resource and attempt, non-SQL +operation, and SQL statement. Orchestrators may provide bounded JSON-compatible +`external_context`; the Dagster integration supplies run, job, step, retry, and partition values +when its execution context exposes them. + +Lifecycle payloads exclude full SQL, parameter values, credentials, arbitrary user messages, and +raw process output. Command output and local logs can contain sensitive data and require an explicit +retention and access policy. + +## Failure semantics + +A start without a terminal fact is unknown or presumed lost. Consumers must not fabricate success +or failure. Sink delivery is bounded and best effort; destination failure does not change successful +warehouse work, and successful delivery does not prove the command succeeded. + +See [Typed Sinks](observability/sinks.md) to export lifecycle and command-output records. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md new file mode 100644 index 0000000000..cf4d2d323f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -0,0 +1,89 @@ + + +# Typed Sinks + +> Export lifecycle facts and command output through project-owned providers. + +Online: https://docs.sqlbuild.com/concepts/observability/sinks + +Project sinks consume one declared record type. Lifecycle and command-output streams remain separate +even when they share a provider and destination. + +## Lifecycle sink + +```python +# sinks/publish.py +from providers.destination_client import DestinationClient +from sqlbuild.sinks import ( + LifecycleEvent, + LifecycleEventKind, + lifecycle_event_sink, + lifecycle_event_to_json, +) + +@lifecycle_event_sink( + event_kinds={ + LifecycleEventKind.INVOCATION, + LifecycleEventKind.RUN, + LifecycleEventKind.RESOURCE, + LifecycleEventKind.AUDIT, + } +) +def publish_lifecycle(event: LifecycleEvent, destination_client: DestinationClient) -> None: + destination_client.publish( + route="sqlbuild.lifecycle.v2", + key=event.invocation_id, + payload=lifecycle_event_to_json(event).encode("utf-8"), + ) +``` + +Using `invocation_id` as a partition key keeps one invocation on one Kafka partition. Keep +`event_id` in the payload as the deduplication identity. Confirm the destination topic is not +compacted when complete lifecycle history must remain replayable. + +## Command-output sink + +```python +from sqlbuild.sinks import CommandOutputRecord, command_output_sink, command_output_to_json + +@command_output_sink(streams={"stdout", "stderr"}) +def publish_output(record: CommandOutputRecord, destination_client: DestinationClient) -> None: + destination_client.publish( + route="sqlbuild.command_output.v1", + key=record.record_id, + payload=command_output_to_json(record).encode("utf-8"), + ) +``` + +Command output groups adjacent text into bounded byte chunks and flushes on elapsed time, stream +change, size, or close. A loss record reports bounded-queue drops. This stream is useful for remote +transcripts but is not lifecycle evidence. + +## Configuration + +Runtime configuration can narrow declaration filters: + +```toml +[sinks.lifecycle] +event_kinds = ["run", "resource", "operation", "statement", "audit"] +min_severity = "info" + +[sinks.lifecycle.named.publish_lifecycle] +event_kinds = ["resource", "statement"] +min_severity = "warning" +``` + +SQLBuild owns envelope validation, local queueing, filtering, and bounded dispatch. The project owns +destination credentials, serialization, routes/topics, acknowledgements, retries, retention, and +durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. + +## Delivery behavior + +- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. +- Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. +- A failing or timed-out sink is isolated from command correctness. +- Provider setup and declaration errors fail before execution because the project configuration is + invalid. +- Providers shared by multiple sinks are set up and torn down once per command. + +Use lifecycle facts—not command-output records—to build execution state and timelines. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md new file mode 100644 index 0000000000..de2aad4ccc --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md @@ -0,0 +1,150 @@ + + +# Planning and change detection + +> How SQLBuild explains build work, detects changes, and chooses safe model actions. + +Online: https://docs.sqlbuild.com/concepts/planning + +## Contents + +- Reading a plan +- What SQLBuild compares +- Change reasons +- Cascades and incremental actions +- Selection and coherence +- Recorded state +- Related topics + +`sqb plan` previews the work for a build without executing it. `sqb build` uses the same planner +before running the selected resources. + +Planning and selection are separate concerns: + +- **Selection** determines which resources are in scope. By default, that is the whole project; use + `--select` and `--exclude` to choose a smaller scope. +- **Change detection** compares compiled resources with recorded warehouse state. It explains why a + resource needs work, chooses the correct action for incremental models, propagates upstream + changes, and warns when a partial selection would be incoherent. + +In the default **direct mode**, a build runs its selected scope. Change detection still matters: the +plan can show query and configuration differences, calculate replay or full-refresh actions, and +identify stale upstreams. It does not filter a direct build to stale models. + +[Virtual environments](virtual-environments.md) additionally bind each model to a versioned +physical relation. Virtual builds can use those bindings to reuse existing versions or limit an +invocation to stale work. That virtual execution behavior is documented under +[Building in a virtual environment](virtual-environments/building.md#stale-driven-execution). + +## Reading a plan + +Run the plan command with the same target and selectors you intend to build: + +```bash +sqb plan +sqb --target dev plan --select +fact_orders +``` + +A plan answers four questions: + +1. Which resources are selected? +2. What does SQLBuild know about their current warehouse state? +3. Why does each selected model need work? +4. What materialization or incremental action will the build take? + +Plans may include query diffs, schema changes, backfill ranges, source freshness signals, cascade +reasons, and warnings about stale dependencies. Planning does not modify model relations. + +## What SQLBuild compares + +### Models and functions + +Each model and function has a version identity derived from the inputs that affect its result: + +- normalized SQL after macro expansion and reference resolution; +- materialization settings, contracts, rendered SQL hooks, Python hook versions, and other + version-relevant configuration; +- referenced function identities; and +- upstream version identities where dependency changes must propagate. + +A function change therefore appears on the function itself and on dependent models. + +### Seeds + +Seeds use a content hash plus load-affecting configuration. SQLBuild can distinguish an unchanged +seed from one whose file or loading behavior changed. + +### Python nodes + +Loaders, tasks, assets, checks, and Python hooks have identities derived from project-owned source, +transitive project dependencies, and decorator configuration. Python hook identities also +participate in the identity of models that invoke them. + +For standalone Python nodes, this identity is primarily planning evidence. SQLBuild cannot observe +arbitrary external inputs such as APIs or files, so a node controls its own no-work decision with +`ctx.skip()` rather than relying on SQL identity alone. + +### Sources + +[Source freshness](planning/source-freshness.md) records an observable data version for a +source. A new observation can mark downstream models as affected even when their SQL is unchanged. + +## Change reasons + +The plan assigns reasons to explain detected work: + +| Reason | Meaning | +|--------|---------| +| First run | No previous identity exists for the resource | +| Query changed / checksum changed | Compiled SQL or seed content differs from the recorded identity | +| Config changed | Version-relevant configuration differs | +| Schema changed | A relevant warehouse or upstream schema changed | +| Function changed | A referenced function has a different identity | +| Upstream changed | A dependency change propagated to this model | +| Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | + +These reasons explain the plan; they are not a promise that direct mode will skip entries labelled +current. Direct builds execute the selected scope. Virtual mode can compare expected identities with +the versions already bound to a VDE and avoid unnecessary version creation. + +## Cascades and incremental actions + +Changes propagate through the DAG in dependency order. The resulting action depends on the +materialization: + +- views are recreated when selected for a direct build; +- tables rebuild when selected; +- incremental models use their cursor state, backfill policy, and `replay_on_change` setting to + determine the affected range; and +- a changed upstream can alter the action of downstream models even when their own SQL is unchanged. + +See [Cascade propagation](planning/cascade-propagation.md) for the detailed materialization +rules. + +## Selection and coherence + +SQLBuild reasons about dependencies outside a scoped selection without silently adding them to the +plan. If building the selected resources would use a stale or missing upstream, the planner warns or +blocks instead of presenting an incoherent partial build as current. + +See [Selection and staleness](planning/selection-and-staleness.md) for closure selectors and +stale-upstream handling. + +## Recorded state + +Direct mode records append-only planning evidence in the warehouse: + +- `_sqlbuild_fingerprints` stores resource identities; +- `_sqlbuild_source_freshness` stores source observations; and +- `_sqlbuild_node_results` stores Python node outcomes. + +The planner reads the latest applicable facts and appends new facts after successful work. Virtual +mode stores version bindings and environment-scoped planning state in its configured state backend; +see [Virtual environment setup](virtual-environments/setup.md). + +## Related topics + +- [Cascade propagation](planning/cascade-propagation.md) +- [Source freshness](planning/source-freshness.md) +- [Selection and staleness](planning/selection-and-staleness.md) +- [Building in a virtual environment](virtual-environments/building.md) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md new file mode 100644 index 0000000000..a6fdd644fb --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md @@ -0,0 +1,41 @@ + + +# Cascade propagation + +> How a change signal propagates downstream through the DAG, and how each materialization type responds. + +Online: https://docs.sqlbuild.com/concepts/planning/cascade-propagation + +When a model or function changes, the change signal propagates downstream through the DAG. Every model downstream of a changed node is marked `Upstream changed` in the plan, even if its own SQL and config are identical. + +The cascade walk is topological: it processes models in dependency order, so each model sees the resolved state of all its upstreams before deciding its own effective action. + +## What cascades + +- A query, config, or schema change on any model cascades to all its downstream dependents. +- A function change cascades to every model that calls it (directly or transitively). +- Source freshness changes propagate downstream the same way. See [Source freshness](source-freshness.md). + +## How materialization types respond + +- **Views** are recreated on every build regardless, so a cascade has no extra cost. +- **Tables** are fully rebuilt, same as if they had changed themselves. +- **Incremental models** receive a replay window from the cascade. A `full` replay always cascades. A `bounded` replay only cascades when the upstream and downstream models share the same `cursor_type` (e.g. both use `timestamp`), so unrelated cursor types don't inherit bounded windows that don't apply. The downstream model's own `replay_on_change` policy takes precedence over any cascaded signal. + +## Resolution when multiple upstreams are stale + +If a model has multiple stale upstreams with different replay actions, the most aggressive action wins. `full` beats `bounded`, and among bounded actions, the longer duration wins. Ties are broken alphabetically by model name for determinism. + +## Overriding cascaded replay + +Downstream incremental models can set their own `replay_on_change` policy to override the cascaded signal. If a downstream model declares its own policy, that policy applies instead of the upstream's. If it has no policy, it inherits the upstream's replay scope. + +## Replay on change + +When a change is detected on an incremental model, `replay_on_change` controls how much data to reprocess: + +- **`forward`** (default) - run the normal incremental delta from the cursor. No reprocessing. +- **`bounded-`** (e.g. `bounded-14d`) - replay the specified window of data. +- **`full`** - drop and rebuild the entire table. + +See [Incremental Models: Replay on change](../incremental.md#replay-on-change) for configuration details. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md new file mode 100644 index 0000000000..c548b17ad5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md @@ -0,0 +1,73 @@ + + +# Selection and staleness + +> How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. + +Online: https://docs.sqlbuild.com/concepts/planning/selection-and-staleness + +Change detection is selection-aware. When you scope a run with `--select`, SQLBuild only runs (and only displays) the resources you selected, but it still reasons about the *whole* graph to keep the result honest. The key case it handles: a selected model whose upstream changed but is **not** in the selection. + +## Only selected resources appear in the plan + +For a scoped run like `sqb dbt build --select agg_daily_revenue`, the plan shows only the selected resources, not their passive upstream closure. SQLBuild still walks upstreams internally for change propagation (a selected model flips to run when an upstream changed), but those passive upstreams are not listed as plan entries or counted in the header. + +`--full-refresh` is likewise scoped to the selected models, not the expanded upstream closure. + +## Stale warnings instead of silent partial rebuilds + +If a selected model has an upstream that changed but is **outside the selection**, rebuilding the selected model alone would run it on top of a stale upstream and silently produce a result that looks current but isn't. SQLBuild does not do that. It leaves the selected model current (it is not rebuilt) and emits a stale warning naming the changed upstreams, with the selector you can use to incorporate them. + +Suppose `stg_orders` (upstream) changed, and you select only the downstream `agg_daily_revenue`: + +```bash +sqb dbt plan --select agg_daily_revenue +``` + +``` +Plan ready (1 selected resources) + +dbt (1 selected resources) + planned models: 0 run, 1 current, 0 blocked + + Model plan + Current (1) + model.jaffle_analytics.agg_daily_revenue no change + +Warnings (1) + - selected dbt model 'agg_daily_revenue' is stale: upstream stg_orders changed + but will not be rebuilt or is stale; rebuild with a closure selector + (e.g. +model) to incorporate it +``` + +The model stays current rather than being rebuilt on a stale input, and the warning tells you exactly which upstreams changed and how to pull them in. + +## Incorporating the changed upstreams + +Use a closure selector to include the changed upstreams in the run. With `+agg_daily_revenue`, SQLBuild pulls in the changed `stg_orders`, cascades the change through the intermediate models, and rebuilds: + +```bash +sqb dbt plan --select +agg_daily_revenue +``` + +``` +Plan ready (8 selected resources) + planned models: 5 run, 1 current, 0 blocked + + Model plan + Checksum changed (1) + model.jaffle_analytics.stg_orders checksum changed + Upstream changed (4) + model.jaffle_analytics.agg_daily_revenue upstream changed + model.jaffle_analytics.fct_orders upstream changed + model.jaffle_analytics.int_order_payments upstream changed + model.jaffle_analytics.stg_order_statuses upstream changed +``` + +## Seeds follow the same rule + +Seed changes are tracked the same selection-aware way. A changed seed that is outside the current selection does not trigger a partial rebuild of its dependents; instead the dependents are left current and a stale warning is emitted, exactly as for models. This keeps a scoped run from quietly building on a seed that the run didn't reload. + +## Why this matters + +A run that reports success but silently built a model on stale inputs is a correctness hazard - the data looks fresh but isn't. Surfacing staleness as an explicit warning (and refusing the misleading partial rebuild) keeps scoped runs honest: you either get a result built on current inputs, or a clear warning telling you it would not be, with the fix. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/source-freshness.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/source-freshness.md new file mode 100644 index 0000000000..af8cae2b16 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/source-freshness.md @@ -0,0 +1,42 @@ + + +# Source freshness + +> Observe external source changes and propagate them through planning. + +Online: https://docs.sqlbuild.com/concepts/planning/source-freshness + +Source freshness lets SQLBuild observe whether external source data changed between runs. A changed +observation propagates through the dependency graph, so plans can explain which downstream models +are affected and choose the correct incremental action. In a virtual environment, the same signal +also participates in [stale-driven execution](../virtual-environments/building.md#stale-driven-execution). + +## Configuration + +Source freshness is configured per source in `sources/*.yml` with a `freshness:` block. See [Sources: Source freshness](../sources.md#source-freshness) for the full configuration reference. + +## How observations work + +During planning, SQLBuild observes the current data version of each source that has freshness configured (or that the adapter can observe automatically): + +1. **Observe** - query the source's current data version using the configured strategy. +2. **Compare** - compare the observed version against the last recorded observation from `_sqlbuild_source_freshness` in the target schema. +3. **Propagate** - walk the DAG downstream from changed or unknown sources to identify which models are affected. + +Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata and the source has a physical table (not an expression source, not a managed source). + +## Lag tolerance + +For timestamp-based freshness, `lag_tolerance` controls how much the observed value can drift before being considered a real change. If the current timestamp is within the tolerance of the previous observation, the source is treated as unchanged. This is useful for sources where the freshness timestamp moves by seconds or minutes on every query but the underlying data hasn't meaningfully changed. + +## State storage + +In direct mode, source freshness observations are appended to `_sqlbuild_source_freshness` in each +target schema. In virtual mode, observations are scoped to the VDE in the configured state backend. +In both modes, failed work does not replace the prior successful observation, so the next plan still +sees the pending source change. + +Direct observations are resolved across all target schemas in the project, so a source referenced by +models in different schemas is tracked consistently. + +Use [`sqb freshness`](../../cli/freshness.md) to observe source freshness on demand without triggering a build. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md new file mode 100644 index 0000000000..177a0e55dc --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -0,0 +1,498 @@ + + +# Project Configuration + +> Configure your SQLBuild project with sqlbuild_project.toml and sqlbuild_local.toml. + +Online: https://docs.sqlbuild.com/concepts/project-configuration + +## Contents + +- sqlbuild_project.toml +- Targets +- Defaults +- Constants +- Path defaults +- Settings +- Rules +- Project variables +- Janitor +- Scenario +- dbt +- Skills +- sqlbuild_local.toml + +SQLBuild projects are configured with two files in the project root: + +- **`sqlbuild_project.toml`** - shared project configuration, committed to version control +- **`sqlbuild_local.toml`** - local developer overrides, gitignored + +Macro, constant, and enum visibility is not configured in either file. Declaration scopes use filesystem conventions; see [Declaration Scopes](declaration-scopes.md). + +Most projects need only one committed `sqlbuild_project.toml`. Define shared targets such +as `dev` and `prod` there, including clone policies and team-wide defaults. Do not maintain +separate complete project files for each environment. + +Create `sqlbuild_local.toml` only when a developer or execution environment needs different +target selection, credentials, schemas, adapter settings, or variables. SQLBuild loads it +automatically and merges its explicitly configured values over the shared project config. + +```text +project/ + sqlbuild_project.toml # committed: shared targets and behavior + sqlbuild_local.toml # gitignored: this developer's overrides +``` + +Add the local file to `.gitignore`: + +```gitignore +sqlbuild_local.toml +``` + +## sqlbuild_project.toml + +A complete example: + +```toml +name = "waffle_shop" +adapter = "duckdb" +default_target = "dev" + +[connections.local] +database = "waffle_shop_control.duckdb" + +[settings] +default_audit_severity = "warn" + +[defaults] +materialized = "table" + +[constants] +collection_rendering = "value_list" + +[targets.prod] +connection = "local" +schema = "prod" + +[targets.dev] +connection = "local" +schema = "dev" + +[path_defaults.staging] +materialized = "view" +``` + +### Required fields + +| Field | Description | +|-------|-------------| +| `name` | Project name. Used in fingerprint tracking and manifest generation. | +| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, or `sqlserver`. See [Adapters](adapters.md). | +| `default_target` | Name of the target to build against when none is selected (see [Targets](#targets)). | + +### Named connections + +Define reusable connections under `[connections.]`, then reference one by name from +each target. Connections own endpoint, authentication, and compute settings. Targets own the +authoritative `database`, `schema`, variables, and operational policy. + +```toml +[connections.local] +database = "my_project.duckdb" + +[targets.dev] +connection = "local" +schema = "dev" +``` + +Multiple targets can reuse one connection while keeping separate namespaces and policies: + +```toml +[connections.warehouse] +account = "my_org-my_account" +user = "${ENV:SNOWFLAKE_USER}" +password = "${ENV:SNOWFLAKE_PASSWORD}" +warehouse = "TRANSFORM_WH" + +[targets.prod] +connection = "warehouse" +database = "ANALYTICS" +schema = "PROD" + +[targets.dev] +connection = "warehouse" +database = "ANALYTICS" +schema = "DEV_ALICE" +``` + +A `database` or `schema` present in a named connection is connection/session metadata only; +it does not satisfy the mandatory namespace strategy for a named target. Put the target's +authoritative database and schema on `[targets.]`. SQLBuild validates connection +references while loading configuration, without opening a warehouse connection, and reports +an unknown `targets..connection` name as an offline configuration error. + +For migration only, SQLBuild still maps legacy `[connection]` to an implicit connection and +legacy `[targets..connection]` blocks to target-specific implicit connections. These +forms are compatibility syntax, not the canonical format for new or updated projects. + +## Targets + +A target is a named build context - the database and schema you build into, plus execution policy (for example `dev` and `prod`). Each target references a named connection and can configure: + +| Field | Description | +|-------|-------------| +| `schema` | Schema for all models in this target; required for named targets | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | +| `database` | Database for all models in this target | +| `connection` | Name from `[connections.]` used for endpoint, authentication, and compute | +| `vars` | Target-specific project variables | +| `defer_sources_to` | Target name to read managed source data from (see [Loaders](python-nodes/loaders.md#source-deferral)) | +| `clone` | Clone policy (see below) | + +```toml +[connections.warehouse] +account = "my_org-my_account" +warehouse = "TRANSFORM_WH" + +[targets.prod] +connection = "warehouse" +database = "analytics" +schema = "analytics_prod" +loader_schema = "raw_prod" +defer_sources_to = "prod" + +[targets.dev] +connection = "warehouse" +database = "analytics" +schema = "analytics_dev" +loader_schema = "raw_dev" +defer_sources_to = "prod" + +[targets.staging] +connection = "warehouse" +database = "analytics" +schema = "staging" + +``` + +Managed loader writes use the active target's `loader_schema`, falling back to its model +`schema`. Managed source reads use the target named by `defer_sources_to`, or the active +target itself when deferral is omitted. In the example, `load --target dev` writes to +`raw_dev`, while models built in dev read from `raw_prod`. + +SQLBuild rejects targets on the same warehouse/database when their managed loader writes +resolve to the same schema. Two targets may read the same schema through deferral, but they +cannot both own loader writes there. + +### Selecting a target + +The active target is determined by (in order of precedence): + +1. `--target` on the command line (highest priority) +2. `sqlbuild_local.toml` `target` field +3. `default_target` in `sqlbuild_project.toml` +4. No target (models build to the default schema) + +A typical developer keeps shared target definitions in `sqlbuild_project.toml` and selects +their normal target once in the optional local file: + +```toml +# sqlbuild_local.toml +target = "dev" + +[targets.dev] +connection = "warehouse" +schema = "dev_alice" +loader_schema = "raw_alice" + +[connections.warehouse] +user = "alice" +password = "${ENV:SNOWFLAKE_PASSWORD}" +``` + +Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` +still takes precedence for that invocation. + +### Clone policies + +Targets can declare whether they allow cloning to or from: + +```toml +[targets.prod] +schema = "prod" + +[targets.prod.clone] +allow_as_clone_origin = true +allow_as_clone_destination = false + +[targets.dev] +schema = "dev" + +[targets.dev.clone] +allow_as_clone_origin = false +allow_as_clone_destination = true +``` + +Both policies default to `false`. `sqb clone --from prod --to dev` requires `allow_as_clone_origin = true` on `prod` and `allow_as_clone_destination = true` on `dev`. + +## Defaults + +Project-wide model defaults. Any field you can set in a `MODEL()` header can be set here as a default: + +```toml +[defaults] +materialized = "table" +incremental_strategy = "delete_insert" +replay_on_change = "full" +tags = ["managed"] +``` + +These apply to all models unless overridden by path defaults or the model's own `MODEL()` header. + +## Constants + +Collection constants default to parenthesized SQL value lists. Set a project-wide default when lists and sets should instead compile to first-class adapter-native arrays: + +```toml +[constants] +collection_rendering = "array" +``` + +`collection_rendering` accepts `value_list` (the SQLBuild default) or `array`. A public constant's `render_as` field or a model-local `constant(...)` wrapper overrides the project setting. The complete precedence order is declaration override, project setting, then `value_list`. + +This setting does not make unsupported adapter features portable. In particular, SQL Server rejects native arrays, and BigQuery rejects nested arrays. See [Collections and Rendering](constants/collections-and-rendering.md#project-default) for syntax, adapter output, and value-list usage constraints. + +## Path defaults + +Per-directory model defaults. Useful for applying different config to different parts of your project: + +```toml +[path_defaults."models/staging"] +materialized = "view" +tags = ["staging"] + +[path_defaults.marts] +materialized = "table" +tags = ["marts"] +replay_on_change = "full" +``` + +Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the +`staging` path default. + +### Config layering order + +Configuration is layered in this order, with later layers overriding earlier ones: + +1. **Project defaults** (`defaults`) +2. **Path defaults** (`path_defaults`) - if the model's path matches +3. **MODEL() header** - the model's own config + +Most keys are overridden by the more specific layer, but three merge instead: + +- `tags` are *unioned* across layers. A model with `tags [marts]` in its header that matches a path default with `tags [managed]` will have both tags. +- `row_diff_exclude_columns` lists are unioned across layers. +- `row_diff_tolerances` mappings are deep-merged across layers, so a header tolerance for one column adds to (rather than replaces) tolerances declared in defaults or path defaults. + +Diff sampling values use ordinary replacement precedence. `row_diff_sample_rows = 0` explicitly +disables an inherited sample for a path or model. CLI `--sample-rows`, `--sample-seed`, and +`--exhaustive` override the compiled configuration for one invocation. + +## Settings + +Global feature toggles: + +```toml +[settings] +sql_analysis = true +query_change_tracking = true +column_contract_mode = "implicit" +concurrency = 1 +microbatch_concurrency = false +auto_load_sources = true +table_promotion_mode = "staged" +default_audit_severity = "warn" +default_audit_run_scope = "final" +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | +| `changes_only` | `false` | In virtual mode, limit `plan` and `build` to [stale-driven execution](virtual-environments/building.md#stale-driven-execution) without passing `--changes-only`. Rejected in direct mode. Can also be set per target under `[targets.]`. The CLI flag takes precedence, then the selected target, local settings, and this project setting. | +| `virtual_environments` | `false` | Enable [virtual environments](virtual-environments.md) (versioned model outputs, promotion, rollback, state management). When `false`, the project runs in direct mode. | +| `query_change_tracking` | `true` | Track query fingerprints for change detection | +| `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | +| `concurrency` | `1` | Maximum parallel model execution (currently serial only) | +| `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | +| `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | +| `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | +| `default_audit_severity` | `warn` | Default severity for audits: `warn` or `error` | +| `default_audit_run_scope` | `final` | Default run scope for audits: `final` or `delta_and_final` | + +### Table promotion mode + +- **`staged`** (default for most adapters): Materializes into a staging table, runs audits, then swaps into the target. If audits fail, the production table is untouched. +- **`immediate`**: Creates the table directly at the target location. Audits run after materialization. Simpler but no pre-promotion safety net. + +## Rules + +Rules configuration belongs in the shared `sqlbuild_project.toml` so local and CI compilation use +the same checks: + +```toml +[rules] +select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] +``` + +Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in +codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See +[Compiler-integrated Rules](rules.md) for configuration, authoring, and suppressions. + +## Project variables + +Variables are simple string substitutions available in model SQL via the `@@name` syntax: + +```toml +[vars] +schema_prefix = "analytics" +retention_days = "90" +``` + +Target-specific variables override project-level ones: + +```toml +[vars] +schema_prefix = "analytics" + +[targets.prod.vars] +schema_prefix = "prod_analytics" +``` + +See [Macros](macros.md) for details on variable substitution and how variables interact with macros. + +## Janitor + +Configuration for the `sqb janitor` command, which cleans up stale warehouse relations: + +```toml +[janitor] +enabled = false +retention_days = 30 +delete_tracked_only = true +exclude_patterns = ["audit_*", "tmp_*"] +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `enabled` | `false` | Whether janitor is active | +| `retention_days` | `30` | Only clean relations older than this many days | +| `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | +| `exclude_patterns` | `[]` | Glob patterns for relations to skip | + +## Scenario + +Configuration for scenario snapshot capture safety limits and local type overrides: + +```toml +[scenario.snapshot_limits] +max_rows_per_relation = 10000 +max_total_rows = 50000 +max_bytes_per_relation = 10485760 +max_total_bytes = 52428800 +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `max_rows_per_relation` | none | Maximum rows per captured relation | +| `max_total_rows` | none | Maximum total rows across all relations in one scenario | +| `max_bytes_per_relation` | none | Maximum JSONL bytes per relation file | +| `max_total_bytes` | none | Maximum total JSONL bytes per scenario | + +Local type overrides for DuckDB replay are configured per adapter dialect: + +```toml +[scenario.local_type_overrides.snowflake] +"OBJECT" = "JSON" +"ARRAY" = "JSON" +``` + +See [Scenarios](scenarios.md) for details on local type overrides and capture limits. + +## dbt + +Configuration for running SQLBuild alongside an existing dbt project: + +```toml +[dbt] +project_dir = "../dbt_project" +profiles_dir = "../profiles" +target_path = "../dbt_project/target" +target = "dev" +``` + +| Field | Description | +|-------|-------------| +| `project_dir` | Path to the dbt project root (where `dbt_project.yml` lives) | +| `profiles_dir` | Path to the directory containing `profiles.yml` | +| `target_path` | Path to dbt's `target/` directory (where `manifest.json` is written) | +| `target` | dbt target name override (optional) | + +Paths can be absolute or relative to the SQLBuild project root. See [Using SQLBuild with dbt](dbt-compatibility/overview.md) for setup and usage details. + +## Skills + +Configuration for AI agent skill file installation: + +```toml +[skills] +targets = ["agents", "claude"] +auto_update = false +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `targets` | `["agents", "claude"]` | Agent targets to install. OpenCode consumes `.agents`; use `opencode` only as an explicit override. | +| `auto_update` | `false` | Refresh stale SQLBuild-owned generated files from the installed package during normal commands; custom collisions are never overwritten. | + +See [skills CLI reference](../cli/skills.md) for usage details. + +## sqlbuild_local.toml + +Local developer overrides. This optional file is loaded automatically and should be +gitignored. Only put values that differ from the shared project configuration here. + +```toml +target = "dev" + +[targets.dev] +schema = "dev_alice" +loader_schema = "raw_alice" + +[connections.local] +database = "my_local.duckdb" + +[settings] +sql_analysis = false +concurrency = 4 + +[vars] +debug_mode = "true" +``` + +| Field | Description | +|-------|-------------| +| `target` | Override which target is active for this developer | +| `adapter` | Override the database adapter (e.g. use DuckDB locally while prod uses Snowflake) | +| `connections` | Override named connection fields; entries merge by connection name | +| `settings` | Override global settings (only explicitly set fields take effect) | +| `vars` | Developer-specific variable overrides (merged on top of project + target vars) | + +Project and local configuration merge named connections by name and merge their explicitly +configured fields. Target blocks merge the same way and may override the connection reference, +`database`, `schema`, `loader_schema`, variables, source deferral, and policy fields. Unspecified +values continue to come from `sqlbuild_project.toml`; a local reference to an unknown merged +connection still fails offline during configuration loading. + +This replaces the common dbt pattern of switching profiles or setting environment variables +to change targets. Each developer sets their target, named connection, and preferences once in +`sqlbuild_local.toml` and it persists across sessions. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md new file mode 100644 index 0000000000..1837d0b325 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -0,0 +1,146 @@ + + +# Assets + +> Produce or observe external artifacts as nodes in the SQLBuild graph. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/assets + +## Contents + +- Defining an asset +- Materialization +- Dependencies +- Columns and column lineage +- Skipping and retries +- Decorator parameters +- Running assets + +Assets are Python nodes that produce or observe an external artifact - an exported file, a published dataset, a dashboard refresh, an ML model. They behave like [tasks](tasks.md) but add dataset-like metadata (columns, column lineage) and a materialization flag. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. + +## Defining an asset + +Place Python files under `assets/` and decorate functions with `@asset`: + +```python +# assets/exports.py +from sqlbuild.assets import asset, AssetContext + +@asset +def orders_export(ctx: AssetContext): + path = write_orders_csv() + return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) +``` + +The asset receives an `AssetContext` and returns through `ctx.result(...)`. + +## Materialization + +Assets record whether they actually produced an artifact via `materialized`: + +```python +@asset +def orders_export(ctx: AssetContext): + if nothing_changed(): + return ctx.result(metadata={"status": "unchanged"}, materialized=False) + return ctx.result(metadata={"path": export()}, materialized=True) +``` + +- `materialized=True` - the artifact was produced this run. +- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. + +Only assets have `materialized`; tasks do not. + +## Dependencies + +Assets declare dependencies with `depends_on` (a single function, tuple, or list), and may depend on tasks, assets, and loaders: + +```python +from sqlbuild.assets import asset +from tasks.orders import export_orders + +@asset(depends_on=export_orders) +def orders_dashboard(ctx): + result = ctx.result_of(export_orders) + return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) +``` + +To read a SQL model or source, declare a typed reference and resolve it at runtime - see [SQL references](sql-references.md): + +```python +from sqlbuild.assets import asset +from sqlbuild.refs import model + +@asset(depends_on=model("fact_orders")) +def orders_extract(ctx): + relation = ctx.relation(model("fact_orders")) + rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] + return ctx.result(metadata={"rows": rows}, materialized=True) +``` + +## Columns and column lineage + +Assets can declare a schema for catalog and lineage purposes. This does not enforce anything at runtime; it describes the artifact: + +```python +@asset( + columns=[ + {"name": "order_id", "type": "INTEGER"}, + {"name": "total_cents", "type": "INTEGER", "nullable": False}, + ], + column_lineage={ + "order_id": [{"node": "fact_orders", "column": "order_id"}], + "total_cents": [{"node": "fact_orders", "column": "total_cents"}], + }, +) +def orders_export(ctx): + ... +``` + +- `columns` - column declarations with `name`, optional `type`, `nullable`, `description`, and `meta`. +- `column_lineage` - maps each asset column to upstream `{node, column}` references, surfaced in the DAG artifact and integrations. + +Tasks and checks do not support `columns` or `column_lineage`. + +## Skipping and retries + +Assets support the same `ctx.skip(...)` and `retry` behavior as tasks: + +```python +from sqlbuild.assets import asset, SkipMode +from sqlbuild.retries import RetryPolicy + +@asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) +def export(ctx): + if not ready(): + return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT + return ctx.result(metadata={"path": do_export()}, materialized=True) +``` + +See [Tasks](tasks.md#retries) for the full retry policy fields. + +## Decorator parameters + +| Parameter | Description | +|-----------|-------------| +| `name` | Override the node name (defaults to the function name) | +| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs | +| `columns` | Column declarations for the produced artifact | +| `column_lineage` | Map of asset column to upstream `{node, column}` references | +| `tags` | Labels for selection and grouping | +| `group` | Display/catalog grouping | +| `description` | Docs (defaults to docstring) | +| `meta` | Freeform JSON metadata | +| `retry` | A `RetryPolicy` | + +## Running assets + +```bash +# Run an asset and its required graph +sqb build --select orders_export --no-tests --no-audits + +# Include in a full build with its upstreams +sqb build --select +orders_export +``` + +Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md new file mode 100644 index 0000000000..b14f845951 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -0,0 +1,128 @@ + + +# Checks + +> Validate tasks, assets, and loaders with Python checks. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/checks + +## Contents + +- Defining a check +- Results +- Severity +- What checks can depend on +- Decorator parameters +- Running checks +- Checks vs audits + +Checks are Python nodes that validate other Python nodes. They are the Python analog of SQL [audits](../audits.md): audits validate SQL relations, checks validate the output of tasks, assets, and loaders. See [Python Nodes](overview.md) for the shared model. + +Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`. + +## Defining a check + +Place Python files under `checks/` and decorate functions with `@check`. `depends_on` is required: + +```python +# checks/orders.py +from sqlbuild.checks import check, CheckContext +from tasks.orders import export_orders + +@check(depends_on=export_orders) +def check_orders_exported(ctx: CheckContext): + result = ctx.result_of(export_orders) + if result.metadata.get("rows", 0) == 0: + return ctx.fail("no orders exported") + return ctx.pass_("orders exported") +``` + +The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. + +## Results + +Return a result through the context helpers, or a bool shorthand: + +```python +@check(depends_on=orders_asset) +def rows_present(ctx): + return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail +``` + +| Return | Meaning | +|--------|---------| +| `ctx.pass_(message=None, metadata=None)` | Passing | +| `ctx.fail(message, metadata=None)` | Failing, using the check's severity | +| `ctx.warn(message, metadata=None)` | Warning, regardless of severity | +| `True` | Pass | +| `False` | Fail | + +Returning `None` is not allowed - checks must be explicit. + +## Severity + +`@check` takes a `severity` of `error` (default) or `warn`: + +```python +@check(depends_on=export_orders, severity="warn") +def orders_freshness(ctx): + if stale(): + return ctx.fail("export is stale") # recorded as a warning, does not fail the build + return ctx.pass_() +``` + +- `error` (default) - a failing check fails `sqb build`. +- `warn` - a failing check is reported but does not fail the build. + +`ctx.warn(...)` always produces a warning regardless of the declared severity. + +## What checks can depend on + +- Checks may depend on **tasks, assets, and loaders**. +- Checks may **not** depend on SQL models, sources, seeds, or functions. Use SQL [audits](../audits.md) to validate SQL relations. +- Checks may **not** depend on other checks. +- Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead. + +A check that depends on a single node is displayed grouped under that node. Multi-dependency checks are shown as standalone validation nodes, grouped by `group`, tags, or path. + +## Decorator parameters + +| Parameter | Description | +|-----------|-------------| +| `depends_on` | Required. Tasks/assets/loaders to validate (function, tuple, or list) | +| `name` | Override the node name (defaults to the function name) | +| `severity` | `error` (default) or `warn` | +| `tags` | Labels for selection and grouping | +| `group` | Display/catalog grouping | +| `description` | Docs (defaults to docstring) | +| `meta` | Freeform JSON metadata | + +Checks do not support `columns`, `column_lineage`, or `retry`. + +## Running checks + +Checks run automatically during `sqb build` when their Python dependencies run. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md): + +```bash +# Run all checks +sqb check + +# Run a specific check (and its required dependencies) +sqb check --select +check_orders_exported + +# Run checks by tag +sqb check --select tag:exports +``` + +`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. + +## Checks vs audits + +| | Checks | Audits | +|---|--------|--------| +| Validates | Python tasks, assets, loaders | SQL relations | +| Authored in | `checks/` (Python) | `MODEL()` headers / `audits/` (SQL) | +| Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | +| Severity | `error`, `warn` | `error`, `warn` | + +`sqb build` runs both. `sqb audit` runs SQL audits only; `sqb check` runs Python checks only. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md new file mode 100644 index 0000000000..196aa0881d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -0,0 +1,97 @@ + + +# Factories + +> Generate Python nodes programmatically with @factory. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/factories + +A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [tasks](tasks.md), [assets](assets.md), [loaders](loaders.md), or [checks](checks.md) from a list, a config, or a loop, rather than hand-writing each one. + +Factories are an advanced feature. Most projects author nodes directly; reach for a factory when you find yourself copy-pasting near-identical node definitions. + +## Defining a factory + +A `@factory` function takes **no arguments** and returns one or more decorated node functions: + +```python +# factories/exports.py +from sqlbuild.factories import factory +from sqlbuild.assets import asset + +TABLES = ["orders", "customers", "payments"] + +@factory +def export_assets(): + nodes = [] + for table in TABLES: + @asset(name=f"export_{table}", tags=("export",)) + def export(ctx, table=table): + return ctx.result(metadata={"table": table}, materialized=True) + nodes.append(export) + return nodes +``` + +The returned nodes are discovered and added to the graph exactly as if you had written them by hand. They participate in selection, lifecycle, the DAG artifact, and integrations like any other node. + +A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. + +## Folder rules + +Where a factory lives determines what it is allowed to emit: + +| Location | May emit | +|----------|----------| +| `loaders/` | loaders only | +| `tasks/` | tasks only | +| `assets/` | assets only | +| `checks/` | checks only | +| `factories/` | any kind, including a mix | + +- A **single-kind** factory can live in that kind's folder (e.g. an asset-only factory in `assets/`), keeping it next to the nodes it generates. It can also live in `factories/`. +- A factory that emits **more than one kind** must live in `factories/`. + +This keeps each kind folder honest: everything in `assets/`, whether hand-written or factory-generated, is an asset. + +SQLBuild enforces this at discovery time. A factory in a kind folder that returns a foreign kind raises an error pointing you to `factories/`: + +``` +Factory export_pipeline in assets/ returned a loader 'raw_orders'; +mixed-kind factories must live in factories/. +``` + +The `factories/` folder is not created by `sqb init` - add it when you need it. + +## Mixed-kind factories + +A factory in `factories/` can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: + +```python +# factories/orders.py +from sqlbuild.factories import factory +from sqlbuild.loaders import loader +from sqlbuild.assets import asset +from sqlbuild.checks import check + +@factory +def orders_pipeline(): + @loader(name="raw_orders") + def load(ctx): + return fetch_orders() + + @asset(name="orders_export", depends_on=load) + def export(ctx): + return ctx.result(materialized=True) + + @check(depends_on=export) + def orders_export_check(ctx): + return ctx.pass_("export ready") + + return [load, export, orders_export_check] +``` + +Generated nodes follow the same rules as directly-authored ones, including the [SQL boundary](overview.md#the-sql-boundary): a factory-generated loader still binds to a managed source, and a factory-generated check still may not depend on SQL models. + +## Naming + +Generated nodes need unique names across the project. Pass an explicit `name=` to each node a factory creates (factories almost always generate names from a loop variable or config), since relying on the function's own name would produce duplicates. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md new file mode 100644 index 0000000000..ac9dd81c60 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -0,0 +1,391 @@ + + +# Loaders + +> Load external data into source tables with Python functions. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/loaders + +## Contents + +- How it works +- Defining a loader +- Binding to a source +- Write strategies +- Self-managed loaders +- Loader context +- Loader dependencies +- Decorator parameters +- Auto-load during builds +- Source deferral +- Schema evolution +- Project structure +- Config reference + +Loaders are Python functions that load data into source tables. They replace expression sources and manual ETL scripts with code that lives inside your project, runs as part of the build, and supports incremental write strategies. Loaders are one of the four [Python node](overview.md) kinds, and the only one that writes into a SQL source. + +## How it works + +1. Write a Python function under `loaders/` decorated with `@loader` +2. Declare a managed source in `sources/*.yml` with `managed: true` and **the same name as the loader function** +3. SQLBuild calls the function, writes returned rows to a staging table, then applies the configured write strategy to the target + +Loaders participate in the build lifecycle. When `sqb build` runs, managed sources are loaded before any dependent model is materialized. + +## Defining a loader + +Place Python files under `loaders/` in your project directory. Each file can contain one or more loader functions: + +```python +# loaders/raw_sources.py +from sqlbuild.loaders import loader +from sqlbuild.executor.load.models import LoaderContext + +@loader +def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: + return [ + {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, + {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, + ] +``` + +The function receives a `LoaderContext` and returns rows as a list of dicts, an iterator of dicts, or `None` for self-managed loaders. + +## Binding to a source + +Declare a managed source in `sources/*.yml`. A managed source is bound to the loader function **with the same name** - there is no separate `loader` field: + +```yaml +sources: + - name: raw_customers + managed: true + write_strategy: table + columns: + - name: id + type: INTEGER + - name: name + type: VARCHAR + - name: email + type: VARCHAR +``` + +Setting `managed: true` makes this a **managed source** - SQLBuild owns both the loading and the schema. The binding is by name: the source `raw_customers` is populated by the `@loader` function named `raw_customers`. SQLBuild raises an error if a managed source has no loader function of the same name. + +Models reference managed sources the same way as any other source: + +```sql +SELECT id, name FROM __source("raw_customers") +``` + +## Write strategies + +The `write_strategy` field controls how returned rows are written to the target table. + +### table + +Full replace. The target is dropped and recreated from the loader output on every run. + +```yaml +sources: + - name: raw_countries + managed: true + write_strategy: table + columns: + - name: country_id + type: INTEGER + - name: country_code + type: VARCHAR +``` + +### append + +Insert all returned rows into the target. No deduplication. + +```yaml +sources: + - name: raw_webhook_events + managed: true + write_strategy: append + columns: + - name: event_id + type: INTEGER + - name: event_name + type: VARCHAR +``` + +### delete_insert + +Delete rows in the cursor range, then insert replacements. Requires `cursor_column`. + +```yaml +sources: + - name: raw_order_events + managed: true + write_strategy: delete_insert + cursor_column: event_at + columns: + - name: event_id + type: INTEGER + - name: event_at + type: TIMESTAMP + - name: amount_cents + type: INTEGER +``` + +The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_column)` from the target, so it can fetch only new or updated data. Its function name matches the source name (`raw_order_events`): + +```python +@loader +def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + if ctx.current_cursor_value is None: + return fetch_all_events() + return fetch_events_since(ctx.current_cursor_value) +``` + +### merge + +Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. + +```yaml +sources: + - name: raw_customers + managed: true + write_strategy: merge + unique_key: customer_id + cursor_column: updated_at + columns: + - name: customer_id + type: INTEGER + - name: plan_name + type: VARCHAR + - name: updated_at + type: TIMESTAMP +``` + +Existing rows matching the unique key are updated; new rows are inserted. + +## Self-managed loaders + +If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader is responsible for writing data to the target itself, using whatever approach makes sense - `ctx.execute_sql()`, an external library, a subprocess, or anything else: + +```python +@loader +def raw_status(ctx: LoaderContext) -> None: + ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") + ctx.execute_sql( + f"CREATE TABLE {ctx.destination} AS " + "SELECT 1 AS status_id, 'loaded' AS status_name" + ) +``` + +The source is still declared as managed, just without a `write_strategy`: + +```yaml +sources: + - name: raw_status + managed: true + columns: + - name: status_id + type: INTEGER + - name: status_name + type: VARCHAR +``` + +Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool like [dlt](../../integrations/dlt.md), or handle writes in a way that doesn't fit the dict-return pattern. + +## Loader context + +Every loader function receives a `LoaderContext` as its first argument. It provides access to the destination relation, cursor state, active target, and helper methods. + +### Properties + +| Property | Type | Description | +|----------|------|-------------| +| `destination` | `str` | Fully-qualified destination relation name (where rows are written) | +| `destination_database` | `str \| None` | Destination database | +| `destination_schema` | `str \| None` | Destination schema | +| `destination_name` | `str` | Unqualified destination table name | +| `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | +| `run_id` | `str` | Unique identifier for this execution run | +| `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | +| `vars` | `dict` | Project variables (merged from project, target, and local config) | +| `is_reload` | `bool` | `True` when `--reload` was passed | +| `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | +| `end_cursor_ts` | `datetime \| None` | Timestamp cursor end override from `--end-cursor-ts` | +| `start_cursor_int` | `int \| None` | Integer cursor start override from `--start-cursor-int` | +| `end_cursor_int` | `int \| None` | Integer cursor end override from `--end-cursor-int` | +| `adapter` | `BaseAdapter` | The database adapter instance | +| `connection` | `object` | The active database connection | +| `logger` | `Logger` | Python logger scoped to the loader | + +### Methods + +| Method | Description | +|--------|-------------| +| `execute_sql(sql)` | Execute a SQL statement against the connection | +| `query(sql)` | Execute a SQL query and return the cursor | +| `log(message)` | Log a message to the execution lifecycle output | +| `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | +| `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | +| `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | +| `result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | +| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency | +| `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | + +### LoaderRelationRef + +Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream relation: + +| Property / Method | Description | +|-------------------|-------------| +| `destination` | Fully-qualified relation name | +| `current_cursor_value` | Current `MAX(cursor_column)` from the relation | +| `max(column)` | Return the `MAX` of any column from the relation | + +## Loader dependencies + +Loaders can depend on other loaders using `depends_on`. Dependencies are executed first, and their destination relations are available via `ctx.loader()`: + +```python +from sqlbuild.loaders import loader +from sqlbuild.executor.load.models import LoaderContext + +@loader +def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + return [ + {"account_id": 1, "account_name": "Pawnee Parks"}, + {"account_id": 2, "account_name": "Eagleton"}, + ] + +@loader(depends_on=[raw_accounts]) +def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + accounts = ctx.loader(raw_accounts) + rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") + return [ + {"account_id": row[0], "metric": "active"} + for row in rows.fetchall() + ] +``` + +Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. + +Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: + +```python +@loader(destination="staging.shared_accounts") +def raw_accounts(ctx: LoaderContext): + ... +``` + +## Decorator parameters + +The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence. + +| Parameter | Description | +|-----------|-------------| +| `depends_on` | List of loader functions this loader depends on | +| `destination` | Override the destination relation name (can include schema or database) | +| `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | +| `cursor_column` | Column used for incremental cursor tracking | +| `unique_key` | Column(s) used as the merge key (string or list of strings) | +| `columns` | Column specifications with name, type, nullable, and description | +| `contract` | `enforced` or `none` | + +## Auto-load during builds + +By default, `sqb build` automatically loads managed sources before building dependent models. This is controlled by the `auto_load_sources` setting: + +```toml +[settings] +auto_load_sources = true # default +``` + +You can also control this per-run with CLI flags: + +```bash +# Explicitly load sources before building +sqb build --load + +# Skip source loading +sqb build --no-load + +# Reload sources (passes is_reload=True to loaders) +sqb build --reload +``` + +When `--reload` is passed, `ctx.is_reload` is `True` in the loader function. This lets loaders implement different behavior for full reloads versus normal incremental loads. + +## Source deferral + +Loader writes and managed source reads are resolved separately. `loader_schema` controls +where a target's loaders write; `defer_sources_to` optionally selects another target whose +managed sources models should read. + +```toml +[targets.dev] +schema = "analytics_dev" +loader_schema = "raw_dev" +defer_sources_to = "prod" + +[targets.prod] +schema = "analytics_prod" +loader_schema = "raw_prod" +``` + +With this config: + +- `sqb load --target dev` writes to `raw_dev`. +- `sqb load --target prod` writes to `raw_prod`. +- Models built in dev read from `raw_prod` because dev defers source reads to prod. +- Models built in prod read from `raw_prod` because omitted deferral defaults to the active target. + +If `loader_schema` is omitted, loaders fall back to the active target's model `schema`, then +the adapter default schema. An explicit `schema` on a managed source overrides the target +default. SQLBuild validates final managed-loader write namespaces and rejects two targets +on the same warehouse/database that could write to the same schema. + +`defer_sources_to` never redirects loader writes. It only changes `__source()` and Python +`source()` reads. This lets developers test loaders in an isolated schema without making +development models consume partial development source data. + +## Schema evolution + +When a loader returns rows with columns not present in the existing target table, SQLBuild detects the schema change and adds the new columns automatically. Type mismatches between the staging table and the existing target raise an error. + +## Project structure + +``` +my-project/ + loaders/ + raw_sources.py # loader functions + api_sources.py # more loader functions + sources/ + raw.yml # managed source declarations (managed: true) + models/ + staging/ + stg_customers.sql # __source("raw_customers") +``` + +SQLBuild discovers all `.py` files under `loaders/` recursively (excluding `__init__.py` and files starting with `_`). Each file is scanned for functions decorated with `@loader`. + +## Config reference + +### Source YAML fields for managed sources + +| Field | Description | +|-------|-------------| +| `managed` | Set to `true` to bind the source to the `@loader` function of the same name | +| `write_strategy` | `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | +| `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | +| `unique_key` | Merge key column(s) (required for `merge`) | +| `columns` | Column declarations with types | +| `contract` | `enforced` or `none` | + +### Validation rules + +- `append` cannot have `unique_key` +- `merge` requires `unique_key` +- `table` cannot have `cursor_column` or `unique_key` +- `delete_insert` requires `cursor_column` and cannot have `unique_key` +- `cursor_column` requires one of `append`, `delete_insert`, or `merge` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md new file mode 100644 index 0000000000..d80f3cfe76 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -0,0 +1,177 @@ + + +# Overview + +> Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/overview + +## Contents + +- The SQL boundary +- Decorators +- Identity tracking +- Runtime context +- Returns and skips +- Result persistence +- Selection +- Lifecycle: run, build, check +- Try it + +Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. + +There are four kinds, authored in dedicated top-level folders: + +| Kind | Decorator | Folder | Purpose | +|------|-----------|--------|---------| +| Loader | `@loader` | `loaders/` | Load external data into a managed source | +| Task | `@task` | `tasks/` | Run Python computation or side effects | +| Asset | `@asset` | `assets/` | Produce or observe an external artifact | +| Check | `@check` | `checks/` | Validate tasks, assets, and loaders | + +All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: + +- [Loaders](loaders.md) - load data into managed sources +- [Tasks](tasks.md) - Python computation and side effects +- [Assets](assets.md) - external artifacts +- [Checks](checks.md) - Python validations +- [Factories](factories.md) - generate nodes programmatically with `@factory` +- [SQL references](sql-references.md) - read SQL models and sources from Python + +Nodes can also be generated programmatically with [`@factory`](factories.md) instead of authored one at a time. + +## The SQL boundary + +The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. + +- SQL models depend only on other SQL resources (models, sources, seeds, functions). +- Python nodes may depend on other Python nodes. +- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`. +- Python nodes may **read** SQL models and sources at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency. + +This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. + +``` + depends_on + task ----------------> task + | | + | (read only) v + | asset + v | + source <-- loader | check (validates tasks/assets/loaders) + | + v + model (SQL) ----> model (SQL) +``` + +## Decorators + +Every decorator accepts the same organizational metadata: + +| Argument | Description | +|----------|-------------| +| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()` references) | +| `tags` | Labels for selection, filtering, and catalog grouping | +| `group` | A display/catalog grouping string | +| `description` | Human-readable docs (defaults to the function docstring) | +| `meta` | Freeform JSON metadata for catalogs and integrations | + +`@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. + +## Identity tracking + +Every Python node is fingerprinted by source code hash, decorator config hash, and transitive dependency hashes (scoped to the git root, so third-party package updates don't affect identities). The plan shows source and dependency diffs when a node's identity changes, giving you visibility into what changed in your Python code. + +Unlike SQL models, Python nodes may depend on external inputs the framework cannot observe (APIs, files, third-party services). Skip/run decisions are therefore user-controlled via `ctx.skip()`: the node's own logic decides whether it needs to run. See [Planning and Change Detection](../planning.md#python-nodes) for details. + +## Runtime context + +Each node receives a context object as its first argument (`TaskContext`, `AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: + +| Helper | Description | +|--------|-------------| +| `ctx.run_id` | Unique identifier for this run | +| `ctx.target` | Active target name | +| `ctx.vars` | Project variables | +| `ctx.is_reload` | `True` when `--reload` was passed | +| `ctx.adapter` / `ctx.connection` | Adapter and live connection | +| `ctx.log(message)` | Log to the run output | +| `ctx.query(sql)` / `ctx.execute_sql(sql)` | Run SQL on the connection | +| `ctx.qualify_name(name)` | Qualify a relation name | +| `ctx.relation(ref)` | Resolve a declared `model()`/`source()` reference to a relation | +| `ctx.result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | +| `ctx.providers` | Access discovered [providers](providers.md) by name | + +Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. + +Providers can also be injected directly as function parameters by name. See [Providers](providers.md) for details. + +## Returns and skips + +Tasks and assets return through `ctx.result(...)`: + +```python +@task +def export_orders(ctx): + return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) +``` + +- A plain value or `None` is also accepted and normalized to a successful result. +- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. +- Assets may pass `materialized=True`/`False` to record whether an artifact was produced. + +Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. + +## Result persistence + +Node results (payload, metadata, status, errors) are persisted after each execution. In direct mode, results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. In virtual mode, results are stored in the VDE state backend scoped per environment. Results persist across runs, so they are available for observability, debugging, and downstream consumption. + +## Selection + +Python nodes are selected like SQL resources, by bare name or typed selector: + +```bash +sqb build --select export_orders # bare name +sqb build --select task:export_orders # typed +sqb build --select asset:orders_export +sqb build --select check:check_orders +sqb build --select tag:exports # by tag +sqb build --select +orders_export # with upstreams +``` + +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, and hooks. + +## Lifecycle: run, build, check + +Python nodes run in two phases relative to SQL: + +- **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. +- **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. + +The commands differ in what they include by default: + +| Command | SQL | Loaders / tasks / assets | Checks | Audits | +|---------|-----|--------------------------|--------|--------| +| `sqb build` | Yes | Yes | Yes | Yes | +| `sqb build --no-tests --no-audits` | Yes | Yes | No | No | +| `sqb check` | No | No | Selected checks only | No | + +- `sqb build` is the complete build-and-validate command: it runs SQL, the required Python nodes, SQL audits, and Python checks. +- `sqb build --no-tests --no-audits` executes the DAG without validation, for fast iteration. +- `sqb check` runs Python checks only. See [`sqb check`](../../cli/check.md). + +Use `--no-python` on `plan` and `build` to suppress read-side tasks/assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](../../cli/build.md) reference. + +## Try it + +The `python_nodes` playground is a small working project with a task, loader, model, asset, and check: + +```bash +sqb playground --template python_nodes +cd sqlbuild-playground +sqb build --select +fact_orders --select +orders_export +sqb check --select +check_orders_export +``` + +See [`sqb playground`](../../cli/playground.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md new file mode 100644 index 0000000000..0c02d37600 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -0,0 +1,209 @@ + + +# Providers + +> Shared runtime services for Python nodes and hooks. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/providers + +## Contents + +- Defining a provider +- Provider name +- Using providers in Python nodes +- Using providers in hooks +- Using providers via context +- Lifecycle +- Configuration from environment variables +- Discovery rules +- Plan output +- Project layout + +Providers are shared runtime services that SQLBuild discovers, configures, and injects into your Python nodes and hooks. Use them for external connections, API clients, or any stateful service that multiple nodes need access to. + +## Defining a provider + +Create a Python file under `providers/` in your project. A provider is a class that subclasses `Provider` from `sqlbuild.providers`: + +```python +# providers/warehouse_client.py +from sqlbuild.providers import Provider + +class WarehouseClient(Provider): + api_key: str + endpoint: str = "https://api.example.com" + + def setup(self, ctx): + self.session = create_session(self.api_key, self.endpoint) + + def teardown(self): + self.session.close() +``` + +`Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. + +## Provider name + +Each provider has a runtime name used for injection. By default, the name is derived from the class name by converting to `lower_snake_case`: + +- `WarehouseClient` becomes `warehouse_client` +- `SlackNotifier` becomes `slack_notifier` + +Override the name explicitly with `provider_name`: + +```python +class WarehouseClient(Provider): + provider_name = "warehouse" + api_key: str +``` + +Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and hooks. + +## Using providers in Python nodes + +Providers are injected into Python node functions by **parameter name**. Add a parameter whose name matches the provider's runtime name: + +```python +from sqlbuild.tasks import task + +@task +def export_orders(ctx, warehouse_client): + warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) +``` + +SQLBuild matches the parameter name `warehouse_client` to the discovered provider with that name, sets it up if it hasn't been already, and passes it to the function. + +You can also type-annotate the parameter for IDE support and compile-time validation: + +```python +from sqlbuild.tasks import task +from providers.warehouse_client import WarehouseClient + +@task +def export_orders(ctx, warehouse_client: WarehouseClient): + warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) +``` + +When a type annotation is present, SQLBuild validates that the discovered provider is an instance of the annotated class. A mismatch raises a compile-time error. + +Provider injection works in all Python node types: + +- **Loaders** (`@loader`) +- **Tasks** (`@task`) +- **Assets** (`@asset`) +- **Checks** (`@check`) + +## Using providers in hooks + +Python lifecycle hooks also support provider injection by parameter name: + +```python +# hooks/python/notify.py +from sqlbuild.hooks import hook + +@hook +def notify_complete(ctx, slack_notifier): + slack_notifier.send(f"Model {ctx.model_name} built successfully") +``` + +```sql +MODEL ( + materialized table, + post_hooks [python("notify_complete")], +); +``` + +Providers are also available on the `HookContext` via `ctx.providers`: + +```python +@hook +def notify_complete(ctx): + notifier = ctx.providers.slack_notifier + notifier.send(f"Model {ctx.model_name} built successfully") +``` + +## Using providers via context + +All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: + +```python +@task +def export_orders(ctx): + client = ctx.providers.warehouse_client + client.session.upload(ctx.query("SELECT * FROM orders")) +``` + +Both approaches (parameter injection and `ctx.providers`) are equivalent. Parameter injection is more explicit and enables type checking; `ctx.providers` is useful when provider access is conditional or dynamic. + +## Lifecycle + +Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: + +1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). +2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. +3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. + +```python +class WarehouseClient(Provider): + api_key: str + + def setup(self, ctx): + # Called once, the first time any node accesses this provider + self.connection = connect(self.api_key) + + def teardown(self): + # Called after the command completes + self.connection.close() +``` + +Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. + +## Configuration from environment variables + +Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: + +```python +class SlackNotifier(Provider): + slack_token: str # reads SLACK_TOKEN from environment + channel: str = "#builds" # has a default, environment variable is optional +``` + +See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. + +## Discovery rules + +- Provider classes are discovered from `.py` files under `providers/` recursively +- Files named `__init__.py` or starting with `_` are skipped +- Each concrete (non-abstract) subclass of `Provider` is registered +- Provider names must be globally unique across project resources +- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime + +## Plan output + +When providers are used by Python nodes or hooks, `sqb plan` shows a Providers section listing each provider and its consumers: + +``` +Providers + warehouse_client 2 nodes + slack_notifier 1 node +``` + +Use `--verbose` to see which specific nodes consume each provider. + +## Project layout + +``` +my-project/ + providers/ + warehouse_client.py + slack_notifier.py + hooks/ + sql/ + record_notification.sql + python/ + notify.py + loaders/ + load_orders.py + tasks/ + export_orders.py +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md new file mode 100644 index 0000000000..6f074a7af8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -0,0 +1,64 @@ + + +# SQL References + +> Read SQL models and sources from Python nodes without creating SQL dependencies. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/sql-references + +Python nodes can **read** SQL models and sources at runtime, but they cannot depend on them as graph edges (see the [SQL boundary](overview.md#the-sql-boundary)). Typed references make this read-only access explicit and safe across direct and virtual modes. + +## Declaring a reference + +Use `model()` and `source()` from `sqlbuild.refs` in a node's `depends_on`: + +```python +from sqlbuild.refs import model, source +from sqlbuild.tasks import task + +@task(depends_on=model("fact_orders")) +def export_orders(ctx): + ... + +@task(depends_on=source("raw_orders")) +def inspect_raw(ctx): + ... +``` + +Declaring the reference does two things: + +1. It tells SQLBuild the node reads that SQL resource, so the node is scheduled **after** the resource is built (read-side). +2. It authorizes `ctx.relation(...)` to resolve that reference at runtime. + +It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse. + +## Resolving with ctx.relation + +Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name: + +```python +@task(depends_on=model("fact_orders")) +def export_orders(ctx): + relation = ctx.relation(model("fact_orders")) + rows = ctx.query(f"SELECT * FROM {relation}").fetchall() + return ctx.result(metadata={"rows": len(rows)}) +``` + +`ctx.relation(...)` returns the adapter-qualified relation name for the current run. Passing a reference that was not declared in `depends_on` raises an error. + +Do not hardcode model or source names in raw SQL. In virtual mode a model resolves to a versioned VDE relation (for example `marts__feature_x.fact_orders`), not the bare name. A query like `ctx.query("SELECT * FROM fact_orders")` will fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`. + +## model vs source + +| Reference | Resolves to | +|-----------|-------------| +| `model("name")` | The built model relation (the VDE logical relation in virtual mode) | +| `source("name")` | The source read relation, following deferred-source semantics | + +`source(...)` respects `defer_sources_to`, so a node reading a source in `dev` can read the deferred target's data just like SQL models do. + +## Where references are allowed + +- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`. +- **Checks** may not reference SQL resources. Validate SQL with [audits](../audits.md). +- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md new file mode 100644 index 0000000000..71d3828b52 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md @@ -0,0 +1,143 @@ + + +# Tasks + +> Run Python computation and side effects as nodes in the SQLBuild graph. + +Online: https://docs.sqlbuild.com/concepts/python-nodes/tasks + +## Contents + +- Defining a task +- Dependencies +- Returning results +- Skipping +- Retries +- Decorator parameters +- Running tasks + +Tasks are Python functions that run as part of the DAG. Use them for computation, side effects, and orchestration steps that are not loading data into a source and not producing a tracked artifact. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. + +## Defining a task + +Place Python files under `tasks/` and decorate functions with `@task`: + +```python +# tasks/orders.py +from sqlbuild.tasks import task, TaskContext + +@task +def export_orders(ctx: TaskContext): + rows = fetch_orders() + return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) +``` + +The task receives a `TaskContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. + +## Dependencies + +Tasks declare dependencies with `depends_on`, accepting a single function, a tuple, or a list: + +```python +@task +def fetch_orders(ctx: TaskContext): + return ctx.result(payload=download_orders()) + +@task(depends_on=fetch_orders) +def summarize_orders(ctx: TaskContext): + result = ctx.result_of(fetch_orders) + return ctx.result(metadata={"count": len(result.payload)}) +``` + +`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. + +Tasks may depend on other tasks, assets, and loaders. They may **not** depend on SQL models or sources as graph dependencies, but they can read them at runtime with typed references - see [SQL references](sql-references.md). + +## Returning results + +```python +@task +def build_export(ctx: TaskContext): + return ctx.result( + payload={"path": "/exports/orders.csv"}, + metadata={"rows": 1200}, + ) +``` + +- `payload` is the value downstream nodes read with `ctx.result_of(...)`. +- `metadata` is structured JSON for catalogs and downstream reads. +- Tasks cannot set `materialized` - that is for [assets](assets.md). + +## Skipping + +Return `ctx.skip(...)` to skip a task: + +```python +@task +def export_if_present(ctx: TaskContext): + if not new_files_available(): + return ctx.skip("no new files") + return ctx.result(payload=do_export()) +``` + +- `"soft"` (default) skips only this task; dependents may still run if another upstream succeeded. +- `"hard"` skips this task and blocks its dependents. + +`mode` accepts either a plain string or the `SkipMode` enum: + +```python +from sqlbuild.tasks import task, SkipMode + +@task +def optional_step(ctx): + return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" +``` + +## Retries + +`@task` accepts a `retry` policy for transient failures: + +```python +from sqlbuild.retries import RetryPolicy +from sqlbuild.tasks import task + +@task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) +def call_api(ctx): + return ctx.result(payload=fetch_from_flaky_api()) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `max_attempts` | `3` | Total attempts including the first | +| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | +| `initial_delay_seconds` | `1.0` | Delay before the first retry | +| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | +| `max_delay_seconds` | `30.0` | Cap on any single delay | +| `max_elapsed_seconds` | `None` | Overall time bound for retries | +| `jitter` | `True` | Randomize delays to avoid synchronized retries | + +The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. + +## Decorator parameters + +| Parameter | Description | +|-----------|-------------| +| `name` | Override the node name (defaults to the function name) | +| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs | +| `tags` | Labels for selection and grouping | +| `group` | Display/catalog grouping | +| `description` | Docs (defaults to docstring) | +| `meta` | Freeform JSON metadata | +| `retry` | A `RetryPolicy` | + +## Running tasks + +```bash +# Run a task and its required graph +sqb build --select export_orders --no-tests --no-audits + +# Include in a full build +sqb build --select export_orders +``` + +Tasks run during `sqb build`. They are not validated by checks unless you also write a [check](checks.md) that depends on them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md new file mode 100644 index 0000000000..6e6c9a3820 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md @@ -0,0 +1,57 @@ + + +# Resource Identities + +> Canonical names for SQLBuild resources, selectors, state, and integrations. + +Online: https://docs.sqlbuild.com/concepts/resource-identities + +SQLBuild resource identities use lowercase ASCII snake_case. A name must start with a lowercase +letter, end with a lowercase letter or digit, and contain only lowercase letters, digits, and +underscores. Consecutive underscores are valid, so established names such as +`commerce__mart_v_order` remain canonical. + +This contract applies to models, seeds, sources, SQL and Python functions, generic and singular +audits, attached audit definitions and instance names, SQL tests and parameterized cases, +scenarios, SQL and Python hooks, macros, model schemas, enums, constants, materializations, +providers, event exporters, loaders, tasks, assets, and checks. Private scoped declarations use +exactly one leading underscore, such as `_country_codes`; public declarations must not use a +leading underscore. + +Provider classes are the one conventional derivation: when `provider_name` is omitted, +`AnalyticsApiProvider` resolves to `analytics_api_provider`. An explicit `provider_name` is an +authored identity and must already be canonical snake_case; SQLBuild does not normalize it. + +Names derived from files use the filename stem. For example, `models/daily_orders.sql` defines +the model identity `daily_orders`, while `audits/generic/expression_is_true.sql` defines the +generic audit identity `expression_is_true`. Directories organize resources but do not change +their names. + +Physical warehouse identifiers are separate from SQLBuild resource identities. Database, schema, +table, model alias, column, tag, group, and directory names retain their existing adapter-specific +contracts and do not need to follow this profile. + +## Invalid names + +Compilation fails during discovery with `D016` when an authored identity is not canonical: + +```text +error[D016]: Invalid model identity 'DailyOrders' in models/DailyOrders.sql; +use snake_case 'daily_orders' +``` + +SQLBuild suggests a corrected spelling but never silently normalizes an identity. Silent +normalization would make selectors, manifests, persisted execution state, and integration keys +disagree about which resource ran. + +## Migrating existing projects + +1. Rename file-derived resources and explicit `name` values to snake_case. +2. Update `__ref`, `__seed`, `__source`, function, hook, macro, audit, test, and dependency + references. +3. Update selectors and external integrations, including Dagster asset/check keys, that use the + old identity. +4. Compile before building. + +A rename intentionally creates a new resource identity. Existing fingerprints, audit history, +and other persisted state under the old name are not silently reassigned to the new resource. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md new file mode 100644 index 0000000000..ae0422f808 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -0,0 +1,237 @@ + + +# Overview + +> Turn repeated SQL and project review decisions into compiler-enforced requirements. + +Online: https://docs.sqlbuild.com/concepts/rules + +## Contents + +- What Rules are for +- How Rules work +- Where SQL linting went +- Built-in and custom Rules +- One definition of project validity +- Rules compared with tests, audits, and contracts +- Findings and intentional exceptions +- Performance is part of the design +- Continue + +Code review is an expensive place to keep enforcing a decision the team has already made. + +Comments such as these are useful once, but repetitive after that: + +- “This query needs deterministic ordering before `LIMIT`." +- “Move this model into the agreed area and layer." +- “Final models cannot depend on another area's private intermediate models." +- “This incremental output needs a declared grain, an enforced contract, and a focused test." +- “External sources must enter through staging." + +Rules turn those decisions into configurable compile-time requirements. They run over the same +project representation SQLBuild uses to produce executable artifacts, so a failed Rule is a failed +compile rather than an optional report somebody must remember to run. + +## What Rules are for + +Start with built-in Rules, then add custom Rules for requirements specific to your project. + +- **Built-in Rules** are opinionated, reusable checks for areas such as SQL safety, contracts, and + dependency boundaries. Select or ignore each Rule individually. +- **Custom Rules** encode local decisions such as directory structure, architectural boundaries, + public interfaces, or the contract and test coverage required for a particular kind of model. + +Both evaluate authored SQL and the compiled project without querying warehouse data. They use +compiler-owned facts about SQL, models, paths, configuration, columns, contracts, tests, audits, +declarations, and dependencies. + +That context can be combined in one check—for example, allowing `SELECT *` in staging while requiring +contracted outputs in final models, or allowing cross-area dependencies only through an `interface` +model. + +## How Rules work + +Use `sqb rules show` to understand a Rule before enabling it: + +```bash +sqb rules show SQBRSQL004 +``` + +```text +SQBRSQL004: Row selection is nondeterministic +Family: SQBRSQL +Slug: unordered-limit +Subject: model +Enabled by default: yes +Remediation: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. +``` + +`Enabled by default` means the Rule is included when a matching family is selected. Configurable +Rules do not run unless `select` contains an exact code or matching family. + +Select Rules in `sqlbuild_project.toml`. This example enables the complete built-in SQL family: + +```toml +[rules] +select = ["SQBRSQL"] +``` + +An exact code such as `SQBRSQL004` selects one Rule. A family code such as `SQBRSQL` selects every +Rule in that family. An empty `select` disables configurable Rules, but mandatory compiler +correctness still applies. + +Now consider a model that returns an arbitrary row because its `LIMIT` has no ordering: + +```sql +-- models/orders.sql +SELECT order_id FROM orders LIMIT 1 +``` + +Run normal compilation: + +```bash +sqb compile +``` + +The selected Rule reports the source location, problem, and remediation: + +```text +error[SQBRSQL004]: Row selection is nondeterministic + --> models/orders.sql:2:36 + | + 2 | SELECT order_id FROM orders LIMIT 1 + | ^ + = help: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. +``` + +Compilation fails before SQLBuild completes artifacts or opens a warehouse connection. Add +deterministic ordering and compile again: + +```sql +SELECT order_id FROM orders ORDER BY order_id LIMIT 1 +``` + +Use focused commands when discovering or adopting Rules: + +```bash +sqb rules list +sqb rules run SQBRSQL004 +``` + +A focused run checks only the requested Rule or family. Run `sqb compile` before treating the whole +project as valid. + +## Where SQL linting went + +SQL linting is part of the Rules system rather than a separate command. + +The `SQBRSQL` built-in family covers deterministic SQL diagnostics such as: + +- unsafe `NULL` comparisons; +- implicit cartesian joins; +- unordered `LIMIT` and `OFFSET`; +- unused CTEs; +- duplicate aliases; +- unstable window ordering; +- risky set-operation shapes; +- selected SQL structure and convention requirements. + +These checks run through ordinary compilation. There is intentionally no separate `sqb lint` +lifecycle. If an SQL requirement affects whether the project is acceptable, the compiler enforces +it with the rest of the configured Rules. + +Not every traditional lint concern should become a diagnostic. SQLBuild separates three kinds of +SQL responsibility: + +| Responsibility | Owner | Example | +|---|---|---| +| Correctness required to understand the project | Compiler | Invalid syntax, unresolved references, incompatible output shape | +| Configurable, deterministic requirements | `SQBRSQL` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | +| Canonical presentation | [`sqb format`](../cli/format.md) (separate from the compiler) | Capitalization, indentation, spacing, comma and clause layout | + +Rules never rewrite source. `sqb format` rewrites source and does not decide whether a project +passes its configured Rules. This keeps diagnostics and source mutation separate. + +## Built-in and custom Rules + +SQLBuild maintains native built-in Rules for reusable requirements. Their codes begin with `SQBR`, +for example `SQBRSQL004` and `SQBRGRAPH101`. + +Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for +example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, +and cache system as built-ins. + +```toml +[rules] +select = [ + "SQBRSQL", + "SQBRGRAPH", + "XSQBRARCH", +] +``` + + Define typed model or project checks over compiler-owned facts with `@rule`, `RuleContext`, and + `Finding`. + +## One definition of project validity + +`sqb compile` is authoritative. SQLBuild evaluates a project in this order: + +```text +mandatory compiler correctness +→ selected native built-in Rules +→ selected custom Python Rules +→ artifact completion +``` + +Build and planning commands use the same compiler path. Configured findings stop execution before a +warehouse connection is opened. Compile artifacts are not completed from a project that failed its +Rules. + +## Rules compared with tests, audits, and contracts + +Rules do not replace every form of validation: + +| Capability | Question it answers | +|---|---| +| Mandatory compiler correctness | Can SQLBuild construct a trustworthy project and executable representation? | +| Rules | Does the authored and compiled project satisfy selected static requirements? | +| Contracts | Does a model declare and preserve the required output shape? | +| SQL tests | Does SQL logic produce an expected result for controlled inputs? | +| Audits | Does warehouse data satisfy a runtime quality requirement? | +| Runtime checks | Are external resources and execution state valid now? | + +Use a Rule when the answer can be determined from source and compiler facts. Use a test when you +need to execute SQL against controlled data. Use an audit when the answer depends on warehouse data. + +## Findings and intentional exceptions + +A finding contains a stable code, project-relative path, line, column, explanation, and remediation. +This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. + +Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores +require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently +accumulate. Mandatory compiler correctness cannot be suppressed. + +See [Findings and exceptions](rules/findings-and-exceptions.md) for configuration examples. + +## Performance is part of the design + +Model Rules are evaluated and cached per model. Editing one model invalidates affected subjects +rather than every invocation. Project Rules run once over the complete project view and are the +explicit choice for genuinely project-wide requirements, with broader invalidation. SQLBuild tracks +the inputs each Rule depends on so cached findings are reused only while those inputs remain valid. + +Required CI exercises custom Rule packs over generated projects, checking compilation time, +memory, and cache invalidation. Separate 20- and 100-Rule profiles measure the cost of custom +validation at 5,000-model scale. + +See [Benchmarks](../benchmarks.md#rules-performance) for the benchmark profiles and reproduction commands, +and [Execution and caching](rules/execution-and-caching.md) for the Rules lifecycle. + +## Continue + + Select exact Rules or families and configure project-owned options. + Understand diagnostics, exact exceptions, path ignores, and stale checks. + Learn evaluation order, focused execution, and dependency-aware reuse. + List, inspect, run, and generate guidance for configured Rules. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md new file mode 100644 index 0000000000..0d4a09e6c6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -0,0 +1,67 @@ + + +# Configuration and selection + +> Select built-in and custom Rules by exact code or family prefix. + +Online: https://docs.sqlbuild.com/concepts/rules/configuration-and-selection + +Configure built-in and custom Rules together: + +```toml +[rules] +select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] +ignore = ["SQBRSQL004"] +``` + +An empty `select` disables configurable Rules. It does not disable mandatory compiler correctness. + +## Codes and families + +Built-in codes use `SQBR`, such as `SQBRSQL004`, `SQBRMODEL101`, and +`SQBRGRAPH101`. Custom codes use `XSQBR`, such as +`XSQBRARCH001`. + +A family is always the code with its final three digits removed: + +```text +SQBRSQL004 → SQBRSQL +XSQBRARCH001 → XSQBRARCH +XSQBR001 → XSQBR +``` + +Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching +semantics and takes precedence over `select`. + +Use the catalogue to inspect available codes and metadata: + +```bash +sqb rules list +sqb rules show SQBRSQL004 +``` + +## Rule options + +Custom Rules can declare typed options. Configure them by exact code: + +```toml +[rules.rule_options.XSQBRNAME001] +required_prefix = "order" +``` + +Unknown Rule codes, option names, and invalid option values fail configuration. + +## Focused selection + +Run one exact Rule or family while developing or adopting it: + +```bash +sqb rules run SQBRSQL004 +sqb rules run SQBRSQL +sqb rules run XSQBRARCH --select customer_orders +``` + +Focused execution constructs the compiler facts needed by the selected Rules. It does not replace +the complete configured enforcement performed by `sqb compile`. + + See every `sqb rules` command and exit code. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/overview.md new file mode 100644 index 0000000000..b68d4119b5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/overview.md @@ -0,0 +1,84 @@ + + +# Overview + +> Define repository-owned checks with one typed Python API. + +Online: https://docs.sqlbuild.com/concepts/rules/custom-rules/overview + +Custom Rules are ordinary Python under `rules/**/*.py`. Only functions decorated with `@rule` +register; helpers, constants, dataclasses, and classes remain ordinary Python. + +## Model-subject Rules + +```python +from sqlbuild.rules import Finding, Model, RuleContext, rule + +@rule( + code="XSQBRARCH001", + message="Final models must declare an order identifier", + remediation="Declare order_id in the model contract.", +) +def final_order_identifier(*, model: Model, ctx: RuleContext) -> list[Finding]: + declared = {column.name for column in ctx.columns.declared(model)} + return [] if "order_id" in declared else [ctx.finding(subject=model)] +``` + +Annotations determine invocation. The signature must contain exactly one typed `Model` or `Project` +subject and one typed `RuleContext`, all keyword-only. Parameter names and position are not semantic. +A model-subject Rule can still inspect project-wide facts. + +## Project-subject Rules + +Use `Project` when an invariant has no natural model subject: + +```python +from sqlbuild.rules import Finding, Project, RuleContext, rule + +@rule( + code="XSQBRARCH002", + message="The project must contain a final model directory", + remediation="Add models/final and place final outputs beneath it.", +) +def final_directory(*, ctx: RuleContext, project: Project) -> list[Finding]: + del project + paths = ctx.project.tree.glob("models/final/**") + return [] if paths else [ctx.finding(subject="models")] +``` + +Rules cannot mutate SQL, resources, adapter lowering, or compiler output. They return diagnostics +only. + +## Typed options + +```python +from sqlbuild.rules import Finding, Model, RuleContext, RuleOption, rule + +REQUIRED_PREFIX = RuleOption.string( + name="required_prefix", + default="customer", + description="Required model-name prefix.", +) + +@rule( + code="XSQBRNAME001", + message="The model name uses the wrong prefix", + remediation="Rename the model with the configured prefix.", + options=(REQUIRED_PREFIX,), +) +def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: + prefix = ctx.option(REQUIRED_PREFIX) + return [] if model.name.startswith(prefix) else [ctx.finding(subject=model)] +``` + +```toml +[rules.rule_options.XSQBRNAME001] +required_prefix = "order" +``` + +Unknown codes, option names, or invalid values fail configuration. + +## Continue + + Inspect SQL, graph, contracts, tests, audits, declarations, and project structure. + Test findings and keep helper and project observations cacheable. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md new file mode 100644 index 0000000000..300b392770 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -0,0 +1,74 @@ + + +# RuleContext and compiler facts + +> Inspect stable compiler-owned facts and opt into lazy SQL AST access. + +Online: https://docs.sqlbuild.com/concepts/rules/custom-rules/rule-context + +`RuleContext` exposes typed, read-only views over the compiled project: + +- `ctx.sql`: authored and expanded SQL +- `ctx.graph`: compiler-resolved dependencies and dependents +- `ctx.columns`: declared and inferred output columns +- `ctx.contracts`: enforcement and grain facts +- `ctx.tests` and `ctx.audits`: checks associated with a model +- `ctx.declarations`: public and model-scoped enums and constants +- `ctx.project`: compiled resources and deterministic project-tree observations + +## Authored and expanded SQL + +Choose the SQL representation that matches the requirement: + +```python +sql = ctx.sql.for_model(model) + +authored_text = sql.authored.source +expanded_text = sql.expanded.source +``` + +An authoring convention may inspect what a developer wrote. A compiler-output convention may need +the expanded SQL after interpolation, declarations, and macros. + +## Common SQL structures + +Common structures are available through typed source nodes: + +```python +for cte in sql.expanded.ctes(): + ... + +for star in sql.authored.star_projections(): + ... +``` + +## Lazy Polyglot access + +Use the full Polyglot AST only when common projections are insufficient: + +```python +ast = sql.expanded.polyglot_ast() +for node in ast.walk(): + ... +``` + +AST construction is lazy. A path or contract Rule does not pay for SQL parsing merely because AST +access exists elsewhere. + +## Project structure + +Use the compiler-owned tree rather than direct filesystem calls: + +```python +parts = ctx.project.tree.relative_parts(path=model.path, under="models") +models = ctx.project.tree.resources_under("models/orders") +config = ctx.project.tree.read_text("rules/requirements.yaml") +``` + +Paths remain project-relative, and observations participate in cache invalidation. + +## Choose the subject deliberately + +A model-subject Rule is evaluated and cached per model. A project-subject Rule runs once and can +iterate `ctx.project.models`. Use a project subject only when the invariant genuinely needs a +project-wide view; its cache invalidation is intentionally broader. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md new file mode 100644 index 0000000000..6d786b2e8e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -0,0 +1,64 @@ + + +# Testing and deterministic Rules + +> Test custom findings and keep implementations safe for dependency-aware caching. + +Online: https://docs.sqlbuild.com/concepts/rules/custom-rules/testing-and-determinism + +## Test Rules + +Use the public harness to exercise discovery, compilation, and Rules evaluation: + +```python +from sqlbuild.rules.testing import RuleCase, evaluate_rule +from rules.final_outputs import final_order_identifier + +def test_given_missing_order_id_when_evaluating_then_reports_finding() -> None: + result = evaluate_rule( + rule=final_order_identifier, + test_case=RuleCase( + description="missing order identifier", + source='MODEL (description "Orders");\nSELECT 1 AS customer_id\n', + path="models/final/customer_orders.sql", + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 +``` + +`RuleCase.files` adds supporting project files. `RuleCase.config` supplies option values. Include +passing cases, failing cases, near misses, and exact source-position assertions. + +Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. + +## Helpers + +Custom Rules can import repository-owned helpers under `rules/` and supported pure Python modules. +Changing an imported helper invalidates the Rules that depend on it. + +Only decorated functions register. Ordinary functions, constants, dataclasses, and classes remain +helpers. + +## Deterministic inputs + +Direct filesystem, environment, clock, randomness, network, and subprocess access is rejected. +Those inputs cannot be reproduced safely by the Rules cache. + +Read supported project text and structure through `ctx.project.tree`: + +```python +text = ctx.project.tree.read_text("rules/requirements.yaml") +matches = ctx.project.tree.glob("models/*/interface/*.sql") +``` + +Both positive and negative observations are tracked. If `glob` returns no paths, adding a matching +path invalidates the cached result. + +## Cache granularity + +Cache identity incorporates the Rule implementation, imported helper closure, configured options, +subject, accessed compiler facts, tracked project observations, dialect, and compatibility +versions. Prefer model subjects for independent per-model checks and project subjects for genuine +cross-project invariants. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md new file mode 100644 index 0000000000..47f78c109c --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -0,0 +1,56 @@ + + +# Execution and caching + +> Understand Rule enforcement order, focused runs, and dependency-aware reuse. + +Online: https://docs.sqlbuild.com/concepts/rules/execution-and-caching + +SQLBuild evaluates a project in this order: + +1. discovery, expansion, and mandatory compiler correctness +2. selected native built-in Rules +3. selected custom Python Rules +4. artifact completion + +Native built-ins always run before custom Rules. Structured compile output reports built-in and +custom durations separately, together with cache hits and misses. + +## Authoritative enforcement + +`sqb compile` runs the complete configured Rules set. Build and execution commands use the same +compiler path and reject Rule findings before opening a warehouse connection. + +`sqb rules run` is intentionally focused. Use it to inspect one exact code or family, but run the +normal compiler before treating the complete project as valid. + +## Dependency-aware caching + +Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and +imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project +observations, and compatibility versions. + +Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model +subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for +genuinely project-wide invariants, with correspondingly broader invalidation. + +Positive and negative project-tree observations are dependencies. If a Rule observes that a glob +has no matches, adding a matching file invalidates that result. + +## Built-in-only execution + +When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every +custom result is cached, the host is not launched for evaluation. + +## Agent guidance + +Generate project-specific guidance from the configured Rules: + +```bash +sqb rules skills +sqb rules skills --check +``` + +SQLBuild manages `.agents/skills/sqlbuild-rules/SKILL.md`, +`.claude/skills/sqlbuild-rules/SKILL.md`, and `.opencode/skills/sqlbuild-rules/SKILL.md` without +overwriting divergent or unowned content. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md new file mode 100644 index 0000000000..ffe81e254b --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -0,0 +1,77 @@ + + +# Findings and exceptions + +> Understand Rule diagnostics and record intentional exceptions safely. + +Online: https://docs.sqlbuild.com/concepts/rules/findings-and-exceptions + +A Rule finding identifies an exact code, project-relative path, line, column, message, and +remediation. Error findings make compilation fail and prevent artifact completion. + +## Exact exceptions + +Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: + +```toml +[[rules.rule_exceptions]] +rule = "SQBRSQL004" +path = "models/examples/sample_orders.sql" +reason = "This example intentionally demonstrates one sampled row." +``` + +An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports +the stale exception so obsolete configuration does not accumulate silently. + +## Path- and resource-scoped ignores + +Use a path ignore when a documented project area intentionally follows a different convention: + +```toml +[[rules.rule_ignores]] +rules = ["SQBRSQL"] +paths = ["models/examples/**"] +reason = "Examples retain intentionally minimal SQL." +``` + +Path ignores accept exact codes and family prefixes. Keep their scope narrow and explain why the +project differs from the selected requirement. + +Use `selectors` when the exception follows graph-resource identities or lineage rather than files: + +```toml +[[rules.rule_ignores]] +rules = ["SQBRSQL021"] +selectors = ["intermediate_*"] +reason = "These intermediate interfaces intentionally preserve upstream columns." +``` + +Selectors use the same grammar as SQLBuild commands, including exact names, name globs, tags, +resource paths, and graph expansion such as `+intermediate_*`. Use `paths` for authored-file glob +matching, including SQL tests and audits that are not graph resources: + +```toml +[[rules.rule_ignores]] +rules = ["SQBRSQL021"] +paths = ["models/**/intermediate_*.sql"] +reason = "These intermediate SQL files intentionally preserve upstream columns." +``` + +`paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. + +## Mandatory correctness + +Mandatory compiler correctness is not configurable and cannot be suppressed. A project must first +compile into a trustworthy representation before any selected Rule can run. + +## Machine-readable findings + +Use JSON when another tool consumes findings: + +```bash +sqb compile --json +sqb rules --json run SQBRSQL +``` + +Machine-readable output remains on stdout. Lifecycle progress and terminal status are written to +stderr. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md new file mode 100644 index 0000000000..04fdfe7214 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -0,0 +1,290 @@ + + +# Scenarios + +> End-to-end tests that build real project graphs against coherent fixture data. + +Online: https://docs.sqlbuild.com/concepts/scenarios + +## Contents + +- When to use scenarios vs unit tests +- Authoring scenarios +- How scenarios execute +- Local scenario testing +- Cleanup +- Limitations + +Scenarios are end-to-end tests for coherent slices of your warehouse. You define fixture inputs, SQLBuild builds the real project graph against them in isolated physical relations, and you assert that the result is correct. + +## When to use scenarios vs unit tests + +[SQL unit tests](testing.md) compile to a single comparison query. They are fast, inline, and work well for checking individual model transformations. But when you need to test across many models with related entities - customers that have orders, orders that have payments, refunds that reference original orders - a single query can become unwieldy or hit SQL size limits. + +Scenarios materialize physical relations in the warehouse and build the actual model graph. This means: + +- Fixture inputs are real tables, not CTEs inside one query +- Models execute with real SQL, real materializations, and real dependency ordering +- You can test end-to-end business logic across an entire pipeline slice +- Debugging is easier because you can inspect intermediate tables with `--retain` + +Use unit tests for transformation logic. Use scenarios for coherent end-to-end validation. + +## Authoring scenarios + +Scenario files live under `tests/scenarios/` and use one file per scenario: + +``` +tests/ + scenarios/ + revenue/ + daily_revenue_minimal.sql + daily_revenue_multi_order.sql + fulfillment/ + fulfillment__late_shipment.sql +``` + +Folders are organizational only. The scenario name is the filename stem and must be globally unique across all discovered scenario files. + +### Scenario format + +Each file has a `SCENARIO()` header followed by CTEs that define inputs, expected outputs, and assertions: + +```sql +SCENARIO ( + description "Daily revenue includes only successful payments", + tags ["revenue", "example"] +); + +WITH +__ref__stg_orders AS ( + SELECT + 1 AS order_id, 10 AS customer_id, 1 AS waffle_type_id, + 2 AS quantity, CAST('2026-04-01 09:15:00' AS TIMESTAMP) AS ordered_at, + 'completed' AS status +), + +__ref__stg_payments AS ( + SELECT + 1 AS payment_id, 1 AS order_id, 1700 AS amount_cents, + 'credit_card' AS payment_method, + CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, + 'success' AS payment_status + UNION ALL + SELECT + 2 AS payment_id, 2 AS order_id, 1050 AS amount_cents, + 'credit_card' AS payment_method, + CAST('2026-04-01 10:01:00' AS TIMESTAMP) AS paid_at, + 'failed' AS payment_status +), + +__expected__daily_revenue AS ( + SELECT + CAST('2026-04-01' AS DATE) AS revenue_date, + 1 AS order_count, 2 AS waffles_sold, + 1700 AS total_revenue_cents +), + +__assert__all_orders_have_payments AS ( + SELECT * + FROM __ref("fact_orders") + WHERE payment_amount_cents IS NULL +) + +SELECT 1 +``` + +### CTE conventions + +| Prefix | Purpose | +|--------|---------| +| `__source__` | Fixture source input. Materializes as a scenario table. | +| `__ref__` | Fixture model boundary. Prevents SQLBuild from building that model upstream. | +| `__seed__` | Fixture seed data. | +| `__expected__` | Full expected output. Compared order-insensitively against the scenario-built model. | +| `__assert__` | Zero-row assertion. Passes if the query returns no rows; fails with the returned rows as diagnostics. | + +For sources with two-part identity, use double underscores: `__source__raw__orders`. + +Every scenario must have at least one fixture CTE and at least one `__expected__` or `__assert__` CTE. + +Scenario SQL uses macros, enums, and constants available from the scenario file's directory under +`tests/scenarios/`. Public enums and constants available to a model are also available when the +scenario defines `__expected__model_name` output for that model. Model-private values and macros +available only to the model are not included. See +[How Visibility Works](declaration-scopes/visibility.md#tests-and-expected-output). + +### SCENARIO() header + +The header is metadata only: + +| Field | Description | +|-------|-------------| +| `description` | Optional description string | +| `tags` | Optional list of string tags | + +## How scenarios execute + +### Graph inference + +SQLBuild infers which models to build from your scenario: + +1. Target models come from `__expected__` CTE names and `__ref(...)` calls inside `__assert__` CTEs +2. SQLBuild walks upstream from targets, building every required model +3. `__ref__` fixtures act as boundaries - upstream traversal stops there +4. All required sources must be provided by `__source__` fixtures +5. Required project seeds are loaded automatically unless overridden by `__seed__` fixtures + +### Isolation + +Scenario artifacts are physically isolated from production: + +- Every scenario-owned relation uses a deterministic prefixed name (based on a hash of the project and scenario identity) +- All `ref()`, `source()`, and `seed()` calls resolve to scenario-owned relations, never production tables +- Fixture CTEs can read from existing warehouse relations if you choose, but downstream models always read the materialized scenario fixture, not the original + +### Execution flow + +1. Clean any existing scenario artifacts from a previous run +2. Materialize source, ref, and seed fixtures as physical tables +3. Load required project seeds +4. Build required models in dependency order (incremental models run as full-refresh in scenarios) +5. Run expected-output comparisons (order-insensitive) +6. Run zero-row assertions +7. Clean up all scenario-owned artifacts (unless `--retain`) + +### Inspecting with `--retain` + +When a scenario fails or you want to inspect intermediate state: + +```bash +sqb scenario test --select daily_revenue_minimal --retain +``` + +This keeps all scenario-owned relations in the warehouse and prints a relation map showing the logical-to-physical name mapping. You can then query the scenario tables directly to debug. + +Runtime artifacts (fixture SQL, model lifecycle SQL, check SQL, cleanup SQL) are always written to `target/run/scenarios//` regardless of `--retain`. + +## Local scenario testing + +Scenarios can run locally against DuckDB using captured JSONL snapshots - no warehouse connection needed. This is useful for CI pipelines and fast developer iteration. + +### Capture + +First, capture scenario inputs from the real warehouse: + +```bash +sqb scenario capture --select daily_revenue_minimal +``` + +This: + +1. Materializes scenario input fixtures in the warehouse +2. Runs a preflight `COUNT(*)` on each fixture to check row counts against safety limits +3. Inspects column types and maps them to DuckDB-compatible types +4. Downloads rows as JSONL files (enforcing byte limits during writing) +5. Writes a `scenario.json` manifest with column metadata, types, row counts, and an input fingerprint +6. Cleans up warehouse artifacts + +Snapshots are written to `tests/_scenario_snapshots//` and can be committed to version control. JSONL is human-readable and git-diffable. + +### Capture safety limits + +Capture enforces row and byte limits to prevent accidentally saving large datasets. Limits can be set in `sqlbuild_project.toml`: + +```toml +[scenario.snapshot_limits] +max_rows_per_relation = 10000 +max_total_rows = 50000 +max_bytes_per_relation = 10485760 # 10 MB +max_total_bytes = 52428800 # 50 MB +``` + +Or per-command via CLI flags: + +```bash +sqb scenario capture --max-snapshot-rows 5000 --max-snapshot-total-rows 20000 +sqb scenario capture --max-snapshot-bytes 5242880 +``` + +CLI flags override TOML config. Use `--force` to bypass all limits: + +```bash +sqb scenario capture --force +``` + +When a limit is exceeded, capture fails with a clear error and suggests narrowing the fixture query, raising the limit, or using `--force`. + +### Local replay + +Run scenarios locally against DuckDB: + +```bash +sqb scenario test --local +``` + +This: + +1. Checks snapshot freshness via the input fingerprint (missing/stale snapshots are skipped by default) +2. Creates a temporary DuckDB database at `target/run/scenarios//local.duckdb` +3. Loads JSONL snapshots into typed DuckDB tables using column metadata from `scenario.json` +4. Transpiles model and check SQL from the project adapter dialect to DuckDB +5. Builds functions, models, and runs expected/assertion checks in DuckDB +6. Keeps the local DuckDB file for inspection (it lives under `target/`, so it's always retained) + +### Sync snapshots + +Instead of running capture and test separately, sync snapshots in one command: + +```bash +# Capture missing/stale snapshots, then run locally +sqb scenario test --local --sync-snapshots + +# Recapture all snapshots, then run locally +sqb scenario test --local --refresh +``` + +With `--sync-snapshots`, fresh snapshots are reused. With `--refresh`, all selected snapshots are recaptured even if fresh. + +### Strict mode + +By default, missing or stale snapshots are skipped with a warning. Use `--strict` to treat them as errors: + +```bash +sqb scenario test --local --strict +``` + +### Local type overrides + +When the automatic warehouse-to-DuckDB type conversion produces an incompatible type, you can override it in `sqlbuild_project.toml`: + +```toml +[scenario.local_type_overrides.snowflake] +"OBJECT" = "JSON" +"ARRAY" = "JSON" +"NUMBER(*,0)" = "BIGINT" +``` + +Override keys are structural type patterns. Values can reference matched type arguments with `{1}`, `{2}`, etc: + +```toml +[scenario.local_type_overrides.bigquery] +"NUMERIC({1},{2})" = "DECIMAL({1},{2})" +``` + +These overrides are written into `scenario.json` during capture and used when loading snapshots into DuckDB. + +## Cleanup + +By default, remote scenario artifacts are dropped after each run (pass or fail). The janitor command also recognizes scenario artifacts and can clean retained or orphaned scenario relations: + +```bash +sqb janitor +``` + +See the [CLI reference](../cli/scenario.md) for full command documentation. + +## Limitations + +- Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. +- Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md new file mode 100644 index 0000000000..8a902e3372 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md @@ -0,0 +1,133 @@ + + +# Seeds + +> Load static CSV data into your pipeline as tables. + +Online: https://docs.sqlbuild.com/concepts/seeds + +## Contents + +- Defining a seed +- Referencing seeds +- Loading seeds +- Target overrides +- CSV settings + +Seeds are CSV files under `seeds/` that SQLBuild loads as tables in the warehouse. They're useful for small, static reference data like lookup tables, category mappings, or configuration values. + +## Defining a seed + +Place a CSV file in your `seeds/` directory: + +```csv +waffle_type_id,waffle_name,category,price_cents +1,Classic Belgian,sweet,850 +2,Liege,sweet,950 +3,Brussels,sweet,750 +4,Cheddar Herb,savory,1050 +``` + +Declare the seed's column types in any `.yml` file under `seeds/`. You can use any filename and organize declarations however you like - one file per seed, or group related seeds together: + +```yaml +# seeds/lookups.yml +seeds: + - name: waffle_types + description: Waffle menu items with pricing. + columns: + - name: waffle_type_id + type: INTEGER + - name: waffle_name + type: VARCHAR + - name: category + type: VARCHAR + - name: price_cents + type: INTEGER +``` + +Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). + +## Referencing seeds + +Seeds are referenced in models with `__seed()`: + +```sql +SELECT + o.order_id, + w.waffle_name, + w.price_cents * o.quantity AS line_total_cents +FROM __ref("stg_orders") o +LEFT JOIN __seed("waffle_types") w ON o.waffle_type_id = w.waffle_type_id +``` + +`__seed()` is distinct from `__ref()`. Using `__ref()` with a seed name raises a compile error with a message pointing you to `__seed()`. + +## Loading seeds + +Seeds are loaded automatically during `sqb build`. You can also load them standalone: + +```bash +sqb seed +``` + +Seeds are fully replaced on every run. If the CSV changes, the table is recreated with the new data. + +## Target overrides + +Seeds inherit the project's default database and schema. You can override these per seed: + +```yaml +seeds: + - name: waffle_types + database: analytics + schema: lookups + columns: + - name: waffle_type_id + type: INTEGER + - name: waffle_name + type: VARCHAR +``` + +## CSV settings + +For non-standard CSV formats, configure parsing behavior with `csv_settings`: + +```yaml +seeds: + - name: european_prices + csv_settings: + delimiter: ";" + encoding: utf-8 + quotechar: '"' + columns: + - name: product_id + type: INTEGER + - name: price + type: VARCHAR +``` + +### Available CSV settings + +| Setting | Type | Description | +|---------|------|-------------| +| `delimiter` | string | Field delimiter (default: `,`) | +| `quotechar` | string | Character used to quote fields (default: `"`) | +| `doublequote` | boolean | Whether the quotechar is doubled to escape itself | +| `escapechar` | string | Character used to escape the delimiter or quotechar | +| `skipinitialspace` | boolean | Skip whitespace after the delimiter | +| `lineterminator` | string | Line terminator character | +| `encoding` | string | File encoding (default: `utf-8`) | +| `na_values` | list or mapping | Values to treat as null - a global list or per-column mapping | +| `keep_default_na` | boolean | Whether to use the default set of NA values in addition to `na_values` | + +### Per-column NA values + +`na_values` can be a flat list (applied to all columns) or a mapping from column names to their NA values: + +```yaml +csv_settings: + na_values: + price: ["N/A", ""] + category: ["unknown"] +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md new file mode 100644 index 0000000000..f0a12610a3 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -0,0 +1,171 @@ + + +# Selectors + +> Target specific models, paths, tags, or DAG subsets with select and exclude flags. + +Online: https://docs.sqlbuild.com/concepts/selectors + +## Contents + +- Basic usage +- Selector types +- Graph expansion +- Path-between selectors +- Intersection +- Combining select and exclude +- Error handling + +Selectors let you scope commands to specific subsets of your project. They work with `plan`, `build`, `test`, `audit`, `seed`, `clone`, and `diff`. + +## Basic usage + +```bash +sqb build --select daily_revenue +sqb build --select daily_revenue customer_status_snapshot +sqb build --exclude stg_customers +``` + +`--select` (or `-s` for short) accepts one or more names. Multiple values are unioned. Space-separated names within one `--select` are also unioned. `--exclude` subtracts from the selected set. + +When no `--select` is provided, all models are selected. + +## Selector types + +### Name + +Select a single resource by name: + +```bash +sqb build --select fact_orders +``` + +Bare names accept glob patterns. This selects every resource whose name starts with +`intermediate_`: + +```bash +sqb build --select "intermediate_*" +``` + +Name globs compose with graph expansion. For example, `+intermediate_*` selects every matching +resource and all of their upstream dependencies. Quote patterns in shell commands so your shell +does not expand `*` against files in the current directory. + +### Tag + +Select all models with a specific tag: + +```bash +sqb build --select tag:staging +sqb build --select tag:acceptance +``` + +### Path + +Select all models under a directory path: + +```bash +sqb build --select path:models/marts +sqb build --select models/marts +sqb build --select path:models/intermediate +``` + +Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/`, `tasks/`, `assets/`, `checks/`, or `loaders/`. Nested paths work too: `models/staging/orders`. + +### Seed and source + +```bash +sqb build --select seed:waffle_types +sqb build --select source:raw__orders +``` + +## Graph expansion + +### Upstream + +Select a model plus all its upstream dependencies: + +```bash +sqb build --select +daily_activity_rollup +``` + +### Downstream + +Select a model plus all its downstream dependents: + +```bash +sqb build --select fact_orders+ +``` + +### Bidirectional + +```bash +sqb build --select +fact_orders+ +``` + +Graph expansion works with all selector types: + +```bash +sqb build --select +tag:marts +sqb build --select path:models/staging+ +``` + +## Path-between selectors + +Select all models on the shortest path between two nodes: + +```bash +sqb build --select fact_orders~daily_activity_rollup +``` + +With endpoint expansion: + +```bash +sqb build --select +fact_orders~daily_activity_rollup+ +``` + +This selects: +- All upstreams of `fact_orders` +- Every model on the path between `fact_orders` and `daily_activity_rollup` +- All downstreams of `daily_activity_rollup` + +This is useful for rebuilding a specific slice of the DAG without manually listing every model in between. + +## Intersection + +Use commas to intersect selector results: + +```bash +sqb build --select "tag:staging,path:models/finance" +``` + +This selects only models that match *both* conditions - in this case, models tagged `staging` that are also under the `models/finance` directory. + +## Combining select and exclude + +```bash +# Build all marts except daily_revenue +sqb build --select path:models/marts --exclude daily_revenue + +# Build everything upstream of fact_orders, excluding staging models +sqb build --select +fact_orders --exclude tag:staging +``` + +## Error handling + +Unknown resource names, name patterns with no matches, empty paths, and malformed selectors produce +clear error messages: + +``` +unknown selector name 'nonexistent_model' +unknown selector pattern 'missing_*' +no models found under path 'models/nonexistent'. +no models found with tag 'nonexistent_tag' +path selector 'fact_orders~' requires names on both sides of '~' +``` + +If a path selector omits the root directory, SQLBuild asks for the explicit form: + +``` +path selectors require an explicit root: use 'models/', 'tasks/', 'assets/', 'checks/', or 'loaders/' +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md new file mode 100644 index 0000000000..aaff877967 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md @@ -0,0 +1,368 @@ + + +# Snapshots (SCD Type 2) + +> Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. + +Online: https://docs.sqlbuild.com/concepts/snapshots + +## Contents + +- How snapshots work +- Change detection strategies +- Historical input +- Hard deletes +- Configuration reference +- Full refresh safety +- Audits +- Duplicate handling +- Querying snapshots +- Examples + +Snapshot models maintain historical row versions with validity windows. They answer questions like: what does this entity look like now, what did it look like before, when did it change, and was it absent during a period. + +## How snapshots work + +SQLBuild adds two generated columns to the target table: + +| Column | Meaning | +|--------|---------| +| `valid_from` | When this version became valid (inclusive) | +| `valid_to` | When this version stopped being valid (exclusive). `NULL` means currently active. | + +A point-in-time query uses the interval `valid_from <= point_in_time < valid_to`. + +## Change detection strategies + +### Timestamp strategy + +Use when your source has a reliable column recording when the entity changed. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy timestamp, + updated_at updated_at, +); + +SELECT + customer_id, + name, + plan, + status, + updated_at +FROM __source("customers") +``` + +If the source `updated_at` is newer than the active target row's `updated_at`, SQLBuild closes the old version and inserts the new one. If `updated_at` is unchanged or older, nothing happens. + +### Check strategy + +Use when the source does not have a reliable update timestamp. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy check, + check_columns [name, plan, status], +); + +SELECT + customer_id, + name, + plan, + status +FROM __source("customers") +``` + +SQLBuild compares `check_columns` between source and active target rows. If any checked value differs, a new version is created. Changes to unchecked columns are ignored. + +`check_columns [*]` checks all output columns except `unique_key` and the generated validity columns. Explicit columns are recommended for important models to avoid noisy history from volatile metadata columns. + +## Historical input + +By default, SQLBuild treats the model query as returning the current state of each entity (one row per `unique_key`). When your source contains historical observations over time, add `observed_at` to switch to historical mode. + +### Historical check snapshot + +Use for daily full exports or periodic snapshots without a business update timestamp. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy check, + check_columns [plan, status], + observed_at snapshot_date, +); + +SELECT + customer_id, + plan, + status, + snapshot_date +FROM __source("customers_daily_snapshot") +``` + +Each `observed_at` group is treated as a complete picture of the source at that time. Consecutive unchanged observations are collapsed into a single version. + +### Historical timestamp snapshot + +Use for historical observations that include a business update timestamp. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy timestamp, + updated_at updated_at, + observed_at extract_date, + historical_input snapshot, +); + +SELECT + customer_id, + plan, + status, + updated_at, + extract_date +FROM __source("customers_historical_extracts") +``` + +Each row means: "at `extract_date`, the source's current state for this key had this `updated_at`." Validity windows use `updated_at`, not `observed_at`. + +### Historical change records + +Use for CDC tables, audit logs, or historical backfills where rows are individual version records. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy timestamp, + updated_at updated_at, + observed_at loaded_at, + historical_input changes, +); + +SELECT + customer_id, + plan, + status, + updated_at, + loaded_at +FROM __source("customers_cdc") +``` + +Multiple changes for the same key in one batch are allowed. `updated_at` determines version ordering. `observed_at` is arrival/load time, not validity time. + +### Historical input rules + +| Strategy | `historical_input` | Source shape | Uniqueness | Hard deletes | +|----------|-------------------|-------------|-----------|-------------| +| `check` | `snapshot` (default) | Complete observations over time | `unique_key + observed_at` | Allowed | +| `timestamp` | `snapshot` | Complete observations with update timestamp | `unique_key + observed_at` | Allowed | +| `timestamp` | `changes` | Individual change/version records | `unique_key + updated_at` | Not allowed | +| `check` | `changes` | Not supported | - | - | + +Timestamp snapshots with `observed_at` require `historical_input` to be set explicitly. + +## Hard deletes + +```sql +invalidate_hard_deletes true, +``` + +When enabled, active target rows whose keys are missing from the source are closed: + +- **Current-state input**: closed at execution time +- **Historical input** (`historical_input snapshot`): closed at the `observed_at` time of the group where the key is missing + +Hard deletes are not allowed with `historical_input changes` because change-record batches are not complete source snapshots - a missing key just means no change, not deletion. + +Reappearing keys create a new active version. + +## Configuration reference + +| Field | Required | Description | +|-------|----------|-------------| +| `materialized snapshot` | Yes | Enables snapshot lifecycle | +| `unique_key` | Yes | Column(s) identifying one entity | +| `snapshot_strategy` | Yes | `timestamp` or `check` | +| `updated_at` | Timestamp only | Source column with business update time | +| `check_columns` | Check only | Columns compared to detect changes. Use `[*]` for all non-key columns. | +| `observed_at` | No | Source column with observation/extract time. Presence enables historical mode. | +| `historical_input` | Conditional | `snapshot` or `changes`. Required for timestamp with `observed_at`. Defaults to `snapshot` for check with `observed_at`. | +| `invalidate_hard_deletes` | No | Close active rows missing from source. Default `false`. | +| `valid_from_column` | No | Override generated column name. Default `valid_from`. | +| `valid_to_column` | No | Override generated column name. Default `valid_to`. | +| `initial_valid_from` | No | First-version start time: `updated_at`, `observed_at`, or `execution_time`. See defaults below. | +| `snapshot_full_refresh` | No | Model-level full-refresh safety: `deny`, `require_confirmation`, or `allow`. | + +### Initial valid_from defaults + +| Case | Default | +|------|---------| +| Timestamp, no `observed_at` | `updated_at` | +| Timestamp, with `observed_at` | `updated_at` | +| Check, no `observed_at` | `execution_time` | +| Check, with `observed_at` | `observed_at` | + +## Full refresh safety + +Snapshot full refresh can permanently discard history that cannot be reconstructed from the source. SQLBuild guards against this with configurable safety policies. + +### Project config + +```toml +[snapshots] +current_state_full_refresh = "deny" +historical_full_refresh = "require_confirmation" +``` + +| Policy | Behavior | +|--------|----------| +| `deny` | Full refresh is blocked regardless of CLI flags | +| `require_confirmation` | Requires `--allow-snapshot-full-refresh` flag or interactive confirmation | +| `allow` | No snapshot-specific confirmation required | + +### Defaults + +| Snapshot type | Default policy | Reason | +|--------------|---------------|--------| +| Current-state (no `observed_at`) | `deny` | Cannot reconstruct older history from current-state source | +| Historical (with `observed_at`) | `require_confirmation` | Can reconstruct if query returns full history, but SQLBuild cannot prove that | + +### Model override + +The model `snapshot_full_refresh` field can only make the policy **stricter** than the project setting. A model cannot weaken `deny` to `allow`. + +```sql +MODEL ( + materialized snapshot, + ... + snapshot_full_refresh deny, +); +``` + +### CLI usage + +```bash +# Fails if any selected snapshot has effective policy 'deny' +sqb build --full-refresh + +# Satisfies 'require_confirmation' policy (cannot override 'deny') +sqb build --full-refresh --allow-snapshot-full-refresh +``` + +## Audits + +Snapshot models support the same audit system as other materializations. Audits with `delta_and_final` run scope execute against the snapshot delta relation before target mutation, blocking promotion if an error-severity audit fails. Final audits run after target mutation. + +```sql +MODEL ( + materialized snapshot, + unique_key [customer_id], + snapshot_strategy timestamp, + updated_at updated_at, + columns ( + customer_id (audits [not_null (run_scope delta_and_final)]), + ), +); +``` + +## Duplicate handling + +SQLBuild fails with an actionable error if the source query produces duplicate rows at the snapshot identity grain: + +- Current-state: duplicate `unique_key` +- Historical snapshot: duplicate `unique_key + observed_at` +- Historical changes: duplicate `unique_key + updated_at` + +Deduplicate in your model SQL: + +```sql +WITH ranked AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY customer_id, snapshot_date + ORDER BY loaded_at DESC + ) AS rn + FROM __source("customer_daily") +) + +SELECT customer_id, plan, status, snapshot_date +FROM ranked +WHERE rn = 1 +``` + +## Querying snapshots + +### Current rows + +```sql +SELECT * FROM customer_snapshot WHERE valid_to IS NULL +``` + +### Point-in-time + +```sql +SELECT * +FROM customer_snapshot +WHERE customer_id = 1 + AND TIMESTAMP '2026-02-15' >= valid_from + AND (valid_to IS NULL OR TIMESTAMP '2026-02-15' < valid_to) +``` + +### Fact-to-dimension historical join + +```sql +SELECT + o.order_id, + o.ordered_at, + c.plan +FROM orders o +JOIN customer_snapshot c + ON o.customer_id = c.customer_id + AND o.ordered_at >= c.valid_from + AND (c.valid_to IS NULL OR o.ordered_at < c.valid_to) +``` + +## Examples + +### Composite key + +```sql +MODEL ( + materialized snapshot, + unique_key [user_id, role_id], + snapshot_strategy check, + check_columns [role_name, role_status], + observed_at snapshot_date, + invalidate_hard_deletes true, +); + +SELECT + user_id, role_id, role_name, role_status, snapshot_date +FROM __source("user_role_daily") +``` + +### Custom validity column names + +```sql +MODEL ( + materialized snapshot, + unique_key [product_id], + snapshot_strategy timestamp, + updated_at modified_at, + valid_from_column effective_from, + valid_to_column effective_to, +); + +SELECT + product_id, name, price, modified_at +FROM __source("products") +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md new file mode 100644 index 0000000000..339e750117 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -0,0 +1,248 @@ + + +# Sources + +> Declare external data inputs for your pipeline. + +Online: https://docs.sqlbuild.com/concepts/sources + +## Contents + +- Table sources +- Expression sources +- Source audits +- Type enforcement +- Managed sources (loaders) +- Source freshness +- Config reference + +Sources declare external data that your models depend on. They are defined in YAML files under `sources/` in your project directory. + +## Table sources + +Point at an existing table or view in your warehouse: + +```yaml +sources: + - name: raw_events + database: analytics + schema: raw + table: events +``` + +Reference it in models with `__source("raw_events")`. + +## Expression sources + +Define source data inline as a SQL expression. No external tables or setup scripts needed: + +```yaml +sources: + - name: raw__customers + expression: | + SELECT * FROM (VALUES + (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), + (2, 'Ron', 'Swanson', 'ron@pawnee.gov', TIMESTAMP '2026-02-01 08:00:00') + ) AS raw__customers(id, first_name, last_name, email, created_at) +``` + +Expression sources are resolved at compile time. They're the escape hatch for anything the framework doesn't natively model: external tables, warehouse-specific syntax, function calls, or any other relation type that doesn't fit a standard table reference. + +An inline source expression uses macros, constants, and enums available from its source definition +under `sources/`. Declarations limited to one source folder are not available in sibling folders. +See [How Visibility Works](declaration-scopes/visibility.md). + +## Source audits + +Sources support the same audit system as models. Audits attached to sources run before any dependent model is built: + +```yaml +sources: + - name: raw_orders + columns: + - name: id + audits: + - not_null + - unique + audits: + - expression_is_true: + name: no future orders + expression: "ordered_at <= CURRENT_TIMESTAMP" +``` + +If a source audit with `error` severity fails, all downstream models that depend on that source are blocked. + +## Type enforcement + +Type enforcement is implicit for sources. If any column declares a `type`, SQLBuild activates source type enforcement, casts that column during source resolution, and uses declared types for schema-change detection: + +```yaml +sources: + - name: raw__customers + expression: | + SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name + columns: + - name: id + type: INTEGER + - name: first_name + - name: last_name +``` + +In this example, only `id` has a type declared, so only `id` is cast. Columns without a `type` are passed through unchanged. + +For expression sources, SQLBuild probes the expression's output columns and builds a projection that casts typed columns while preserving the rest. For table sources, it uses warehouse metadata to validate that declared column names exist and applies casts accordingly. + +You can explicitly set `type_enforcement: false` on a source to disable casting even when column types are declared. + +## Managed sources (loaders) + +Sources can be loaded by Python functions instead of pointing at existing tables or inline expressions. Set `managed: true` to bind a source to the `@loader` function **of the same name**, and SQLBuild will call it to populate the source table: + +```yaml +sources: + - name: raw_customers + managed: true + write_strategy: table + columns: + - name: id + type: INTEGER + - name: name + type: VARCHAR +``` + +Managed sources support incremental write strategies (`table`, `append`, `delete_insert`, `merge`), cursor-based loading, and concurrent execution. + +See [Loaders](python-nodes/loaders.md) for the full guide on writing loader functions, write strategies, the loader context API, and auto-load behavior during builds. + +### Declarative integrations (no Python) + +For common ingestion you can declare the source entirely in YAML, with no `@loader` function. SQLBuild generates the loader for you and runs it during the build: + +- [dlt](../integrations/dlt.md) - declare `dlt_sources` for `rest_api`, `sql_database`, and `filesystem` sources. +- [ingestr](../integrations/ingestr.md) - add an `ingestr` block to a source to pull from 50+ sources. + +## Source freshness + +Source freshness lets SQLBuild observe whether a source's data changed between runs. This feeds into +[planning and change detection](planning.md): changed observations propagate to downstream +models and can alter their planned action. Virtual environments can also use the signal for +[stale-driven execution](virtual-environments/building.md#stale-driven-execution). + +Configure freshness per source with a `freshness:` block: + +```yaml +sources: + - name: raw_events + schema: raw + table: events + freshness: + strategy: column + column: updated_at + type: timestamp + lag_tolerance: 15m +``` + +### Strategies + +| Strategy | Description | Required fields | +|----------|-------------|-----------------| +| `adapter` | Uses warehouse metadata (e.g. Snowflake `LAST_ALTERED`). No query against the source table. | None | +| `column` | Reads `MAX(column)` from the source table. | `column`, `type` | +| `sql` | Runs a custom query that returns a single scalar value. | `query`, `type` | + +#### adapter + +```yaml +freshness: + strategy: adapter +``` + +Uses adapter-level table metadata. Supported on Snowflake, BigQuery, Databricks, PostgreSQL, DuckDB, and SQL Server. Does not support `type`, `column`, or `query`. + +#### column + +```yaml +freshness: + strategy: column + column: updated_at + type: timestamp +``` + +Queries `MAX(column)` from the source table. The column must be a plain column name (no expressions; use `sql` strategy for those). Requires `type`. + +#### sql + +```yaml +freshness: + strategy: sql + query: "SELECT MAX(version_id) FROM raw.events" + type: integer +``` + +Runs an arbitrary SQL query that returns a single scalar. Requires `type`. Does not support `column`. + +### Type + +The `type` field declares the value kind for comparison: + +| Type | Description | +|------|-------------| +| `timestamp` | Datetime value. Supports `lag_tolerance`. | +| `integer` | Integer value. Change detected by exact comparison. | +| `string` | String value. Change detected by exact comparison. | + +### Lag tolerance + +`lag_tolerance` is optional and only valid with `type: timestamp`. It declares how much the observed timestamp can drift from the previous observation before being treated as a change: + +```yaml +freshness: + strategy: column + column: updated_at + type: timestamp + lag_tolerance: 2h +``` + +Accepts positive durations: `15m` (minutes), `2h` (hours), `1d` (days). If the current observation is within the tolerance of the previous one, the source is treated as unchanged. + +### Auto-observation + +Sources without an explicit `freshness:` block are auto-observed using the `adapter` strategy if: + +- The source has a physical table (not an expression source) +- The source is not managed +- The adapter supports table freshness metadata + +This means most unmanaged table sources get freshness tracking automatically on adapters that support it, with no configuration needed. + +Use [`sqb freshness`](../cli/freshness.md) to observe source freshness on demand without triggering a build. See [Source freshness](planning/source-freshness.md) for how freshness feeds into change-aware builds. + +### Freshness config reference + +| Field | Description | +|-------|-------------| +| `strategy` | Observation strategy: `adapter`, `column`, or `sql` | +| `type` | Value kind: `timestamp`, `integer`, or `string` | +| `column` | Column name for `column` strategy | +| `query` | SQL query for `sql` strategy | +| `lag_tolerance` | Duration tolerance for timestamp comparisons (e.g. `15m`, `2h`, `1d`). Only valid with `type: timestamp`. | + +## Config reference + +| Field | Description | +|-------|-------------| +| `name` | Source name, used in `__source("name")` references | +| `database` | Target database (optional) | +| `schema` | Physical source schema (optional). For managed sources, overrides target `loader_schema`; otherwise managed loaders fall back to `loader_schema`, then target `schema`. | +| `table` | Target table name (defaults to `name` if omitted) | +| `expression` | Inline SQL expression (alternative to table reference) | +| `managed` | Set to `true` to bind the source to the `@loader` function of the same name (see [Loaders](python-nodes/loaders.md)) | +| `write_strategy` | How the loader writes data: `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | +| `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | +| `unique_key` | Merge key column(s) (required for `merge`) | +| `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | +| `description` | Human-readable description | +| `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | +| `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | +| `columns` | Column declarations with optional types and audits | +| `audits` | Source-level audits | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md new file mode 100644 index 0000000000..ce65030bb4 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -0,0 +1,604 @@ + + +# Testing + +> SQL unit tests and multi-model tests with macro support, assertions, and model chaining. + +Online: https://docs.sqlbuild.com/concepts/testing + +## Contents + +- How tests work +- CTE conventions +- Multi-model tests +- Mocking refs and seeds +- Mocking table functions in model tests +- Multiple expected models +- Macro-powered mocks +- Macro mocking +- Assertions +- Test modes +- Multiple tests per file +- Repeating test logic +- Test placement +- Running tests + +SQLBuild supports SQL-native unit tests that validate model logic by comparing actual query results against expected values. Tests can chain across multiple models (multi-model tests), use macros for reusable mock data, and include zero-row assertions. + +For end-to-end testing across many models with physical warehouse relations, see [Scenarios](scenarios.md). + +## How tests work + +A test file defines mock inputs and expected outputs using CTEs. SQLBuild substitutes the mock data into the real model SQL, executes it, and compares the result against the expected CTE using `EXCEPT` queries. Zero mismatched rows means the test passes. + +```sql +-- tests/unit/test_stg_orders.sql +TEST(); + +WITH +__source__raw__orders AS ( + SELECT + 1 AS id, + 100 AS customer_id, + 2 AS waffle_type_id, + 3 AS quantity, + '2026-04-01 10:00:00' AS ordered_at, + 'completed' AS status +), +__expected__stg_orders AS ( + SELECT + 1 AS order_id, + 100 AS customer_id, + 2 AS waffle_type_id, + 3 AS quantity, + '2026-04-01 10:00:00' AS ordered_at, + 'completed' AS status +) +SELECT 1 +``` + +The test: +1. Mocks the `raw__orders` source with the `__source__raw__orders` CTE +2. Runs the real `stg_orders` model SQL with the mock substituted in +3. Compares the output against `__expected__stg_orders` +4. Passes if row counts match and there are zero mismatched rows + +When an expected-output comparison fails, SQLBuild reports the unexpected and missing row counts +and best-effort samples from each direction. Samples are deliberately bounded to three rows, 12 +columns, and 120 characters per value, and pass through diagnostic redaction. They are also present +as structured `unexpected_samples` and `missing_samples` in JSON output. Sampling failure never +replaces the known test failure. + +When exactly one row exists in each difference direction and the bounded samples have the same +columns, SQLBuild also aligns the rows and reports only changed columns with their redacted actual +and expected values. It does not guess an alignment when several rows differ or sampling is +incomplete. + +Before opening a warehouse connection, SQLBuild statically checks fixture shapes when SQL analysis +is enabled. Missing columns are reported together with the test path, fixture resource, and models +that read them. Statically provable collection-versus-scalar type conflicts identify the affected +column and suggest explicit `CAST`, `ARRAY_CONSTRUCT`, or `PARSE_JSON` expressions. SQLBuild never +invents missing values. + +The trailing `SELECT 1` is required as a ceremonial closing statement. + +## CTE conventions + +| Prefix | Purpose | +|--------|---------| +| `__source__` | Mock a source. Replaces `__source("")` in the model SQL. | +| `__ref__` | Mock a model. Replaces `__ref("")` in the model SQL. | +| `__seed__` | Mock a seed. Replaces `__seed("")` in the model SQL. | +| `__table_fn__` | Mock a table function in model mode. Replaces the complete `__table_fn("")(...)` invocation with the fixture relation. | +| `__expected__` | Define expected output for a model. SQLBuild resolves the model's real SQL and compares against this. | +| `__assert__` | Zero-row assertion. Passes if the query returns no rows; fails with the returned rows as diagnostics. | +| `__macro__` | Mock a macro. Replaces every `@(...)` call with the mock value. | + +Any CTE without one of these prefixes is treated as a helper CTE, available to all mock and model SQL in the test. + +## Multi-model tests + +Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL. + +```sql +-- Mock two sources, assert on the final mart. +-- stg_orders and stg_payments resolve automatically from their real SQL. +TEST(); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 2 AS waffle_type_id, 3 AS quantity, + '2026-04-01 10:00:00' AS ordered_at, 'completed' AS status +), +__source__raw__payments AS ( + SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, + 'credit_card' AS method, '2026-04-01 10:01:00' AS paid_at, 'success' AS status +), +__expected__fact_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 1500 AS payment_amount_cents, + 'credit_card' AS payment_method +) +SELECT 1 +``` + +SQLBuild topologically sorts the expected models, resolves each intermediate model's real SQL with mocks substituted, and chains the outputs forward. Every model between the mocked sources and the expected model is computed automatically. + +Inspect that boundary without opening a warehouse connection: + +```bash +sqb test --select fact_orders --inspect +``` + +The resolved plan lists mocked sources, refs, seeds, real models in execution order, expected +models, and any unsatisfied leaf dependencies. A mocked ref is called out explicitly because its +real model SQL will not execute. Inspection exits non-zero when the plan contains unresolved +dependency errors. + +## Mocking refs and seeds + +You can mock models directly with `__ref__` and seeds with `__seed__`, not just sources. This skips the model's real SQL (or the seed's real CSV data) and provides controlled data instead: + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT + 1 AS order_id, + 100 AS customer_id, + 2 AS waffle_type_id, + 3 AS quantity, + CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, + 'completed' AS status +), +__ref__stg_payments AS ( + SELECT + 10 AS payment_id, + 1 AS order_id, + 2850 AS amount_cents, + 'card' AS payment_method, + CAST('2026-04-01 10:05:00' AS TIMESTAMP) AS paid_at, + 'success' AS payment_status +), +__seed__waffle_types AS ( + SELECT + 2 AS waffle_type_id, + 'Liege' AS waffle_name, + 'sweet' AS category, + 950 AS price_cents +), +__expected__fact_orders AS ( + SELECT + 1 AS order_id, + 100 AS customer_id, + 2 AS waffle_type_id, + 'Liege' AS waffle_name, + 'sweet' AS waffle_category, + 3 AS quantity, + 2850 AS line_total_cents, + CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, + 'completed' AS order_status, + TRUE AS is_completed_order, + 'card' AS payment_method, + 'success' AS payment_status, + 2850 AS payment_amount_cents +) +SELECT 1 +``` + +You can mix `__source__`, `__ref__`, and `__seed__` mocks in the same test. As long as every leaf dependency is satisfied (either by a source mock, a ref mock, a seed mock, or by being in the expected chain), the test resolves. + +## Mocking table functions in model tests + +Use `__table_fn__` when the model under test reads a managed table function but the test +should provide rows directly. SQLBuild replaces the complete invocation, including its argument +expressions, with the fixture relation. The mocked function is not deployed as a test dependency. + +```sql +-- models/customer_order_totals.sql reads: +-- __table_fn("customer_orders")(42) +TEST(); + +WITH +fixture_orders AS ( + SELECT 101 AS order_id, 42 AS customer_id, 2500 AS amount_cents +), +__table_fn__customer_orders AS ( + SELECT order_id, customer_id, amount_cents FROM fixture_orders +), +__expected__customer_order_totals AS ( + SELECT 42 AS customer_id, 2500 AS total_cents +) +SELECT 1 +``` + +The fixture name must match a declared table function. Helper CTEs are available inside the fixture, +just as they are for source, ref, and seed mocks. A table-function fixture can also be used directly +inside a zero-row assertion. + +This is separate from `TEST (mode table_fn)`, which tests the table function implementation itself. +Model-mode fixtures isolate a model from that implementation; table-function mode executes and +compares the real function. + +## Multiple expected models + +A single test can assert on multiple models. SQLBuild resolves and compares each one independently: + +```sql +TEST(); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 2 AS waffle_type_id, 3 AS quantity, + '2026-04-01 10:00:00' AS ordered_at, 'completed' AS status +), +__source__raw__payments AS ( + SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, + 'credit_card' AS method, '2026-04-01 10:01:00' AS paid_at, 'success' AS status +), +__source__raw__customers AS ( + SELECT 100 AS id, 'Leslie' AS first_name, 'Knope' AS last_name, + 'leslie@pawnee.gov' AS email, '2026-01-15 09:00:00' AS created_at +), +__expected__fact_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 1500 AS payment_amount_cents, + 'credit_card' AS payment_method +), +__expected__dim_customers AS ( + SELECT 100 AS customer_id, 1 AS total_orders, 1500 AS lifetime_spend_cents +) +SELECT 1 +``` + +If the expected models form a chain (e.g. `stg_orders` feeds into `fact_orders` which feeds into `dim_customers`), SQLBuild resolves them in dependency order, using the output of earlier steps as input to later ones. + +When a test defines `__expected__model_name` output, it may also use the public enums and constants +available to that model. A test that checks several models may use the public enums and constants +available to each of them. Public names are unique, so those values cannot conflict. + +Only explicit expected CTEs create grants. A matching test filename, `__ref__` mock, or nearby model path does not. Model-private declarations and macros are never granted through expected models. + +## Macro-powered mocks + +Because unit tests are written in SQL, they support macro calls. This lets you write reusable mock generators instead of copy-pasting mock data across test files: + +```sql +TEST(); + +WITH +__source__raw__orders AS ( + @mock_orders() +), +__source__raw__payments AS ( + SELECT 1 AS payment_id, 1 AS order_id, 1500 AS amount_cents, 'credit_card' AS method +), +__expected__fact_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 1500 AS total_cents, + 'credit_card' AS payment_method +) +SELECT 1 +``` + +The `@mock_orders()` call expands at compile time to whatever SQL the Python macro function returns. + +Test SQL uses macros, enums, and constants available from the test file's directory under +`tests/unit/`. Public enums and constants available to a model are also available when the test +defines `__expected__model_name` output for that model. Model-private values are not available to +tests. See [How Visibility Works](declaration-scopes/visibility.md#tests-and-expected-output). + +## Macro mocking + +When a model uses macros that you want to control in tests (e.g. target-specific logic, dynamic SQL generation), you can override their output with `__macro__` CTEs: + +```sql +TEST(); + +WITH +__macro__country_filter AS ( + SELECT 'country_code = ''US''' +), +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 'US' AS country_code, 'completed' AS status +), +__expected__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status +) +SELECT 1 +``` + +When a `__macro__` mock is defined, every call to `@country_filter(...)` in any model SQL resolved by the test is replaced with the mock value (`country_code = 'US'`). The macro's actual Python function is not called, and the arguments are ignored. + +The mock value must be a single `SELECT` with one string literal. Use doubled single quotes for quotes within the value (standard SQL escaping). + +This is useful for: +- Testing models that use target-specific macros without depending on target config +- Controlling dynamic SQL generation to produce predictable test inputs +- Isolating model logic from macro implementation details + +## Assertions + +Unit tests can include `__assert__` CTEs for property-based checks. An assertion passes if the query returns zero rows - any returned rows are failing examples. + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 3 AS quantity, + CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at, 'completed' AS status +), +__assert__order_ids_are_not_null AS ( + SELECT * FROM __ref("stg_orders") WHERE order_id IS NULL +) +SELECT 1 +``` + +Assertions can be mixed with `__expected__` CTEs in the same test, or used on their own. They are useful when the natural check is "no rows should violate this rule" rather than "the output should exactly equal these rows" - for example, duplicate checks, negative-value constraints, or conditional business rules. + +During `sqb test` and `sqb build`, assertion results appear as nested check rows alongside expected comparisons. + +## Test modes + +By default, `TEST()` runs in model mode - mocking sources/refs and comparing model outputs. Three additional modes let you test reusable logic directly without needing a model chain. + +### Macro tests + +Test macro output by calling the macro in `__macro_actual__` and comparing against `__macro_expected__`: + +```sql +TEST (mode macro, name "calculates_line_total_cents"); + +WITH +input_values AS ( + SELECT 950 AS price_cents, 3 AS quantity +), +__macro_actual__ AS ( + SELECT @line_total_cents("price_cents", "quantity") AS line_total_cents + FROM input_values +), +__macro_expected__ AS ( + SELECT 2850 AS line_total_cents +) +SELECT 1 +``` + +Macros are compile-time code, so macro tests expand the macro at compile time and compare the results. During `sqb build`, macro tests run before any model that uses the tested macro. + +### UDF tests + +Test scalar UDFs by calling them in `__udf_actual__` and comparing against `__udf_expected__`: + +```sql +TEST (mode udf, name "detects_completed_orders"); + +WITH +input_values AS ( + SELECT 'completed' AS order_status + UNION ALL + SELECT 'pending' AS order_status +), +__udf_actual__ AS ( + SELECT + order_status, + __udf("udf__is_completed_order")(order_status) AS is_completed_order + FROM input_values +), +__udf_expected__ AS ( + SELECT 'completed' AS order_status, TRUE AS is_completed_order + UNION ALL + SELECT 'pending' AS order_status, FALSE AS is_completed_order +) +SELECT 1 +``` + +UDFs are warehouse objects, so the function is created before the test runs. During `sqb build`, UDF tests run after the function is created but before any model that uses it. + +### Table function tests + +Test table functions by calling them in `__table_fn_actual__` and comparing against `__table_fn_expected__`: + +```sql +TEST (mode table_fn, name "returns_customer_orders"); + +WITH +__table_fn_actual__ AS ( + SELECT order_id, order_status, is_completed_order + FROM __table_fn("table_fn__customer_orders")(1) +), +__table_fn_expected__ AS ( + SELECT 1 AS order_id, 'completed' AS order_status, TRUE AS is_completed_order + UNION ALL + SELECT 2 AS order_id, 'completed' AS order_status, TRUE AS is_completed_order +) +SELECT 1 +``` + +Table function tests run after the function is created. Since table functions are terminal (models cannot depend on them), these tests validate the function independently. + +### Mode rules + +Each mode has strict CTE validation: + +| Mode | Actual CTE | Expected CTE | Allowed | +|------|-----------|--------------|---------| +| `model` (default) | existing model-chain syntax | `__expected__` | `__source__`, `__ref__`, `__seed__`, `__assert__`, `__macro__` | +| `macro` | `__macro_actual__` | `__macro_expected__` | Helper CTEs, `@macro()` calls in actual | +| `udf` | `__udf_actual__` | `__udf_expected__` | Helper CTEs, `__udf()` calls in actual | +| `table_fn` | `__table_fn_actual__` | `__table_fn_expected__` | Helper CTEs, `__table_fn()` calls in actual | + +CTE prefixes from other modes are not allowed. For example, `__source__` in a macro test or `__macro_actual__` in a model test will produce a clear error pointing you to the right mode. Expected CTEs must not call macros, UDFs, or table functions - they should be independent, inspectable expected data. + +## Multiple tests per file + +A single test file can contain multiple `TEST()` blocks. Each block must have a unique `name`: + +```sql +TEST (name "completed_orders_only"); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 'completed' AS status +), +__expected__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status +) +SELECT 1 + +TEST (name "cancelled_orders_excluded"); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 'cancelled' AS status +), +__expected__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 'cancelled' AS status +) +SELECT 1 +``` + +A file with a single test can omit the `name` field. Files with multiple tests require names on every block. + +## Repeating test logic + +SQLBuild supports three repetition patterns: + +| Pattern | Use when | Result identity | +|---------|----------|-----------------| +| Multiple `TEST` blocks | Cases need different SQL shapes | One result per block | +| Macro or `VALUES` case table | One aggregate comparison is sufficient | One result for the table | +| Native parameters and cases | Cases share a template but need independent status | One result per named case | + +### Native independent cases + +Declare a typed schema and ordered named cases in one header: + +```sql +TEST ( + name "order_status_maps_source_states", + parameters ( + source_status string, + expected_status string, + ), + cases ( + completed (source_status "completed", expected_status "completed"), + cancelled (source_status "cancelled", expected_status "excluded"), + pending (source_status "pending", expected_status "open"), + ), +); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, @param("source_status") AS status +), +__expected__stg_orders AS ( + SELECT 1 AS id, @param("expected_status") AS status +) +SELECT 1 +``` + +The cases report independently as `order status: maps source states [completed]`, `[cancelled]`, +and `[pending]`. Authored order controls display order. Selecting `stg_orders` selects every case by +default. Add `--case` to run one named case without changing the test file: + +```bash +sqb test --select stg_orders --case cancelled +``` + +If the case does not exist within the parent/model selection, SQLBuild reports the available case +names. + +Supported scalar types are `string`, `integer`, `boolean`, `float`, and exact `decimal`. Decimal +values are quoted, such as `tax_rate "0.2000"`, so they never pass through binary float. Boolean +and integer remain distinct. Raw SQL and collection parameters are rejected. + +Nullable parameters use an expanded declaration and are valid where SQL can infer the null type: + +```sql +parameters ( + cancellation_reason (type string, nullable true), +), +cases ( + completed (cancellation_reason null), +) +``` + +Every case provides exactly the declared parameters, and every declaration must appear as +`@param("name")` in the template. SQLBuild rejects missing, extra, incompatible, undeclared, and +unused values before execution. + +Expansion is deterministic: + +```text +parse TEST template and cases + -> render typed @param values with the active adapter + -> expand @const and @enum references + -> expand macros + -> validate SQL and compile expected models/assertions +``` + +Values can therefore feed macro arguments, for example +`@order_fixture(@param("source_status"))`. Parameters work in model, macro, UDF, and table-function +modes and preserve every expected model in multi-model tests. + +Text and JSON output include parent/case identity and safe typed values. Compile JSON, manifests, +DAG checks, and compiled/runtime SQL artifacts retain source, block, case, and content-fingerprint +provenance. Changing one case changes that case's fingerprint without reassigning another case's +stable identity. + +Scenarios are intentionally not parameterized. Keep one scenario per coherent business world so +capture and replay identity remains explicit. + +### Aggregate SQL case tables + +When independent status is unnecessary, a normal SQL case table remains concise: + +```sql +WITH cases(case_name, source_status, expected_status) AS ( + VALUES + ('completed', 'completed', 'completed'), + ('cancelled', 'cancelled', 'excluded') +), +__expected__status_mapping AS ( + SELECT case_name, expected_status FROM cases +) +SELECT 1 +``` + +This produces one aggregate test result. Use native cases when each row must pass or fail +independently. + +Quoted strings in `VALUES` rows may contain commas, brackets, and parentheses. SQLBuild preserves +these literals while extracting test CTEs; use normal SQL quote escaping for embedded quotes. + +## Test placement + +Place unit test files under `tests/unit/` in your project directory. SQLBuild discovers all `.sql` files in this directory recursively. + +``` +tests/ + unit/ + test_stg_orders.sql # model test + test_fact_orders.sql # model test with assertion + test_daily_revenue_chain.sql # multi-model test + test_line_total_cents_macro.sql # macro test + test_is_completed_order_udf.sql # UDF test + test_customer_orders_table_fn.sql # table function test + scenarios/ + ... +``` + +## Running tests + +Tests run automatically during `sqb build` in DAG order, before their target model is materialized. + +Run tests standalone: + +```bash +sqb test +``` + +Scope to specific models: + +```bash +sqb test --select stg_orders +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments.md new file mode 100644 index 0000000000..a5048588f5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments.md @@ -0,0 +1,117 @@ + + +# Overview + +> Version-controlled SQL pipeline environments with instant promotion and rollback. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments + +## Contents + +- When to use virtual environments +- How it works +- Try it +- Example workflow +- What's next + +Virtual environments are in alpha. The core workflow (build, promote, rollback, reconcile) is functional and tested across supported adapters. The API and CLI surface may evolve based on feedback. Do not use virtual environments for production workloads yet. + +Virtual environments (VDEs) let you build, preview, and promote SQL pipeline changes without recomputing models. Each VDE is a set of pointers to versioned physical relations. Creating a new VDE is instant (pointer copy, no data duplication), and promoting one VDE into another is a view swap, not a rebuild. + +## When to use virtual environments + +- **PR preview environments** - build a VDE per pull request against a production warehouse, validate with audits and tests, then promote the built versions to production without rebuilding +- **Blue/green deployments** - build into a staging VDE, promote to production atomically +- **Multi-developer isolation** - each developer works in their own VDE without conflicting with others, sharing physical versions when code is identical +- **Instant rollback** - revert production to a prior finalized state by restoring a checkpoint's pointer set + +Virtual environments are opt-in via `virtual_environments = true` (under `[settings]`) and require a state store. Projects that don't need environment isolation or promotion workflows should use the default direct mode. + +## How it works + +### Targets and virtual environments + +In virtual mode there are two separate concepts: + +**Targets** are the existing SQLBuild build contexts from `sqlbuild_project.toml` (e.g. `dev`, `prod`). They choose the warehouse connection, schema, and state database. In virtual mode they are sometimes called *physical targets* to distinguish them from VDEs. + +**Virtual data environments (VDEs)** are versioned pointer sets stored in the state database. They choose which model versions the logical views point to. + +```bash +sqb --target prod build --virtual-env pr_123 +``` + +This means: use the `prod` physical target (warehouse, state DB), but build into the `pr_123` virtual environment. + +### Version identity + +Model versions are identified by content hashes, not sequence numbers. The hash includes the model's query SQL, version-identity config, and upstream version hashes. If two developers compile identical code with identical upstream versions, they get the same hash and reuse the same physical relation. No data is duplicated. + +### Physical and logical relations + +Virtual mode creates two types of warehouse objects: + +- **Physical version relations** store actual data: `__sqb_physical.__v_` +- **Logical VDE views** point to physical versions: `__.` as `SELECT * FROM ` + +Users query the logical views. SQLBuild manages the physical layer. + +### Zero-copy branching + +Creating a new VDE from a baseline copies only pointer rows in the state database, not data. Unchanged models share the same physical relations across VDEs. + +### Instant promotion + +Promoting VDE `pr_123` to `prod` updates the pointer rows and refreshes the logical views. No models are rebuilt. If every model in the source VDE has already been built and validated, promotion is a metadata operation. + +## Try it + +```bash +sqb playground --template virtual +cd sqlbuild-playground +sqb state init +sqb build +sqb build --virtual-env pr +sqb diff dev:pr --schema-only --allow-partial-diff +sqb promote --from pr --to dev +sqb state checkpoints list +sqb rollback +``` + +This creates a self-contained DuckDB project with loaders, models, tests, scenarios, and a local state store. No warehouse credentials required. + +## Example workflow + +```bash +# Initialize state store +sqb state init + +# First build creates physical versions and default VDE +sqb build + +# Create a PR preview VDE +sqb build --virtual-env pr_123 + +# Modify a model, rebuild in the PR VDE +sqb build --virtual-env pr_123 + +# Compare VDEs +sqb diff dev:pr_123 + +# Promote PR versions to the default VDE +sqb promote --from pr_123 --to dev +``` + +## What's next + +- [Setup](virtual-environments/setup.md) - configuration and state initialization +- [Building](virtual-environments/building.md) - virtual builds, partial builds, seeded incrementals +- [Promotion](virtual-environments/promotion.md) - promoting VDEs +- [Rollback](virtual-environments/rollback.md) - checkpoints and rollback +- [Clone](virtual-environments/clone.md) - hydrating physical versions from a source warehouse +- [Diff](virtual-environments/diff.md) - comparing VDE ref sets +- [Adopt and Detach](virtual-environments/adopt-detach.md) - migrating existing projects +- [Reconcile](virtual-environments/reconcile.md) - diagnosing and repairing drift +- [Locks](virtual-environments/locks.md) - concurrent access control +- [Janitor](virtual-environments/janitor.md) - cleanup and retention +- [Recovery](virtual-environments/recovery.md) - what to do when things break diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/adopt-detach.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/adopt-detach.md new file mode 100644 index 0000000000..21f75fbbb5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/adopt-detach.md @@ -0,0 +1,123 @@ + + +# Adopt and Detach + +> Migrate existing projects to and from virtual mode. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/adopt-detach + +## Contents + +- Adopt +- Detach +- Interrupted operations + +Adopt converts an existing stateless project into virtual mode. Detach reverses the process. Both are interactive operations that require typed confirmation. + +## Adopt + +`sqb state adopt` takes existing warehouse relations (tables and views) and converts them into versioned physical relations with VDE views at the original names. + +### Prerequisites + +1. State store must be initialized (`sqb state init`) +2. `unsuffixed_virtual_env` must be configured so existing object names are preserved: + +```toml +[targets.dev.state] +backend = "duckdb" +schema = "sqlbuild_state" +unsuffixed_virtual_env = "dev" + +[targets.dev.state.connection] +database = "state.duckdb" +``` + +Without `unsuffixed_virtual_env`, adopt blocks with a config error. This is intentional - without it, existing relations would be renamed to suffixed schemas (e.g. `dev__dev.fact_orders`), breaking existing consumers. + +### What happens + +1. For each model, the existing table is moved/renamed into the physical schema (`dev__sqb_physical.fact_orders__v_`) +2. A logical VDE view is created at the original name (`dev.fact_orders`) pointing to the physical version +3. Model versions, physical relations, VDE record, and VDE refs are persisted in state + +### Usage + +```bash +sqb state adopt --allow-copy +``` + +The command prints an adoption plan and requires typed confirmation: + +``` +Type "adopt dev" to confirm: adopt dev +``` + +`--allow-copy` is required when the adapter does not support native same-schema rename (cross-schema moves, some adapters). Without it, adopt blocks with `requires --allow-copy` if a copy fallback would be needed. + +### View models + +View models are adopted the same way - a versioned physical view is created in the physical schema, and the original name becomes a logical VDE view. + +## Detach + +`sqb state detach` reverses adoption, collapsing a VDE back into normal stateless relations. + +### Prerequisites + +The VDE must be `finalized`. If the VDE is working (has stale models), detach blocks: + +``` +error: detach requires a finalized virtual environment +``` + +Build the VDE to finalize it first, or resolve any pending changes. + +### What happens + +1. For each table model, the physical version is moved/renamed back to the original target name +2. For each view model, the view is recreated from compiled SQL at the original target name (not copied from the physical ref) +3. The VDE is marked `detached` in state +4. VDE refs are preserved for audit/recovery and janitor protection + +### Usage + +```bash +sqb state detach --allow-copy +``` + +Requires typed confirmation: + +``` +Type "detach dev" to confirm: detach dev +``` + +### After detach + +A detached VDE is blocked from further virtual operations: + +- `sqb build` blocks with "virtual environment is detached" +- `sqb promote --from ` or `--to ` blocks +- `sqb rollback` on a detached VDE blocks + +The project can continue operating in direct mode, or you can re-adopt to return to virtual mode. + +### Detached VDE cleanup + +Detached VDE refs and state rows are cleaned up by the [janitor](janitor.md), not by detach itself. This preserves refs for recovery if detach fails partway through. + +## Interrupted operations + +Adopt and detach are multi-step warehouse operations that cannot be wrapped in a single transaction. If an operation fails partway through: + +- A `failed` operation record is persisted in state with the error message +- VDE refs and checkpoints remain intact +- The warehouse may have partial artifacts (e.g. table moved to physical schema but view not yet created) + +Recovery path: + +1. Run `sqb reconcile` to diagnose the current state +2. Use `sqb reconcile repair-view` or manual warehouse repair as needed +3. Retry the adopt or detach operation + +There is no automatic resume command. SQLBuild records the failure state and leaves recovery to the operator. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/building.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/building.md new file mode 100644 index 0000000000..d846e664e7 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/building.md @@ -0,0 +1,232 @@ + + +# Building + +> Virtual builds, VDE creation, partial builds, and seeded incrementals. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/building + +## Contents + +- Default VDE +- Explicit VDE +- Physical naming +- VDE status +- Partial builds +- Stale-driven execution +- Stale detection +- Seeded incremental builds +- Custom materializations +- Functions +- Plan + +Virtual builds create versioned physical relations and update VDE pointer sets. The build lifecycle is the same as direct mode (seeds, tests, models, audits), but model outputs are written to versioned physical tables and exposed through logical VDE views. + +Virtual builds run ingress (loaders and Python nodes that feed sources) as a separate phase before SQL model execution. This means independent SQL models that do not depend on loaders will wait for all ingress to complete before starting. Direct mode does not have this limitation. This keeps VDE state persistence simpler and safer but may add wall time when ingress is slow and independent SQL work is available. A future optimization may allow overlapping ingress with independent SQL execution. + +## Default VDE + +When you run `sqb build` in virtual mode without `--virtual-env`, the VDE name defaults to the active physical target name: + +```bash +sqb build +``` + +If the active environment is `dev`, this builds into VDE `dev`. On first run, it creates physical version relations and logical VDE views for all models. + +## Explicit VDE + +Use `--virtual-env` to build into a named VDE: + +```bash +sqb build --virtual-env pr_123 +``` + +If VDE `pr_123` does not exist, SQLBuild creates it by inheriting refs from the baseline VDE (the default VDE for the active physical target). Unchanged models share the same physical relations as the baseline - no data is copied. + +Only models with changed code, config, or upstream versions get new physical version relations. + +## Physical naming + +Physical version relations are stored in a dedicated schema: + +``` +__sqb_physical.__v_ +``` + +For example: `dev__sqb_physical.fact_orders__v_8f3a9c12` + +Logical VDE views are created in a VDE-suffixed schema: + +``` +__. +``` + +For example: `dev__pr_123.fact_orders` + +When `unsuffixed_virtual_env` is configured, the named VDE uses the base schema directly: + +``` +dev.fact_orders +``` + +## VDE status + +| Status | Meaning | +|--------|---------| +| `finalized` | All models match their expected version hashes. The VDE is coherent and eligible for promotion. | +| `active` | Some models are stale (partial build or pending changes). The VDE is a work in progress. | +| `detached` | The VDE has been detached via `sqb state detach`. Build, promote, and rollback are blocked. | + +## Partial builds + +Build a subset of models with `--select`: + +```bash +sqb build --virtual-env pr_123 --select fact_orders +``` + +Partial builds leave the VDE in `active` (working) status if downstream models remain stale. A +follow-up `sqb build --virtual-env pr_123 --changes-only` without `--select` builds the remaining +stale models to finalize the VDE. A plain `sqb build --virtual-env pr_123` runs the whole selection. + +### Stale upstream coherence + +If a selected model has stale required upstream models, the build blocks by default: + +```bash +sqb build --virtual-env pr_123 --select fact_orders +# error: selected models have stale required upstreams: stg_orders +``` + +Pass `--include-stale-upstreams` to expand the selection to the minimal set of stale ancestor models needed to make the selected scope coherent: + +```bash +sqb build --virtual-env pr_123 --select fact_orders --include-stale-upstreams +``` + +## Stale-driven execution + +Virtual environment builds run the full selection by default. Add `--changes-only` to limit a plan +or build to work whose expected version is not already bound in the target VDE: + +```bash +sqb plan --virtual-env pr_123 --changes-only +sqb build --virtual-env pr_123 --changes-only +sqb build --virtual-env pr_123 --select path:models/marts --changes-only +``` + +With selectors, SQLBuild intersects the selected scope with the stale set. Models with changed SQL or +configuration, pending backfills, changed functions, or affected upstreams remain in the plan; +models already bound to their expected version do not need another physical version. + +`--changes-only` is a virtual-mode execution option. Direct mode always runs its selected build +scope, although its plan still reports change reasons and uses them to choose incremental actions. +See [Planning and change detection](../planning.md) for the shared identity and cascade model. + +You can make stale-driven execution the default for virtual plans and builds: + +```toml +[settings] +virtual_environments = true +changes_only = true + +[targets.dev] +changes_only = true +``` + +The CLI flag takes precedence, followed by the selected target, local settings, and project +settings. + +### Run despite unchanged + +`run_despite_unchanged` lets a table remain eligible during stale-driven execution even when its +version identity is unchanged. This is useful when external data changes cannot be represented by +source freshness. + +```sql +MODEL ( + materialized table, + run_despite_unchanged "always", +); +``` + +- `always` runs whenever the model is selected. +- A duration such as `24h`, `30d`, or `90m` keeps the model eligible while the newest timestamp data + version from an upstream source is no older than that duration; it is not a periodic scheduler. + +Only table materializations support this setting. When it triggers, downstream models are also +stale. + +## Stale detection + +SQLBuild determines which models and seeds need rebuilding by comparing expected version hashes against bound version hashes in the VDE refs: + +1. **Expected local hash** - derived from the node's query SQL (for models), content hash (for seeds), version-identity config, and source freshness hashes +2. **Expected version hash** - local hash propagated through upstream dependencies (upstream hash changes cascade downstream) +3. **Bound version hash** - the hash currently stored in the VDE refs from the last successful build + +A node is stale when `bound != expected`. Stale nodes whose own local hash changed are roots (`query changed`, `config changed`, `function changed`). Others are stale due to `upstream changed`. + +Seeds participate in version identity the same way as models. They are loaded into versioned physical tables with logical VDE views on top, and their version hashes and refs are tracked per virtual environment in the state backend. Unchanged seeds are not reloaded. + +Source freshness observations and Python node identities are also persisted per virtual environment and included in version hash computation. When a source's observed data version changes, its downstream models become stale. See [Sources: Source freshness](../sources.md#source-freshness) for configuration. + +## Seeded incremental builds + +When an incremental model's version hash changes, SQLBuild seeds the new physical version from the prior physical version before running the incremental delta. This avoids full rebuilds of large incremental tables. + +| Adapter | Seed strategy | +|---------|--------------| +| Snowflake | Zero-copy clone | +| BigQuery | Table clone | +| Databricks | Deep clone | +| DuckDB, Postgres, SQL Server | CTAS copy | + +For append models with bounded replay (`replay_on_change bounded-7d`), the seed copies only rows before the replay window cutoff. The incremental delta then appends the bounded range without duplicating rows. + +## Custom materializations + +Custom materializations are supported in virtual mode. By default, SQLBuild seeds new physical versions using the direct clone/copy strategy before calling the custom `materialize` function. + +For custom materializations that need different seeding behavior, define a `prepare_version` function alongside `materialize`: + +```python +from sqlbuild.virtual.executor.models import VersionPrepareContext +from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult + +def prepare_version(ctx: VersionPrepareContext) -> None: + """Prepare the new physical destination from the prior version.""" + ctx.execute_sql(f"CREATE TABLE {ctx.destination} AS SELECT * FROM {ctx.prior_relation}") + +def materialize(ctx: MaterializationContext) -> MaterializationResult: + """Run the custom materialization logic against the prepared destination.""" + ... +``` + +If `prepare_version` is not defined, the framework uses the default clone/copy. Most custom materializations do not need to define it. + +`VersionPrepareContext` provides `prior_relation` (the source physical version), `destination` (the new physical destination relation), `adapter`, `connection`, `execute_sql()`, `config`, and `vars`. + +Custom materializations in virtual mode must write only to `ctx.destination`. Side-effect writes to other relations are not tracked by virtual state, not cleaned up by janitor, and not restored by rollback. + +## Functions + +Functions are published into the logical VDE schema, not the physical layer. Each VDE has its own copy of function definitions. Function versions are tracked in state and participate in promotion and rollback. + +## Plan + +Use `sqb plan` to preview what a virtual build would do without executing: + +```bash +sqb plan +sqb plan --virtual-env pr_123 +sqb plan --select fact_orders +``` + +Virtual plan output shows: +- Virtual environment name and status (finalized/working) +- Stale root count and root set +- Stale model count +- Query diffs from prior bound versions +- Remaining stale models after partial selection diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/clone.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/clone.md new file mode 100644 index 0000000000..ff3748de20 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/clone.md @@ -0,0 +1,77 @@ + + +# Clone + +> Hydrate physical versions from a source warehouse. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/clone + +In virtual mode, `sqb clone` hydrates physical version relations from a source warehouse into the target physical storage layer. It does not copy VDE pointer sets or create logical views - it copies the underlying physical data so that builds and promotions can reference those versions locally. + +This is useful for seeding a new target from an existing one, or restoring physical versions that were cleaned up by the janitor. + +As in direct mode, `--from` selects an origin namespace and clone-origin policy, not a +connection. `--to` (or the active target) supplies the only physical connection. The origin +physical namespace must be readable through that destination connection; SQLBuild never opens +the origin target connection or transfers data across accounts, servers, or files using a +second connection. + +## How it works + +Virtual clone looks up expected physical version relations in the source warehouse by their naming convention (`__sqb_physical.__v_`) and copies them into the target warehouse. It then registers the copied relations in the target state store. + +Target VDE refs and logical views are not changed. Clone only populates the physical layer. + +## Default mode + +Without `--virtual-env`, clone hydrates physical versions matching the current workspace's expected fingerprints: + +```bash +sqb clone --from prod --to dev +``` + +This computes expected version hashes from the current code, looks for matching physical relations in the source warehouse, and copies them into the target. + +## VDE ref mode + +With `--virtual-env`, clone hydrates physical versions referenced by a specific VDE in the target state store: + +```bash +sqb clone --from prod --to dev --virtual-env pr_123 +``` + +This reads VDE `pr_123`'s refs from the target state, looks for those physical relations in the source warehouse, and copies them into the target. Useful when a VDE's physical versions have been deleted (by janitor or manual cleanup) but the state refs still exist. + +## Selection + +Scope which models are hydrated: + +```bash +sqb clone --from prod --to dev --select fact_orders +``` + +## Missing source artifacts + +If a source physical relation does not exist, clone reports it as missing and continues with the remaining models. The exit code is non-zero if any models are missing. + +## Model version locks + +If a target model version is locked (another process is building that version), clone blocks by default: + +```bash +# Skip locked versions and hydrate the rest +sqb clone --from prod --to dev --skip-locked +``` + +## What clone does not do + +- Does not create or update VDE refs +- Does not create or refresh logical VDE views +- Does not read the source state database (it uses warehouse-level artifact discovery) +- Does not change the target VDE status + +Clone is a physical-layer operation. VDE pointer management is handled by [build](building.md) and [promote](promotion.md). + +## Comparison with direct-mode clone + +In direct mode, `sqb clone` copies model relations between targets using zero-copy cloning where supported. In virtual mode, clone hydrates versioned physical relations instead. The source and target are still physical targets, but the copied objects are physical version relations rather than normal model targets. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/diff.md new file mode 100644 index 0000000000..b99d70b20b --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/diff.md @@ -0,0 +1,61 @@ + + +# Diff + +> Compare virtual data environments. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/diff + +In virtual mode, `sqb diff` compares VDE ref sets within the same physical target. It shows which models have different version hashes and, for changed models, reports schema and row-level differences. + +## Basic usage + +```bash +sqb diff dev:pr_123 +``` + +The format is `:`. Both VDEs must exist in the active physical target's state store. + +## What diff shows + +1. **Ref comparison** - which models have different version hashes between the two VDEs +2. **Schema differences** - column additions, removals, and type changes for changed models +3. **Row differences** - row counts, matched/unmatched rows, and changed column values + +Identical refs are skipped by default so the output focuses on models that actually differ. A model with a different version hash is included in the comparison even if its resulting data happens to be identical. + +## Options + +```bash +# Schema differences only (no row comparison) +sqb diff dev:pr_123 --schema-only + +# Full row-level comparison +sqb diff dev:pr_123 --full + +# Compare specific models +sqb diff dev:pr_123 --select fact_orders + +# No color output +sqb --no-color diff dev:pr_123 +``` + +## Working VDE guard + +If either VDE is working (has stale models that haven't been built yet), diff is blocked by default because the comparison may be incomplete: + +```bash +# Blocked +sqb diff dev:pr_123 + +# Allowed with explicit override +sqb diff dev:pr_123 --allow-partial-diff +``` + +This guard prevents misleading diff output when one VDE has pending changes that haven't been materialized yet. + +## Comparison with direct-mode diff + +In direct mode, `sqb diff prod:dev` compares physical target schemas and data directly in the warehouse. In virtual mode, `sqb diff dev:pr_123` compares VDE pointer sets within a single physical target, then inspects the physical versions those pointers reference. + +The output format is the same - schema diffs, row counts, changed columns, and example rows. The difference is what is being compared: physical targets vs virtual pointer sets. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/janitor.md new file mode 100644 index 0000000000..fb5f3137ea --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/janitor.md @@ -0,0 +1,76 @@ + + +# Janitor + +> Cleanup of virtual environment artifacts and retention policies. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/janitor + +The janitor manages cleanup of virtual mode artifacts: expired VDEs, old checkpoints, unreferenced physical versions, stale state backups, and expired locks. All cleanup runs through `sqb janitor` with preview and confirmation. + +## Physical version protection + +Physical version relations are never deleted while referenced by: + +- Any active (non-detached) VDE's current refs +- Any retained checkpoint's refs + +The janitor resolves the complete set of protected physical relations before considering any deletions. + +## Checkpoint retention + +Checkpoints are retained according to `[janitor] max_checkpoints` (default: 20): + +```toml +[janitor] +max_checkpoints = 20 +``` + +Values below 1 are rejected. Checkpoint creation never prunes history - pruning is janitor-only, behind preview and confirmation. + +When old checkpoints are pruned, physical versions that were protected only by those checkpoints become eligible for deletion in the same janitor run (if not protected by active VDE refs or remaining checkpoints). + +## Expired VDE cleanup + +Non-active, non-detached VDEs older than `[janitor] retention_days` are pruned: + +```toml +[janitor] +retention_days = 30 +``` + +Active/default VDEs are always protected. This catches abandoned PR preview VDEs that were never promoted or cleaned up. + +## Detached VDE cleanup + +Detached VDEs (created by `sqb state detach`) are eligible for cleanup after `retention_days`: + +- `retention_days = 0` makes them eligible immediately +- Refs, function refs, and VDE row are deleted +- Checkpoint rows remain governed by checkpoint retention +- Physical versions newly unprotected by removed refs can be deleted in the same run + +Active VDE refs continue to protect physical versions even when detached refs are pruned. + +## State cleanup + +The janitor also prunes state-only artifacts: + +- **Migration backups**: old backup schemas are deleted, but the latest backup is always preserved +- **Expired locks**: lock rows with `expires_at` in the past are deleted; active locks are never touched + +## Execution order + +The janitor drops warehouse physical versions before pruning state rows. If a warehouse drop fails, the corresponding state refs are preserved so the janitor can retry on the next run. + +## Usage + +```bash +# Preview what would be cleaned +sqb janitor + +# Execute cleanup with confirmation +sqb janitor --auto-approve +``` + +The janitor shows a preview of all candidates (checkpoints, VDEs, physical versions, state items) and requires confirmation before executing any destructive operations. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/locks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/locks.md new file mode 100644 index 0000000000..1d147cde52 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/locks.md @@ -0,0 +1,60 @@ + + +# Locks + +> Advisory locks for concurrent access control. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/locks + +Virtual mode uses advisory locks with TTL to prevent concurrent operations from conflicting. Locks are stored in the state database and are scoped to specific resources. + +## Lock types + +| Lock key | Protects | Acquired by | +|----------|----------|-------------| +| `virtual_env:` | VDE pointer set and views | build, promote, rollback, reconcile, adopt, detach | +| `model_version::` | Physical version creation | build (per model version) | +| `state_migration` | State schema changes | state init, migrate, rollback, reset | + +Different VDEs can be locked concurrently. Two builds targeting different VDEs do not block each other. + +## Lock behavior + +- Locks have an expiry time (`expires_at`). Active locks are those where `expires_at > now`. +- When a lock is successfully released, the lock row is deleted. +- Acquiring a lock over an expired lock replaces it. +- Owner identity is checked on release - only the owner that acquired the lock can release it. + +## When locks block + +If a lock is active and an operation requires it, the operation fails immediately with a clear error: + +``` +error[S014]: virtual environment 'dev' is locked +``` + +The operation does not wait or retry. This is intentional - SQLBuild does not implement lock queuing. If a lock is active, either wait for the holding operation to complete or clear the lock manually. + +## Inspecting locks + +List active locks: + +```bash +sqb state locks +``` + +## Clearing stuck locks + +If a process crashed while holding a lock, the lock may remain active until it expires. To clear it immediately: + +```bash +sqb state locks clear virtual_env:dev --force +``` + +Only clear locks when you are certain the holding operation is no longer running. Clearing a lock while the operation is still active can cause state corruption. + +## Lock expiry + +Locks are acquired with a TTL (typically 10 minutes for VDE locks). If an operation takes longer than the TTL, the lock expires and another operation can acquire it. This is a safety net against abandoned locks, not a normal operating condition. + +If operations consistently exceed the lock TTL, the TTL may need to be increased in a future configuration option. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/promotion.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/promotion.md new file mode 100644 index 0000000000..7a4dad0d6e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/promotion.md @@ -0,0 +1,72 @@ + + +# Promotion + +> Promote VDE refs and diff virtual environments. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/promotion + +Promotion copies model version refs from one VDE to another and refreshes the target's logical views. No models are rebuilt - it is a pointer swap. + +## Basic usage + +```bash +sqb promote --from pr_123 --to dev +``` + +This updates VDE `dev` to point at the same physical versions that VDE `pr_123` uses. The target's logical views (`dev__dev.*` or `dev.*` if unsuffixed) are refreshed to point at the promoted physical relations. + +## What happens during promotion + +1. Target VDE lock is acquired +2. Source VDE is validated (finalized, current with workspace) +3. Source model refs are copied to target VDE refs +4. Source function refs are copied and function definitions are republished in the target schema +5. Target logical VDE views are refreshed +6. A checkpoint is created if the target is finalized +7. Target VDE lock is released + +## Partial promotion + +Promote a subset of models with `--select`: + +```bash +sqb promote --from pr_123 --to dev --select fact_orders +``` + +Partial promotion can leave the target VDE with stale downstream models - models that still point at older versions than the promoted scope expects. When this happens, promotion is blocked by default so you don't accidentally leave the target in a working state. + +To accept a working target, pass `--allow-partial-promotion`: + +```bash +sqb promote --from pr_123 --to dev --select fact_orders --allow-partial-promotion +``` + +The target VDE is marked `working` after the promotion. You can finalize it later by promoting or building the remaining models. + +If the models you select depend on upstream models that are themselves stale in the source, the promoted scope would not be coherent on its own. Pass `--include-stale-upstreams` to expand the selection to include the minimal set of required upstream models: + +```bash +sqb promote --from pr_123 --to dev --select fact_orders --include-stale-upstreams +``` + +## Source VDE requirements + +**Whole promotion** requires the source VDE to be finalized and current with the workspace (no stale models vs current code). If the source VDE has been built but the code has since changed, you need to rebuild the source VDE first or use partial promotion. + +**Partial promotion** does not require a finalized source. A working source VDE is allowed when using `--select` for a coherent scope. + +## Guards + +| Condition | Behavior | +|-----------|----------| +| Source VDE not finalized (whole promotion) | Blocks. Use `--select` for partial promotion from a working source. | +| Source VDE has stale models vs workspace | Blocks. Rebuild the source VDE or use `--select`. | +| Target VDE is locked | Blocks. Wait or clear the lock with `sqb state locks clear`. | +| Target VDE is detached | Blocks. Detached VDEs cannot be promoted to. | +| Source VDE is detached | Blocks. Detached VDEs cannot be promoted from. | +| Partial promotion leaves target working | Blocks unless `--allow-partial-promotion` is set. | + +## Comparing VDEs before promotion + +Use `sqb diff` to compare VDE ref sets before promoting. See [Diff](diff.md) for details. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/reconcile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/reconcile.md new file mode 100644 index 0000000000..de75534193 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/reconcile.md @@ -0,0 +1,74 @@ + + +# Reconcile + +> Diagnose and repair drift between state and warehouse. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/reconcile + +Reconcile detects and repairs inconsistencies between the virtual state store and the actual warehouse objects. + +## Report + +Run reconcile without a subcommand to get a diagnostic report: + +```bash +sqb reconcile --virtual-env dev +``` + +This inspects the VDE's refs, checks that logical views exist and point to the expected physical relations, and reports any issues without changing anything. + +## Repair view + +Recreate a logical VDE view from trusted state: + +```bash +sqb reconcile repair-view --virtual-env dev --model fact_orders +``` + +This runs `CREATE OR REPLACE VIEW` for the logical VDE view, pointing it at the physical version relation recorded in the VDE's refs. It is idempotent - running it when the view is already correct is a no-op. + +### Guards + +| Condition | Behavior | +|-----------|----------| +| Logical target is a table (not a view) | Blocks. Drop the table manually first, then retry. | +| Physical relation is missing | Blocks. Rebuild the model with `sqb build --select `. | +| Target VDE is locked | Blocks. Wait or clear the lock. | + +No confirmation is needed. The command is explicit and idempotent. + +## Attach + +Rebind a VDE model ref to a different tracked physical relation: + +```bash +sqb reconcile attach --virtual-env dev --model fact_orders \ + --physical-relation dev__sqb_physical.fact_orders__v_8f3a9c12 +``` + +This updates the VDE ref for `fact_orders` to point at the specified physical relation and refreshes the logical view. + +### Guards + +| Condition | Behavior | +|-----------|----------| +| Physical relation not tracked in state | Blocks. Only physical relations registered in SQLBuild state can be attached. | +| Physical relation tracked for a different model | Blocks. Cannot attach a relation that belongs to another model. | +| Logical target is a table (not a view) | Blocks. | +| Target VDE is locked | Blocks. | +| Wrong confirmation | Blocks. Refs remain unchanged. | + +Attach requires confirmation by default. Type the confirmation text when prompted, or cancel to leave refs unchanged. + +## When to use reconcile + +- **Missing logical views** after a failed build, promotion, or detach - use `repair-view` +- **Wrong physical version** if a VDE ref was corrupted or you need to manually override which version a model points to - use `attach` +- **Diagnostic inspection** before or after recovery operations - use the default report + +Reconcile records events in the `reconcile_events` state table for audit trail. + +## Limitations + +Reconcile repairs pointer/view state. It does not rebuild physical versions. If a physical relation is missing from the warehouse, the only remedy is rebuilding the model with `sqb build --select `. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/recovery.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/recovery.md new file mode 100644 index 0000000000..7ebd25f17f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/recovery.md @@ -0,0 +1,127 @@ + + +# Recovery + +> Diagnosing and recovering from failures in virtual mode. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/recovery + +## Contents + +- State corruption +- Warehouse drift +- Lock conflicts +- Interrupted operations +- Mode guards +- Detached VDE guards +- Adopt guards +- Detach guards +- General recovery strategy + +Virtual mode has explicit recovery paths for common failure scenarios. SQLBuild blocks cleanly rather than leaving ambiguous state, and error messages point to the specific recovery action needed. + +## State corruption + +| Scenario | Error | Recovery | +|----------|-------|----------| +| Missing state table | "Cannot backup invalid state schema" on `state migrate` | `sqb state reset --auto-approve` then `sqb state init` | +| Missing state column | Same | Same | +| Wrong state column type | Same | Same | +| Deleted backup schema (explicit id) | Rollback blocks with backup-id error | Use a different backup or reset | +| All backups deleted | "No state backup is available for rollback" | `sqb state reset --auto-approve` then reinitialize | + +State corruption is detected by schema validation during `state migrate`. If the current state schema is invalid, SQLBuild refuses to back it up (to avoid persisting a broken snapshot) and directs you to reset. + +## Warehouse drift + +| Scenario | Detection | Recovery | +|----------|-----------|----------| +| Missing logical VDE view | `sqb reconcile` reports missing view | `sqb reconcile repair-view` for the affected model | +| Missing physical relation | `sqb reconcile` reports it, `repair-view` blocks | Rebuild with `sqb build --select` | +| Logical target is a table, not a view | `repair-view` and `attach` block | Drop the table manually, then `repair-view` | +| Checkpoint physical relation missing | `sqb rollback` blocks | Use a more recent checkpoint, or rebuild | +| Promoted physical relation missing | `sqb promote` blocks | Rebuild the source VDE first | + +## Lock conflicts + +| Scenario | Error | Recovery | +|----------|-------|----------| +| VDE locked by another process | "virtual environment is locked" | Wait for the other process, or clear the lock | +| Lock held by crashed process | Same | `sqb state locks clear` with `--force` | + +Only clear locks when you are certain the holding operation is no longer running. + +## Interrupted operations + +| Scenario | Behavior | Recovery | +|----------|----------|----------| +| Interrupted adopt | Failed operation recorded. Physical schema may have partial artifacts. | `sqb reconcile` to diagnose, manual repair if needed, retry. | +| Interrupted detach | Failed operation recorded. VDE stays finalized. Physical version preserved. | `sqb reconcile` to diagnose, retry detach. | +| Interrupted promotion | Failed operation recorded. Target VDE lock released. | Retry promote. | + +There is no automatic resume command for interrupted operations. SQLBuild records the failure in `state_operations` and `state_operation_events`, preserves VDE refs and checkpoints, and leaves recovery to the operator. + +To inspect a failed operation: + +```sql +SELECT operation_id, operation_type, status, virtual_environment_name +FROM sqlbuild_state.state_operations +WHERE status = 'failed'; + +SELECT operation_id, action, status, message +FROM sqlbuild_state.state_operation_events +WHERE operation_id = '' +ORDER BY created_at; +``` + +## Mode guards + +SQLBuild blocks operations that don't apply to the current mode: + +| Operation | Direct mode | Virtual mode | +|-----------|-------------|--------------| +| `sqb state` subcommands | Blocked | Allowed | +| `sqb build --no-tests --no-audits` | Allowed | Blocked (use `sqb build`) | +| `sqb build --defer-to` | Allowed | Blocked (VDE refs handle upstream resolution) | +| `sqb promote` | Blocked | Allowed | +| `sqb rollback` | Blocked | Allowed | +| `sqb reconcile` | Blocked | Allowed | + +## Detached VDE guards + +After `sqb state detach`, the VDE is marked `detached` and blocked from further virtual operations: + +| Operation | Behavior | +|-----------|----------| +| `sqb build` | Blocks: "virtual environment is detached" | +| `sqb promote --from ` | Blocks | +| `sqb promote --to ` | Blocks | +| `sqb rollback` | Blocks | + +The project can continue in direct mode or re-adopt to return to virtual mode. + +## Adopt guards + +| Condition | Error | +|-----------|-------| +| No `unsuffixed_virtual_env` configured | Blocks with config guidance | +| Copy fallback needed without `--allow-copy` | `requires --allow-copy` | +| Wrong typed confirmation | Cancelled, no changes made | + +## Detach guards + +| Condition | Error | +|-----------|-------| +| VDE not finalized | "requires a finalized virtual environment" | +| Copy fallback needed without `--allow-copy` | `requires --allow-copy` | +| Wrong typed confirmation | Cancelled, no changes made | + +## General recovery strategy + +1. **Diagnose** with `sqb reconcile` or by querying state tables directly +2. **Repair views** with `sqb reconcile repair-view` for missing/broken logical views +3. **Rebuild** with `sqb build --select ` for missing physical versions +4. **Roll back** with `sqb rollback` to restore a prior finalized state +5. **Reset state** with `sqb state reset --auto-approve` as a last resort (drops all virtual state) + +When in doubt, `sqb reconcile` is the starting point. It reports what's wrong without changing anything. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/rollback.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/rollback.md new file mode 100644 index 0000000000..6a68b17923 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/rollback.md @@ -0,0 +1,119 @@ + + +# Rollback + +> Checkpoints and rollback for virtual environments. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/rollback + +## Contents + +- Checkpoints +- Basic rollback +- Explicit checkpoint +- What happens during rollback +- Partial rollback +- Guards +- Checkpoint inspection + +Rollback restores a VDE to a prior finalized state by rebinding its refs to a stored checkpoint. + +## Checkpoints + +Checkpoints are created automatically when a VDE reaches `finalized` status: + +- After a whole build where all models match expected versions +- After a whole promotion where the target VDE is finalized + +Each checkpoint stores the complete set of model refs and function refs for that VDE at that point in time. Checkpoints are retained according to `[janitor] max_checkpoints` (default: 20). + +## Basic rollback + +Restore the previous finalized checkpoint for the default VDE: + +```bash +sqb rollback +``` + +For an explicit VDE: + +```bash +sqb rollback --virtual-env pr_123 +``` + +## Explicit checkpoint + +Restore a specific checkpoint: + +```bash +sqb rollback --checkpoint-id +``` + +Use `sqb state checkpoints list` to see available checkpoints. + +## What happens during rollback + +1. Target VDE lock is acquired +2. The target checkpoint is located (previous finalized by default, or explicit id) +3. Checkpoint physical relations are validated to still exist in the warehouse +4. VDE refs are replaced with the checkpoint's ref set +5. VDE function refs are replaced with the checkpoint's function ref set +6. Logical VDE views are refreshed to point at the restored physical versions +7. Function definitions are republished from the checkpoint's function versions +8. Target VDE lock is released + +## Partial rollback + +Roll back a subset of models with `--select`: + +```bash +sqb rollback --select fact_orders +``` + +Rolling back some models but not others can leave the VDE with stale models - models whose restored versions no longer line up with the rest of the VDE. When this happens, rollback is blocked by default so you don't leave the VDE in a working state by accident. + +To accept a working VDE, pass `--allow-partial-rollback`: + +```bash +sqb rollback --select fact_orders --allow-partial-rollback +``` + +The VDE is marked `working` after the rollback. You can finalize it later by building or rolling back the remaining models. + +If the models you select depend on upstream models that also need to be restored for the scope to be coherent, pass `--include-stale-upstreams` to expand the selection to the minimal set of required upstream refs from the checkpoint: + +```bash +sqb rollback --select fact_orders --include-stale-upstreams +``` + +## Guards + +| Condition | Behavior | +|-----------|----------| +| No previous checkpoint exists | Blocks. Build the VDE first to create a finalized checkpoint. | +| Checkpoint physical relations deleted | Blocks. The physical version tables have been cleaned up by janitor. Use a more recent checkpoint or rebuild. | +| Unknown checkpoint id | Blocks with error. | +| Target VDE is locked | Blocks. Wait or clear the lock. | +| Target VDE is detached | Blocks. Detached VDEs cannot be rolled back. | +| Partial rollback leaves VDE working | Blocks unless `--allow-partial-rollback` is set. | + +## Checkpoint inspection + +List checkpoints for a VDE: + +```bash +sqb state checkpoints list +sqb state checkpoints list --virtual-env pr_123 +``` + +Show a checkpoint's model refs: + +```bash +sqb state checkpoints show +``` + +Diff current VDE refs against a checkpoint: + +```bash +sqb state checkpoints diff +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/setup.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/setup.md new file mode 100644 index 0000000000..414cdcedd8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/virtual-environments/setup.md @@ -0,0 +1,189 @@ + + +# Setup + +> Configure virtual environments and initialize the state store. + +Online: https://docs.sqlbuild.com/concepts/virtual-environments/setup + +## Contents + +- Project configuration +- State configuration +- State initialization +- State lifecycle +- Local overrides +- Per-target state +- Unsuffixed VDE naming +- Janitor configuration +- State tables + +Virtual environments require two things: `virtual_environments = true` in your project settings, and a state store configured for your active target. + +## Project configuration + +```toml +name = "my_project" +adapter = "duckdb" +default_target = "dev" + +[settings] +virtual_environments = true + +[connections.local] +database = "warehouse.duckdb" + +[targets.dev] +connection = "local" +schema = "dev" + +[targets.dev.state] +backend = "duckdb" +schema = "sqlbuild_state" + +[targets.dev.state.connection] +database = "state.duckdb" +``` + +The `virtual_environments` setting switches the project from direct mode (default) to virtual mode. All state, plan, build, promote, rollback, and reconcile commands route through the virtual path when this is enabled. + +## State configuration + +Each physical target that uses virtual mode needs a `[targets..state]` block: + +| Field | Required | Description | +|-------|----------|-------------| +| `backend` | Yes | State store engine: `duckdb` or `postgres` | +| `schema` | Yes | Schema name for state tables | +| `connection` | Yes | Backend-specific connection config | +| `allow_reset` | No | Whether `sqb state reset` is permitted (default: `false`) | +| `unsuffixed_virtual_env` | No | VDE name that uses the base schema without a suffix (for [adopt/detach](adopt-detach.md)) | + +### DuckDB state backend + +```toml +[targets.dev.state] +backend = "duckdb" +schema = "sqlbuild_state" + +[targets.dev.state.connection] +database = "state.duckdb" +``` + +DuckDB state stores are file-based. Relative paths are resolved against the project directory. Suitable for local development and single-user workflows. Not recommended when multiple processes or CI jobs need concurrent state access. + +### Postgres state backend + +```toml +[targets.prod.state] +backend = "postgres" +schema = "sqlbuild_state" + +[targets.prod.state.connection] +host = "state-db.internal" +port = 5432 +user = "sqlbuild_state" +password = "${ENV:STATE_DB_PASSWORD}" +dbname = "sqlbuild_state" +``` + +Postgres is recommended for production. It supports concurrent access from multiple developers or CI jobs. + +## State initialization + +Before using virtual commands, initialize the state store: + +```bash +sqb state init +``` + +This creates the state tables in the configured schema. Run it once per physical target. + +## State lifecycle + +| Command | Description | +|---------|-------------| +| `sqb state init` | Create state tables | +| `sqb state migrate` | Back up current state and re-initialize tables | +| `sqb state rollback` | Restore from the latest backup | +| `sqb state rollback --backup-id ` | Restore from a specific backup | +| `sqb state reset --auto-approve` | Drop all state tables (requires `allow_reset = true`) | + +`state migrate` creates a backup schema (e.g. `sqlbuild_state__backup_`) before re-initializing. This lets you roll back if a migration causes problems. + +`state reset` is destructive and requires both `allow_reset = true` in config and `--auto-approve` on the command line. + +## Local overrides + +Use `sqlbuild_local.toml` to override state connection config per developer: + +```toml +[targets.dev.state.connection] +database = "local-state.duckdb" +``` + +This is useful when each developer uses a local state file while the project config points to a shared state database. + +## Per-target state + +Different physical targets can use different state backends: + +```toml +[targets.dev.state] +backend = "duckdb" +schema = "sqlbuild_state" + +[targets.dev.state.connection] +database = "state.duckdb" + +[targets.prod.state] +backend = "postgres" +schema = "sqlbuild_state" + +[targets.prod.state.connection] +host = "prod-state.internal" +dbname = "sqlbuild_state" +``` + +Each physical target has its own state store. VDEs are scoped to their physical target and cannot be promoted or compared across different physical targets. + +## Unsuffixed VDE naming + +By default, all VDE logical views use a suffixed schema: `dev__kevin.fact_orders`. For the primary VDE that consumers query directly (e.g. the production VDE), you typically want clean unsuffixed names: `dev.fact_orders`. + +Configure this with `unsuffixed_virtual_env`: + +```toml +[targets.dev.state] +backend = "duckdb" +schema = "sqlbuild_state" +unsuffixed_virtual_env = "dev" + +[targets.dev.state.connection] +database = "state.duckdb" +``` + +With this config, VDE `dev` uses `dev.fact_orders` while other VDEs like `kevin` use `dev__kevin.fact_orders`. + +This setting is also required for [adopt and detach](adopt-detach.md) operations so that existing consumer queries continue to work after migrating to virtual mode. + +## Janitor configuration + +When using virtual environments, configure the janitor to run periodically to clean up expired VDEs, old checkpoints, and unreferenced physical versions: + +```toml +[janitor] +enabled = true +retention_days = 30 +max_checkpoints = 20 +``` + +See [Janitor](janitor.md) for details on what gets cleaned up and how retention works. + +## State tables + +The state store contains current-state tables and append-only history tables. You do not need to interact with these directly, but understanding what is stored helps when debugging: + +**Current state:** `state_versions`, `model_versions`, `function_versions`, `physical_relations`, `physical_relation_ancestry`, `virtual_environments`, `virtual_environment_refs`, `virtual_environment_function_refs`, `virtual_environment_checkpoints`, `virtual_environment_checkpoint_refs`, `virtual_environment_checkpoint_function_refs`, `locks`, `state_operations` + +**History:** `plan_runs`, `virtual_environment_ref_events`, `reconcile_events`, `state_migration_events`, `state_operation_events` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md new file mode 100644 index 0000000000..6364977ef6 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md @@ -0,0 +1,140 @@ + + +# Feature Comparison + +> Feature comparison between SQLBuild, dbt, and SQLMesh. + +Online: https://docs.sqlbuild.com/feature-comparison + +## Contents + +- Feature comparison +- Where each tool fits +- Not yet in SQLBuild + +SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common ground but differ in design philosophy and feature focus. + +## Feature comparison + +### Testing + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Multi-model tests | Chain across multiple models | YAML-stub, single model | CTE-based, single model | +| Typed parameterized unit tests | Independent named cases with adapter-rendered scalar values | No | No | +| Macros as test helpers | Tests are SQL - macros work as reusable fixture generators | No (YAML stubs) | No | +| E2E scenario tests | Fixture worlds with real graph execution | No | No | +| Local E2E replay | Capture from warehouse, replay in DuckDB | No | No | +| Macro / UDF / table function tests | `TEST(mode macro)`, `TEST(mode udf)`, or `TEST(mode table_fn)` | No | No | +| Zero-row assertions | `__assert__` CTEs in tests and scenarios | No | No | +| Failure diagnostics | Bounded, redacted unexpected and missing row samples in text and JSON | Adapter/tool dependent | Row diffs | + +### Audits + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Built-in audits | not_null, unique, accepted_values, relationships | not_null, unique, accepted_values, relationships | Extensive (statistical, string pattern, etc.) | +| Blocking audits | Block promotion from staging table | Tests run after materialization | Audits gate plan application; run-time audits execute after the interval is materialized | +| Delta/interval-scoped audits | Per-microbatch audit cycle before DML | No | Audit query filtered to processed intervals for time-range models | +| Measurement audits | Thresholds, minimum samples, bounded evidence, and immutable result history | Package/custom test patterns | Custom audits | +| Audit factories | Typed Python factories generate equivalent reviewed audit instances | Macros/packages | Python audit definitions | + +### Compilation + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| SQL analysis | Offline syntax, binding, type inference, semantic validation, and lineage (Polyglot) | dbt Core: none; dbt Fusion engine: compile-time (proprietary; built on Apache-2.0 dbt Core v2) | Compile-time (SQLGlot) | +| Focused compilation | Full graph integrity with deep analysis limited to selection and required upstream closure | Selected compilation | Selected planning | +| Column-level lineage | Compile-time, fast and rich modes | dbt Core: post-hoc via docs; dbt Fusion engine: compile-time | Compile-time | +| Column contract validation | Compile-time inference plus runtime enforcement with `contract enforced` | YAML schema contracts at runtime | Schema contracts via plan | +| Existing-schema contracts | Online read-only diff and safe repository generation | Codegen/packages | External schema tooling | +| Rules and formatting | Compiler-integrated native and custom diagnostics, reasoned suppressions, and separate canonical formatting | External tools | Built-in formatter plus external linting | +| Compiler-integrated Rules | Native and custom rules over compiler-owned SQL, models, dependencies, contracts, tests, and project paths | Project conventions through packages and external tooling | Built-in audits and external linting | +| SQL transpilation | For local E2E replay into DuckDB | No | For cross-dialect model execution | +| Python macros | `@macro()` syntax | No (Jinja only) | SQLMesh macro syntax | +| Compiler-enforced declaration scopes | Project, descendant-public, exact-owner-private, and model-private tiers with offline `sqb scope` inspection | No lexical declaration scopes | No lexical declaration scopes | +| Jinja support | No (Python macros instead) | Yes (core templating) | Yes | + +### Incremental + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Incremental strategies | append, delete_insert, merge, SCD Type 2 | append, delete_insert, merge, snapshots | delete_insert (time-range), merge (unique-key), SCD Type 2, partition | +| Microbatch execution | Watermark and rolling-window strategies, per-batch audits, limits, and opt-in concurrency | Microbatch | Batch size support | +| Interval progress | Sequential runs derive progress from target/input cursors; concurrent runs coordinate with append-only facts | No | Tracks intervals in a state store | +| SCD Type 2 models | Timestamp and check strategies, historical input, hard deletes | Snapshots (timestamp and check strategies) | `SCD_TYPE_2` model kind (timestamp and check strategies) | + +### Planning and change detection + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Stale-driven virtual builds | Optional `--changes-only` execution over VDE version bindings | dbt State (paid) | Version hash comparison | +| Warehouse-native state | Append-only tables in the warehouse; no external state database | manifest.json artifacts | Requires external state store (SQLite/PostgreSQL) | +| Source freshness | `sqb freshness` with adapter/column/sql strategies, lag tolerance, and CI gating | `dbt source freshness` | No dedicated freshness command; `signals` gate model evaluation until external data is ready | +| Reuse across environments | Virtual environments reuse fingerprint-matched physical tables across environments (shared physical storage) | dbt State clone (paid) | Virtual environments reuse fingerprint-matched physical tables across environments (shared physical storage) | +| Cascade propagation | Topological walk with `replay_on_change` policy inheritance and override | No cascade control | Cascades through version hashes | + +### Environments + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Virtual environments | Pointer swaps with hash-based version reuse (opt-in) | No | Pointer swaps, no compute cost | +| Data diffs | Full row-level data comparison across targets or virtual environments | No | Table diff | +| Zero-copy cloning | `sqb clone` | No | No | + +### Models + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| SQL models | `MODEL()` header with inline config | Jinja-templated SQL + YAML sidecar | `MODEL` DDL | +| Python models | Coming soon | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | +| Custom materializations | Python with full framework hooks | Jinja-based | Python-based custom model kinds | +| Lifecycle hooks | Typed inline SQL, reusable parameterized SQL resources, and Python hooks with compile-time validation and `HookContext` | Jinja pre/post hooks | Python pre/post hooks | + +### Python nodes + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Tasks | `@task` - Python computation as DAG nodes | No | No | +| Assets | `@asset` - external artifact production/observation | No | No | +| Checks | `@check` - Python validation of tasks, assets, and loaders | No | No | +| Factories | `@factory` - programmatic node generation | No | No | +| Providers | Shared runtime services with name-based injection into nodes and hooks | No | No | + +### dbt interoperability + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| dbt compatibility | Reads dbt manifests, coordinates dbt and SQLBuild selection, and supports SQLBuild models downstream | N/A | Jinja compatibility layer plus own macro system | + +### Sources + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Source loaders | Python `@loader` functions with table/append/delete_insert/merge strategies | No (external to dbt) | No (external to SQLMesh) | +| Declarative ingestion | dlt and ingestr integrations - YAML-only source config, no Python | No | No | +| Auto-load during builds | Managed sources loaded before dependent models | No | No | +| Source deferral | `--defer-sources-to` reads source data from another target | No | No | + +### Other + +| Feature | SQLBuild | dbt | SQLMesh | +|---------|----------|-----|---------| +| Reference syntax | `__ref()` - parses as valid SQL | `{{ ref() }}` - Jinja template | `model_name` with dependency tracking | +| Adapters | DuckDB, MotherDuck, Snowflake, BigQuery, Databricks, PostgreSQL, SQL Server | 30+ (community adapters) | DuckDB, Snowflake, BigQuery, Databricks, Spark, Redshift, Postgres, Trino, MySQL | +| State requirements | Stateless by default | manifest.json + target/ | Requires state store (local database or PostgreSQL for production) | +| Playground | `sqb playground` | Clone example repo | Example project | +| AI agent skills | General guidance with `sqb skills`; Rules guidance with `sqb rules skills` | No | No | +| Execution observability | Immutable schema-versioned lifecycle facts, invocation-local ordering, orchestration context, and typed project sinks | Events/artifacts plus external observability | Plans, state, and external observability | + +## Where each tool fits + +| Tool | Best for | +|------|----------| +| **SQLBuild** | Rigor-first SQL pipelines: compile-time verification, pre-promotion audit gating, multi-model tests, and local E2E replay by default. Opt into change-aware builds and warehouse-native state when full rebuilds get expensive, plus ingestion, Python nodes, and virtual environments as the project grows. | +| **dbt** | The most widely adopted SQL transformation framework with the largest adapter and community ecosystem. | +| **SQLMesh** | State-managed pipelines with virtual environments, interval tracking, and cross-dialect transpilation. | + +## Not yet in SQLBuild + +- **Broader adapter support** - ClickHouse, Redshift, Trino, Spark, Athena diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md new file mode 100644 index 0000000000..250334cbd4 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md @@ -0,0 +1,254 @@ + + +# Introduction + +> Verify early, test properly, and deploy reversibly. SQL pipelines with the rigor of real software - free and open source. + +Online: https://docs.sqlbuild.com/index + +## Contents + +- Test properly +- Verify early +- Plans that explain every build +- Deploy reversibly (opt-in) +- Supported adapters +- More in SQLBuild +- What's next +- Quick links + +**Valid isn't the same as correct.** Your SQL compiles, runs, and returns rows - none of that means the number is right, and a silently-wrong number a stakeholder already trusted is the bug that actually hurts. + +SQLBuild brings software-engineering rigor to SQL pipelines: **verify early, test properly, deploy reversibly.** Catch errors before the warehouse runs them and test your logic locally - with change-aware builds and reversible deploys there when you need them, not forced on you on day one. It is a standalone framework for building SQL and Python data pipelines - free and open source. + +## Test properly + +Most data testing checks for nulls and uniqueness. That tells you a column isn't empty, not that your logic is right. SQLBuild tests the logic. + +- **Multi-model tests.** Mock your sources, assert on the model you care about, and SQLBuild resolves every intermediate model from its real SQL. One test file can cover a whole stretch of your pipeline. +- **E2E scenario tests with local replay.** Build the real model graph against coherent fixtures in isolated relations, capture the result as JSONL, and replay it locally through DuckDB. CI runs full end-to-end tests with zero warehouse credentials or compute. +- **Macro-powered mocks.** Tests are SQL, so they support macro calls - write reusable mock generators instead of copy-pasting fixtures. + +```sql +-- Mock two sources, assert on the final mart. +-- stg_orders and stg_payments resolve automatically from their real SQL. +TEST(); + +WITH +__source__raw__orders AS ( + @mock_orders() +), +__source__raw__payments AS ( + SELECT + 1 AS payment_id, + 1 AS order_id, + 1500 AS amount_cents, + 'credit_card' AS method +), +__expected__fact_orders AS ( + SELECT + 1 AS order_id, + 100 AS customer_id, + 1500 AS total_cents, + 'credit_card' AS payment_method +) +SELECT 1 +``` + +See [Testing](concepts/testing.md) for SQL unit tests and [Scenarios](concepts/scenarios.md) for local +E2E replay. + +## Verify early + +Before any model runs, SQLBuild does static analysis of your project - offline, no warehouse connection needed. + +- **Catch errors at compile.** SQL syntax, type inference, contract checks, and column lineage all run before execution. A bad reference or a type mismatch fails at compile, with an error that points at the line - not halfway through a warehouse run. +- **Turn review decisions into compiler checks.** Configure [Rules](concepts/rules.md) over compiler-owned SQL, models, dependencies, contracts, tests, and project paths. +- **Format separately.** `sqb format` owns deterministic source rewriting; Rules remain diagnostic-only. +- **Fast, because it's Rust where it counts.** Static analysis runs on [Polyglot](https://github.com/tobilg/polyglot), a Rust SQL engine (MIT, 32+ dialects), so compile stays quick even on large projects. +- **Open, not paywalled.** The static analysis is part of the Apache-2.0 core - no proprietary engine, no separate login, and no paid tier gating the smart checks. + +## Plans that explain every build + +**Start simple.** By default, `sqb build` runs your full selection - the same predictable mental model as dbt, with nothing to configure. For many projects, that is all they will ever need. + +Every model, seed, function, and Python node has a versioned identity. SQLBuild uses those identities +to explain query and configuration changes, propagate upstream changes, choose incremental replay +actions, and protect scoped builds from stale dependencies. Direct mode still runs the selected +scope; change detection makes that plan explicit rather than silently changing what was selected. + +[Virtual environments](concepts/virtual-environments.md) add version reuse and optional stale-driven +execution when a project needs isolated previews, promotion, and rollback. + +The planner carries that evidence across the whole graph: + +- **Models and functions:** query, configuration, and dependency identities explain local and cascaded changes. +- **Seeds:** content and load-affecting configuration identify when a seed changed. +- **Audits:** audit evidence is associated with the model version it validated. +- **Source freshness:** observable source data versions can affect downstream plans, with lag tolerance to avoid jitter. +- **Cascade propagation:** when a model does change, the signal propagates downstream, with configurable replay windows (`replay_on_change`). +- **Python nodes:** source and dependency identities appear in plans; skip/run remains user-controlled through `ctx.skip()`. + +In direct mode, this evidence is stored as append-only rows in your warehouse. Virtual environments +use their configured state backend for environment-scoped version bindings. + +## Deploy reversibly (opt-in) + +By default, SQLBuild runs in direct mode with state as append-only rows in your warehouse. When you want more, [virtual environments](concepts/virtual-environments.md) add a reversibility layer on top: + +- **Instant branching, promotion, and rollback** as low-copy pointer operations. +- **Partial promotion.** Promote the models that are ready without re-running everything downstream of them - you don't have to rebuild the whole closure to ship one fix. +- **Checkpoints and reconciliation** so a bad change is something you undo, not an incident. + +Virtual environments are opt-in, not a tax you pay upfront - direct mode stays the default, so the floor stays low and you reach for them only when a workflow needs them. + +## Supported adapters + +SQLBuild works with **DuckDB, MotherDuck, Snowflake, BigQuery, Databricks, PostgreSQL, and SQL Server** today. Support for ClickHouse, Redshift, Trino, Spark, and Athena is on the way. + +DuckDB runs entirely locally, so you can try SQLBuild and run full E2E tests without any warehouse credentials. Head to the [Quickstart](quickstart.md) to get a project running in minutes, or see [Adapters](concepts/adapters.md) for connection setup. + +## More in SQLBuild + +### Familiar SQL models + +Models are SQL files with a `MODEL()` header and a `SELECT`. References to other models and sources use `__ref()` and `__source()`, and configuration, schema, and audits are declared inline in the header. If you know dbt or SQLMesh, you already know the shape. + +```sql +-- models/marts/fact_orders.sql +MODEL ( + materialized table, + columns ( + order_id (audits [not_null, unique]), + ), +); + +SELECT + o.order_id, + o.customer_id, + p.amount_cents AS total_cents, + p.method AS payment_method +FROM __ref("stg_orders") o +JOIN __ref("stg_payments") p USING (order_id) +``` + +### Python macros, not Jinja + +- **Real Python functions:** Macros are plain Python, called with `@macro()` in SQL. Testable, debuggable, and composable with standard tooling - no Jinja, no string-templating language to wrangle. + +```python +# macros/grant_target.py +def grant_target(target): + return f"GRANT SELECT ON {target} TO analyst_role" +``` + +### Audits that block bad data + +- **Full table builds:** SQLBuild materializes into a staging table and runs `error`-severity audits before promotion. If any fail, the swap is blocked and the production table is untouched. +- **Incremental models:** Delta-phase audits validate each batch before DML is applied. Bad data is caught before it reaches the target. +- **Measured quality:** Measurement audits evaluate values against warning/error thresholds, distinguish insufficient samples, and can retain bounded evidence and immutable result history. + +### Incremental processing + +- **Cursor-based replay:** SQLBuild resumes by reading the highest timestamp or integer value already in the target table. If a model fails for several runs, the next successful build picks up from the last data it actually wrote, with no manual backfilling. +- **Microbatch mode:** Split large replay windows into configurable batches, each with its own audit cycle. Or process the full range in one pass, the choice is per-model. +- **Replay on change:** When a model's version identity changes, `replay_on_change` controls how much data to reprocess: `forward` (default, just run the next delta), `bounded-14d` (replay a window), or `full` (rebuild the table). + +### Python you can read and extend + +- **The framework is Python.** Adapters, macros, hooks, providers, custom materializations, and Python nodes are all plain Python you can read and extend. (The SQL analysis underneath runs on Rust via Polyglot - see [Verify early](#verify-early) - so the code you work with stays Python while compile stays fast.) + +### Testing in depth + +Beyond the multi-model tests shown above, SQLBuild runs end-to-end scenario tests against the real model graph, with local replay and property assertions. + +- **Real graph execution:** Define coherent fixture inputs, and SQLBuild builds the real model graph against them in isolated warehouse relations. Test end-to-end business logic across your entire pipeline, not just one model at a time. +- **Local replay without a warehouse:** Capture scenario fixtures as JSONL snapshots, then replay them locally through DuckDB. CI pipelines run full E2E tests with zero warehouse credentials or compute cost. +- **Zero-row assertions:** Write property checks that pass when no rows violate a condition. Useful for invariants like "no negative revenue" or "no duplicate customer IDs" alongside full expected-output comparisons. + +```sql +SCENARIO ( + description "Daily revenue includes only successful payments", + tags ["revenue"] +); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 10 AS customer_id, 'completed' AS status +), +__ref__stg_payments AS ( + SELECT + 1 AS payment_id, + 1 AS order_id, + 1700 AS amount_cents, + 'success' AS payment_status +), +__expected__daily_revenue AS ( + SELECT + CAST('2026-04-01' AS DATE) AS revenue_date, + 1700 AS total_revenue_cents +), +__assert__all_orders_have_payments AS ( + SELECT * FROM __ref("fact_orders") WHERE payment_amount_cents IS NULL +) +SELECT 1 +``` + +### Python nodes + +Grow beyond warehouse-only SQL without leaving the graph. Python nodes are ordinary functions, decorated to become first-class nodes in the same DAG as your SQL models, and they run as part of `sqb build`. There are four kinds: + +- **Loaders** (`@loader`) load external data into managed sources, with incremental write strategies (table, append, delete_insert, merge), cursor-based loading, and concurrent execution. +- **Tasks** (`@task`) run Python computation or side effects as graph nodes. +- **Assets** (`@asset`) produce or observe external artifacts, with optional columns and lineage. +- **Checks** (`@check`) validate tasks, assets, and loaders, and run during `sqb build` or on their own with `sqb check`. + +```python +from sqlbuild.loaders import loader +from sqlbuild.executor.load.models import LoaderContext + +@loader +def raw_orders(ctx: LoaderContext): + if ctx.current_cursor_value is None: + return fetch_all_orders() + return fetch_orders_since(ctx.current_cursor_value) +``` + +SQL models never depend on Python nodes - the only path from Python into SQL is a loader populating a source - so the SQL graph stays fully analyzable on its own. Nodes can also be generated programmatically with `@factory`. See [Python Nodes](concepts/python-nodes/overview.md). + +### Multi-target workflows + +- **Data diffs:** Compare schemas and row-level data between targets (or virtual environments) with `sqb diff prod:dev`. + +- **Zero-copy cloning:** Branch targets instantly with `sqb clone` without duplicating data. +- **Deferred references:** Resolve a production namespace with `--defer-to` through the active target's connection while building in dev. +- **No manifest required:** Clone, diff, and defer work directly against live targets. No `manifest.json` generation, no artifact management, no stale state. + +### Extensibility + +- **User-defined functions:** SQL and Python UDFs managed as part of your project. Functions participate in the DAG - definition changes trigger rebuilds of dependent models. Table functions provide predicate-pushdown-friendly alternatives to final-layer views. +- **Custom materializations:** Write materialization logic in Python with full framework integration - including audit hooks, schema change signals, and query change detection. + +```python +def materialize(ctx: MaterializationContext) -> MaterializationResult: + stale = find_untracked_partitions(ctx) + + for partition in stale: + ctx.adapter.create_table_as(ctx.connection, target=staging, sql=partition_sql) + ctx.run_audits(staging) # same audit guarantees as built-in types + ctx.execute_sql(f"INSERT INTO {ctx.destination} SELECT * FROM {staging}") + + return MaterializationResult(relation=ctx.destination) +``` +- **Path-between selectors:** `--select fact_orders~daily_activity_rollup` selects every model on the shortest path between two nodes, with optional upstream/downstream expansion. + +## What's next + +- **Broader adapter support** - ClickHouse, Redshift, Trino, Spark, Athena + +## Quick links + + Get a project running locally in minutes. + Reference for SQLBuild commands and flags. + Understand models, incremental strategies, audits, and selectors. + E2E tests with local DuckDB replay. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md new file mode 100644 index 0000000000..68c5e8c130 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md @@ -0,0 +1,261 @@ + + +# API Reference + +> Dagster integration classes, decorators, and translator hooks. + +Online: https://docs.sqlbuild.com/integrations/dagster-reference + +## Contents + +- SqlBuildProject +- SqlBuildCliResource +- SqlBuildCliInvocation +- sqlbuild_assets +- sqlbuild_scenario_checks +- SqlBuildDagsterTranslator + +## SqlBuildProject + +Project metadata and DAG artifact preparation. + +```python +from sqlbuild.integrations.dagster import SqlBuildProject + +project = SqlBuildProject( + project_dir=".", + target_path="target", + dag_filename="sqlbuild_dag.json", + sqb_command=("sqb",), + prepare_project_cli_args=("compile", "--dag"), +) +``` + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `project_dir` | (required) | Path to the SQLBuild project root | +| `target_path` | `"target"` | Relative path to the target directory | +| `dag_filename` | `"sqlbuild_dag.json"` | DAG artifact filename | +| `sqb_command` | `("sqb",)` | Command to invoke SQLBuild CLI | +| `prepare_project_cli_args` | `("compile", "--dag")` | CLI args used by `prepare()` to generate the DAG | + +### Methods + +| Method | Description | +|--------|-------------| +| `dag_path` | Property. Returns the full path to the DAG artifact (`project_dir / target_path / dag_filename`). | +| `prepare()` | Runs `sqb compile --dag ` to generate the DAG artifact. Raises `DagsterProjectPrepareError` on failure. | +| `prepare_if_dev()` | Calls `prepare()` only when the `DAGSTER_IS_DEV_CLI` environment variable is set. | + +## SqlBuildCliResource + +Dagster `ConfigurableResource` that shells out to the SQLBuild CLI. + +```python +from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject + +# From a project +resource = SqlBuildCliResource(SqlBuildProject(project_dir=".")) + +# Or with explicit paths +resource = SqlBuildCliResource( + project_dir=".", + sqb_command=["sqb"], + dag_path="target/sqlbuild_dag.json", +) +``` + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `project_dir` | `"."` | Path to the SQLBuild project root, or a `SqlBuildProject` instance | +| `sqb_command` | `["sqb"]` | Command to invoke SQLBuild CLI | +| `dag_path` | `None` | Path to the DAG artifact. When set, enables asset selection bridging and structured event emission. | + +### cli() + +Create a CLI invocation for the provided command arguments. + +```python +invocation = resource.cli( + ["build"], + context=context, # Dagster execution context (optional) + raise_on_error=True, # raise Failure on non-zero exit (default) +) +``` + +Returns a `SqlBuildCliInvocation`. + +## SqlBuildCliInvocation + +A running or completed SQLBuild CLI subprocess. + +### Methods + +| Method | Description | +|--------|-------------| +| `wait()` | Wait for the process to complete. Streams stdout/stderr in real time. Returns `self`. | +| `stream()` | Wait for the process, then yield Dagster `MaterializeResult` and `AssetCheckResult` events parsed from execution JSON. | +| `is_successful()` | Returns `True` if exit code is 0. | +| `get_error()` | Returns a Dagster `Failure` if the process failed, `None` otherwise. | +| `get_artifact(name)` | Read a JSON artifact from the project's `target/` directory. | + +### Properties + +| Property | Description | +|----------|-------------| +| `stdout` | Captured stdout after `wait()` or `stream()`. | +| `stderr` | Captured stderr after `wait()` or `stream()`. | +| `returncode` | Process exit code after completion. | +| `execution_payload` | Parsed execution JSON, if available. | + +### stream() behavior + +`stream()` is the primary way to emit Dagster events from a SQLBuild execution: + +1. Waits for the subprocess to complete while streaming output to stdout/stderr +2. Reads the structured execution JSON (from `--json-output` tempfile) +3. Maps each completed asset to a `MaterializeResult` with metadata (status, duration, row counts) +4. Maps each check result to an `AssetCheckResult` with pass/fail, severity, and check metadata +5. If the DAG artifact is available, results are matched to the correct Dagster asset keys +6. Raises `Failure` if the process exited non-zero and `raise_on_error` is set + +## sqlbuild_assets + +Decorator that creates a Dagster multi-asset definition from a SQLBuild DAG artifact. + +```python +from sqlbuild.integrations.dagster import sqlbuild_assets + +@sqlbuild_assets( + project=project, # or dag=path_or_dict + translator=MyTranslator(), # optional + name="my_sqlbuild_assets", # optional + include_scenario_checks=True, # default: True + required_resource_keys={"sqb"}, # optional +) +def my_assets(context, sqb): + yield from sqb.cli(["build"], context=context).stream() +``` + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `project` | `None` | `SqlBuildProject` instance. Mutually exclusive with `dag`. | +| `dag` | `None` | DAG artifact as a path, string, or pre-loaded dict. Mutually exclusive with `project`. | +| `translator` | `SqlBuildDagsterTranslator()` | Translator for asset keys, groups, tags, and metadata. | +| `name` | `None` | Override the Dagster multi-asset name. | +| `include_scenario_checks` | `True` | Whether to include scenario checks alongside test and audit checks. | +| `required_resource_keys` | `None` | Additional Dagster resource keys to require. | + +If neither `project` nor `dag` is provided, the decorator looks for `target/sqlbuild_dag.json` in the current directory. + +## sqlbuild_scenario_checks + +Decorator that creates a Dagster multi-asset-check definition for SQLBuild scenarios only. Useful when you want to run scenarios separately from the main build. + +```python +from sqlbuild.integrations.dagster import sqlbuild_scenario_checks + +@sqlbuild_scenario_checks(project=project) +def my_scenario_checks(context, sqb): + yield from sqb.cli(["scenario", "test"], context=context).stream() +``` + +Parameters are the same as `sqlbuild_assets` except there is no `include_scenario_checks` flag. + +## SqlBuildDagsterTranslator + +Customise how SQLBuild DAG nodes map to Dagster asset metadata. Subclass and override any method. + +### Methods + +| Method | Arguments | Returns | Default behavior | +|--------|-----------|---------|------------------| +| `get_asset_key` | `node` | `AssetKey` | Authored source `meta.dagster.asset_key`, otherwise `AssetKey(node["asset_key"])` | +| `is_asset_node` | `node` | `bool` | Include the node as a Dagster asset | +| `is_asset_check` | `check` | `bool` | Include the check in the multi-asset definition | +| `get_group_name` | `node` | `str \| None` | Authored group, project name, then `sqlbuild` | +| `get_tags` | `node` | `dict[str, str]` | `sqlbuild/kind` tag plus any model tags | +| `get_metadata` | `node` | `dict[str, Any]` | SQLBuild identity/selector plus available path, target, SQL, columns, lineage, group, language, materialization, loader, and authored metadata | +| `get_description` | `node` | `str \| None` | Node description if present | +| `get_check_name` | `check` | `str` | `"{kind}__{name}"` with optional column/target suffix | +| `get_check_metadata` | `check` | `dict[str, Any]` | Check ID, kind, name, and selector | + +### Node structure + +Each `node` dict passed to translator methods contains: + +```json +{ + "id": "model:fact_orders", + "kind": "model", + "name": "fact_orders", + "asset_key": ["dev", "fact_orders"], + "target": { + "database": null, + "schema": "dev", + "name": "fact_orders", + "qualified_name": "dev.fact_orders" + }, + "path": "models/marts/fact_orders.sql", + "description": "Order fact table with waffle and payment details.", + "tags": ["marts"], + "columns": [ + {"name": "order_id", "type": "INTEGER"} + ], + "materialization_type": "table" +} +``` + +Python node asset keys use a two-part key with the node kind as prefix: + +| Kind | Asset key example | +|------|-------------------| +| Task | `("task", "prepare_orders")` | +| Asset | `("asset", "orders_export")` | +| Loader | `("loader", "raw_orders")` | + +A source can provide a stable Dagster-native key without replacing the translator: + +```yaml +sources: + - name: raw__orders + meta: + dagster: + asset_key: [warehouse, raw, orders] +``` + +Override `is_asset_node` or `is_asset_check` when one SQLBuild project intentionally exposes only a +subset through a particular Dagster definition. Filtering changes Dagster ownership; it does not +remove the resource from SQLBuild's compiled graph. + +### Check structure + +Each `check` dict passed to check translator methods contains: + +```json +{ + "id": "audit:not_null:model:fact_orders:order_id", + "kind": "audit", + "name": "not_null", + "checked_asset_ids": ["model:fact_orders"], + "path": "audits/generic/not_null.sql", + "severity": "error", + "attached_target_name": "fact_orders", + "attached_column_name": "order_id" +} +``` + +Scenario checks include additional fields: + +```json +{ + "id": "scenario:daily_revenue_minimal", + "kind": "scenario", + "name": "daily_revenue_minimal", + "checked_asset_ids": ["model:daily_revenue"], + "path": "tests/scenarios/revenue/daily_revenue_minimal.sql", + "assertion_names": ["no_negative_revenue"], + "expected_model_names": ["daily_revenue"], + "fixture_refs": ["stg_orders", "stg_payments"] +} +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md new file mode 100644 index 0000000000..18fa403805 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md @@ -0,0 +1,175 @@ + + +# Overview + +> Orchestrate SQLBuild pipelines with Dagster scheduling, retries, and asset UI. + +Online: https://docs.sqlbuild.com/integrations/dagster + +## Contents + +- Install +- Try it +- How it works +- Quickstart +- Asset selection +- Checks +- Execution correlation +- Scenarios as checks +- Project preparation +- Translator + +SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, tasks, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. + +## Install + +```bash +uv pip install 'sqlbuild[dagster]' +# or +pip install 'sqlbuild[dagster]' +``` + +This installs `dagster` and `dagster-webserver` alongside SQLBuild. + +## Try it + +```bash +sqb playground --template dagster +cd sqlbuild-playground +dagster dev -f dagster/definitions.py +``` + +This creates the waffle shop project with a `dagster/definitions.py` that includes asset definitions, scenario checks, and a configured resource. Open the Dagster UI, materialize the assets, then run the scenario checks. + +## How it works + +1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges, checks) +2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, task, and asset, with dependency edges preserved +3. `SqlBuildCliResource` shells out to `sqb build`, `sqb test`, `sqb scenario test`, etc. as subprocesses +4. Execution results (materializations, audit pass/fail, scenario outcomes) are read from SQLBuild's structured integration-result stream and emitted as Dagster `MaterializeResult` and `AssetCheckResult` events + +SQLBuild tests, audits, and Python checks become Dagster asset checks. Scenarios become asset checks attached to the models they exercise. + +## Quickstart + +```python +# definitions.py +from sqlbuild.integrations.dagster import ( + SqlBuildCliResource, + SqlBuildProject, + sqlbuild_assets, +) +import dagster as dg + +project = SqlBuildProject(project_dir=".") +project.prepare_if_dev() # auto-generates DAG artifact in dagster dev + +@sqlbuild_assets(project=project) +def my_sqlbuild_assets(context: dg.AssetExecutionContext, sqb: SqlBuildCliResource): + yield from sqb.cli(["build"], context=context).stream() + +defs = dg.Definitions( + assets=[my_sqlbuild_assets], + resources={"sqb": SqlBuildCliResource(project)}, +) +``` + +```bash +dagster dev -f definitions.py +``` + +Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. + +For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. + +## Asset selection + +When you select a subset of assets in the Dagster UI, the integration automatically: + +1. Maps selected Dagster asset keys back to SQLBuild node names using the DAG artifact +2. Writes the selectors to a temporary file +3. Passes `--select-file` to the `sqb` CLI so only the selected models are built + +This means Dagster's asset subsetting works naturally with SQLBuild's selector system. + +An explicit `--select`, `--select-file`, or `--exclude` supplied by application code takes +precedence; SQLBuild does not silently combine it with Dagster's implicit asset selection. + +## Checks + +SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: + +- **Unit tests** become checks attached to the models they test +- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` +- **Scenarios** become checks attached to the models they exercise +- **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate + +Check results are emitted with pass/fail status and metadata from the execution JSON. + +## Execution correlation + +When a Dagster execution context is passed to `sqb.cli`, SQLBuild propagates bounded orchestration +metadata to command-output and lifecycle records: + +- integration name (`dagster`) +- Dagster run ID and job name +- step/op key +- retry number +- partition key, when the run is partitioned + +Lifecycle events remain SQLBuild's immutable execution facts. Human stdout/stderr remains a +correlated troubleshooting transcript; the integration does not parse terminal text to establish +resource status. See [Execution Observability](../concepts/observability.md). + +## Scenarios as checks + +Scenarios can be included as asset checks alongside tests and audits (the default), or run separately: + +```python +from sqlbuild.integrations.dagster import sqlbuild_assets, sqlbuild_scenario_checks + +# Include scenario checks with other assets (default) +@sqlbuild_assets(project=project, include_scenario_checks=True) +def my_assets(context, sqb): + yield from sqb.cli(["build"], context=context).stream() + +# Or run scenarios separately +@sqlbuild_scenario_checks(project=project) +def my_scenario_checks(context, sqb): + yield from sqb.cli(["scenario", "test"], context=context).stream() +``` + +## Project preparation + +`SqlBuildProject.prepare()` regenerates the DAG artifact by running `sqb compile --dag`. Use `prepare_if_dev()` to only regenerate during local development: + +```python +project = SqlBuildProject(project_dir=".") +project.prepare_if_dev() # only runs when DAGSTER_IS_DEV_CLI is set +``` + +This keeps the Dagster UI in sync with your latest model changes during development without regenerating in production. + +## Translator + +Customise how SQLBuild nodes map to Dagster assets by subclassing `SqlBuildDagsterTranslator`: + +```python +from sqlbuild.integrations.dagster import SqlBuildDagsterTranslator +import dagster as dg + +class MyTranslator(SqlBuildDagsterTranslator): + def get_asset_key(self, node): + # Prefix all asset keys with the project name + return dg.AssetKey(["my_project", *node["asset_key"]]) + + def get_group_name(self, node): + # Group by materialization type instead of kind + return node.get("materialization_type", "other") + +@sqlbuild_assets(project=project, translator=MyTranslator()) +def my_assets(context, sqb): + yield from sqb.cli(["build"], context=context).stream() +``` + +See the [API reference](dagster-reference.md) for all translator methods. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md new file mode 100644 index 0000000000..a3409ba7d4 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -0,0 +1,226 @@ + + +# dlt + +> Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders. + +Online: https://docs.sqlbuild.com/integrations/dlt + +## Contents + +- Install +- Declarative dlt sources (YAML) +- dlt inside a Python loader +- Build integration + +[dlt](https://dlthub.com) is an open-source Python library for loading data from APIs, databases, cloud storage, and more. SQLBuild integrates with dlt two ways: + +- **Declarative dlt sources** - configure dlt directly in your source YAML, no Python code. SQLBuild runs the dlt pipeline as part of the build lifecycle. Best for the common REST API, SQL database, and filesystem cases. +- **dlt inside a Python loader** - wrap any `dlt.pipeline(...)` in a SQLBuild [source loader](../concepts/python-nodes/loaders.md) for full control when you need dlt features beyond the declarative surface. + +## Install + +```bash +pip install 'sqlbuild[dlt]' +# or +uv pip install 'sqlbuild[dlt]' +``` + +This installs dlt with the `duckdb`, `filesystem`, and `sql_database` extras alongside SQLBuild. + +## Declarative dlt sources (YAML) + +Declare a `dlt_sources` list in a `sources/*.yml` file. Each entry has a `type`, a `config` block passed to the dlt source, and a list of `resources` that become SQLBuild managed sources. SQLBuild generates a synthetic loader per resource and writes into the database its adapter manages, no Python and no destination setup required. + +Supported source types: `rest_api`, `sql_database`, `filesystem`. + +### REST API + +```yaml +# sources/github.yml +dlt_sources: + - type: rest_api + config: + client: + base_url: https://api.github.com/ + headers: + Authorization: "Bearer ${github_token}" + paginator: + type: header_link + resources: + - name: raw_github_issues + write_disposition: append + endpoint: + path: "repos/${github_owner}/${github_repo}/issues" + params: + state: all + per_page: 100 +``` + +`rest_api` requires `client.base_url` in `config`. Each resource needs an `endpoint` mapping; the dlt resource name comes from `endpoint.name` if present, otherwise the resource `name`. + +### SQL database + +```yaml +# sources/postgres.yml +dlt_sources: + - type: sql_database + config: + credentials: "${postgres_connection_string}" + resources: + - name: raw_customers + table: customers + write_disposition: merge + primary_key: id +``` + +`sql_database` requires `credentials` in `config`, and each resource requires a `table`. + +### Filesystem + +```yaml +# sources/files.yml +dlt_sources: + - type: filesystem + config: + bucket_url: "s3://my-bucket/events/" + resources: + - name: raw_events + write_disposition: append +``` + +`filesystem` requires `bucket_url` in `config`. + +### Resource options + +| Key | Description | +|-----|-------------| +| `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | +| `table` | Source table to replicate (`sql_database` only). Required for that type. | +| `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | +| `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | +| `primary_key` / `merge_key` | dlt keys for `merge`. | +| `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | +| `schema` | Per-resource schema override (falls back to the group `schema`). | + +Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](../concepts/interpolation.md): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. + +### Destination + +By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): + +```yaml +dlt_sources: + - type: rest_api + destination: + loader_file_format: parquet + config: + client: + base_url: https://api.example.com/ + resources: + - name: raw_widgets + endpoint: + path: widgets +``` + +### Reference in models + +Declared resources are managed sources, reference them like any other source: + +```sql +SELECT id, title, state FROM __source("raw_github_issues") +``` + +## dlt inside a Python loader + +When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](../concepts/python-nodes/loaders.md). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. + +```python +# loaders/github_sources.py +import dlt +from dlt.sources.rest_api import rest_api_source +from sqlbuild.loaders import loader +from sqlbuild.executor.load.models import LoaderContext + +@loader +def raw_github_issues(ctx: LoaderContext): + source = rest_api_source({ + "client": { + "base_url": "https://api.github.com/", + "headers": {"Authorization": f"Bearer {ctx.vars['github_token']}"}, + "paginator": {"type": "header_link"}, + }, + "resources": [{ + "name": "issues", + "endpoint": { + "path": "repos/{owner}/{repo}/issues", + "params": { + "owner": ctx.vars["github_owner"], + "repo": ctx.vars["github_repo"], + "state": "all", + "per_page": 100, + }, + }, + }], + }) + + pipeline = dlt.pipeline( + pipeline_name="github_issues", + destination=dlt.destinations.duckdb(ctx.connection), + dataset_name=ctx.destination_schema or "main", + ) + pipeline.run(source) +``` + +Bind it to a source with `managed: true` in `sources/*.yml`: + +```yaml +# sources/github.yml +sources: + - name: raw_github_issues + managed: true + table: issues + columns: + - name: id + type: INTEGER + - name: title + type: VARCHAR +``` + +### DuckDB connection sharing + +With the DuckDB adapter, pass `ctx.connection` to dlt's DuckDB destination to reuse SQLBuild's open connection, so dlt writes into the same database without a separate connection string: + +```python +pipeline = dlt.pipeline( + pipeline_name="my_pipeline", + destination=dlt.destinations.duckdb(ctx.connection), + dataset_name=ctx.destination_schema or "main", +) +``` + +### Warehouse destinations + +For Snowflake, BigQuery, or Databricks, configure dlt with its own credentials. dlt writes directly to the warehouse, and SQLBuild reads the resulting tables as sources: + +```python +pipeline = dlt.pipeline( + pipeline_name="api_data", + destination="snowflake", + dataset_name=ctx.destination_schema or "public", +) +``` + +Configure dlt credentials via its own `secrets.toml` or environment variables, as in the [dlt documentation](https://dlthub.com/docs/general-usage/credentials/setup). + +## Build integration + +Whether declarative or Python, loaders run automatically during `sqb build` (when `auto_load_sources` is enabled), so dlt pipelines execute as part of the normal build lifecycle: + +```bash +sqb build # loaders run, then models build +sqb build --no-load # skip loading, use existing source data +sqb load # run loaders standalone +``` + +See [Loaders](../concepts/python-nodes/loaders.md) for write strategies, the loader context API, auto-load behavior, and source deferral. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md new file mode 100644 index 0000000000..68e355cbf3 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -0,0 +1,168 @@ + + +# ingestr + +> Declarative data ingestion from 50+ sources using ingestr. + +Online: https://docs.sqlbuild.com/integrations/ingestr + +## Contents + +- Install +- How it works +- Example: PostgreSQL to DuckDB +- Example: Stripe with incremental merge +- Configuration reference +- Template variables +- Destination URI +- Reload +- Build integration +- Supported sources + +[ingestr](https://github.com/bruin-data/ingestr) is an open-source CLI tool by [Bruin](https://github.com/bruin-data) that copies data from any source to any destination using a single command. SQLBuild integrates with ingestr as a declarative source loader - you configure the ingestion directly in your source YAML, and SQLBuild handles execution as part of the build lifecycle. + +## Install + +```bash +pip install 'sqlbuild[ingestr]' +# or +uv pip install 'sqlbuild[ingestr]' +``` + +This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. + +## How it works + +1. Declare an `ingestr` block on a source in `sources/*.yml` +2. SQLBuild generates a synthetic loader that calls `ingestr ingest` as a subprocess +3. ingestr reads from the configured source and writes directly to the SQLBuild adapter's database +4. The destination URI is built automatically from your SQLBuild connection config - no manual destination setup + +No Python loader code is needed. The YAML declaration is the entire configuration. + +## Example: PostgreSQL to DuckDB + +Replicate a table from PostgreSQL into your local DuckDB project: + +```yaml +# sources/raw.yml +sources: + - name: raw_orders + table: orders + ingestr: + source_uri: "postgresql://user:pass@host:5432/mydb" + source_table: "public.orders" +``` + +That's it. Run `sqb load` or `sqb build` and ingestr copies the `orders` table into your project. + +## Example: Stripe with incremental merge + +Load Stripe charges with incremental merge on a primary key: + +```yaml +sources: + - name: raw_stripe_charges + table: charges + ingestr: + source_uri: "stripe://${stripe_api_key}" + source_table: "charges" + strategy: merge + primary_key: id + incremental_key: created +``` + +On subsequent runs, ingestr merges new and updated records based on the `id` column, using `created` to detect changes. + +## Configuration reference + +The `ingestr` block on a source supports the following fields: + +| Field | Required | Description | +|-------|----------|-------------| +| `source_uri` | Yes | ingestr source connection URI (e.g. `postgresql://...`, `stripe://...`, `shopify://...`) | +| `source_table` | Yes | Source table or resource name to ingest | +| `strategy` | No | Ingestion strategy: `replace`, `append`, `merge`, `delete+insert`, or `truncate+insert` | +| `incremental_key` | No | Column used for incremental change detection | +| `primary_key` | No | Primary key column(s) for merge strategy (string or list) | +| `columns` | No | Comma-separated column list to select from the source | +| `extra_args` | No | Additional CLI arguments passed to `ingestr ingest` (list of strings) | + +### Strategy mapping + +| Strategy | Behavior | +|----------|----------| +| `replace` | Drop and recreate the destination table (default when no strategy is set) | +| `append` | Insert all rows without deduplication | +| `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | +| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements | +| `truncate+insert` | Truncate destination table, then insert all rows | + +## Template variables + +All string fields in the `ingestr` block support SQLBuild project variable substitution and context templates: + +```yaml +sources: + - name: raw_orders + ingestr: + source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" + source_table: "public.orders" +``` + +Variables are resolved from the project's merged variable config (project + target + local). + +Set sensitive values in `sqlbuild_local.toml` (gitignored): + +```toml +[vars] +pg_user = "readonly" +pg_password = "secret" +pg_host = "prod-db.example.com" +pg_database = "analytics" +stripe_api_key = "sk_live_..." +``` + +## Destination URI + +SQLBuild automatically builds the ingestr destination URI from your adapter connection config. All supported adapters work without manual destination configuration: + +| Adapter | Destination URI format | +|---------|----------------------| +| DuckDB | `duckdb:///path/to/db.duckdb` | +| MotherDuck | `motherduck://database` | +| PostgreSQL | `postgresql://user:pass@host:port/db` | +| Snowflake | `snowflake://user:pass@account/db/schema` | +| BigQuery | `bigquery://project` | +| Databricks | `databricks://token:...@host` | +| SQL Server | `mssql://user:pass@host:port/db` | + +## Reload + +When `--reload` is passed, ingestr runs with `--full-refresh`, forcing a complete reload regardless of the configured strategy: + +```bash +sqb load --reload +sqb build --reload +``` + +## Build integration + +ingestr sources are managed sources - they participate in the same lifecycle as Python loaders: + +```bash +# ingestr runs automatically before dependent models +sqb build + +# run ingestr sources standalone +sqb load + +# skip loading +sqb build --no-load +``` + +See [Loaders](../concepts/python-nodes/loaders.md) for details on auto-load behavior, source deferral, and the `--load` / `--no-load` / `--reload` flags. + +## Supported sources + +ingestr supports 50+ sources including databases, SaaS APIs, and file systems. See the [ingestr documentation](https://bruin-data.github.io/ingestr/) for the full list of supported sources and their URI formats. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md new file mode 100644 index 0000000000..d028d112ec --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md @@ -0,0 +1,181 @@ + + +# Rivers + +> Orchestrate SQLBuild pipelines with Rivers scheduling, jobs, and asset tracking. + +Online: https://docs.sqlbuild.com/integrations/rivers + +## Contents + +- Install +- Try it +- How it works +- Quickstart +- SqlBuildProject +- sqlbuild_assets +- SqlBuildRiversTranslator +- Running locally + +SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, tasks, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. + +## Install + +```bash +pip install 'sqlbuild[rivers]' +# or +uv pip install 'sqlbuild[rivers]' +``` + +This installs `rivers` alongside SQLBuild. + +## Try it + +```bash +sqb playground --template rivers +cd sqlbuild-playground +uv run rivers dev rivers_pipeline.definitions +``` + +This creates the waffle shop project with a `rivers_pipeline/definitions.py` that includes asset definitions and a configured job. Open the Rivers UI to inspect assets and trigger materializations. + +## How it works + +1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges) +2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, task, and asset, with dependency edges preserved +3. The decorated function runs `sqb build` as a subprocess and yields `Materialization` events for each output + +## Quickstart + +```python +# definitions.py +from __future__ import annotations + +import subprocess +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import rivers as rs + +from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets + +PROJECT_DIR = Path(__file__).resolve().parent.parent +SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) +SQLBUILD_PROJECT.prepare_if_dev() + +@sqlbuild_assets(project=SQLBUILD_PROJECT) +def my_assets(context: Any) -> Iterator[Any]: + completed = subprocess.run( + ["sqb", "build"], + cwd=PROJECT_DIR, + capture_output=True, + check=False, + text=True, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + for output_name in context.output_selection: + yield rs.Materialization(output_name=output_name) + +repo = rs.CodeRepository( + assets=[my_assets], + jobs=[ + rs.Job( + name="sqlbuild_pipeline", + assets=[my_assets], + executor=rs.Executor.in_process(), + ), + ], +) +``` + +## SqlBuildProject + +`SqlBuildProject` manages paths and DAG artifact generation: + +```python +project = SqlBuildProject(project_dir=Path(".")) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `project_dir` | required | Path to the SQLBuild project root | +| `target_path` | `target` | Directory for build artifacts | +| `dag_filename` | `sqlbuild_dag.json` | DAG artifact filename | +| `sqb_command` | `("sqb",)` | Command to invoke SQLBuild | +| `prepare_project_cli_args` | `("compile", "--dag")` | CLI arguments for DAG generation | + +### Methods + +| Method | Description | +|--------|-------------| +| `prepare()` | Generate the DAG artifact by running `sqb compile --dag` | +| `prepare_if_dev()` | Generate the DAG artifact only when `RIVERS_DEPLOYMENT=dev` | +| `dag_path` | Property returning the full path to the DAG artifact | + +Use `prepare_if_dev()` so the DAG artifact is auto-generated during local development but not during production deployments where it should already exist. + +## sqlbuild_assets + +The `@sqlbuild_assets()` decorator creates a Rivers multi-asset definition from a SQLBuild DAG artifact: + +```python +@sqlbuild_assets(project=project) +def my_assets(context): + ... +``` + +| Parameter | Description | +|-----------|-------------| +| `project` | `SqlBuildProject` instance (generates and locates the DAG artifact) | +| `dag` | Alternative: pass a DAG path or dict directly instead of a project | +| `translator` | Optional `SqlBuildRiversTranslator` for custom name/tag/metadata mapping | +| `name` | Optional asset name override | + +Pass either `project` or `dag`, not both. + +## SqlBuildRiversTranslator + +Override the default mapping from SQLBuild DAG nodes to Rivers asset metadata: + +```python +from sqlbuild.integrations.rivers import SqlBuildRiversTranslator + +class CustomTranslator(SqlBuildRiversTranslator): + def get_group_name(self, node): + return "my_custom_group" + + def get_tags(self, node): + tags = super().get_tags(node) + tags.append("team:analytics") + return tags + +@sqlbuild_assets(project=project, translator=CustomTranslator()) +def my_assets(context): + ... +``` + +### Translator methods + +| Method | Default behavior | +|--------|-----------------| +| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `task__prepare_orders`, `asset__orders_export`) | +| `get_group_name(node)` | Uses the project name | +| `get_tags(node)` | Includes `sqlbuild/kind:` plus model tags | +| `get_kinds(node)` | Returns `["sqlbuild", ""]` | +| `get_metadata(node)` | Includes SQLBuild name, kind, path, target, columns, etc. | + +## Running locally + +Run the pipeline directly without the Rivers UI: + +```bash +uv run python rivers_pipeline/definitions.py +``` + +Or start the Rivers development server: + +```bash +uv run rivers dev rivers_pipeline.definitions +``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md new file mode 100644 index 0000000000..7dd8a6681d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md @@ -0,0 +1,279 @@ + + +# Quickstart + +> Get a SQLBuild project running locally with DuckDB in under a minute. + +Online: https://docs.sqlbuild.com/quickstart + +## Contents + +- Prerequisites +- 1. Create the playground +- 2. Plan +- 3. Build +- 4. Verify +- 5. Common operations +- What you just built +- Next steps + +This guide walks you through creating and running a complete transformation project - views, tables, incremental models, audits, tests, and a custom materialization - all running locally against DuckDB. No external data setup or warehouse credentials required. + +## Prerequisites + +- Python 3.12+ +- SQLBuild installed: `uv pip install sqlbuild` or `pip install sqlbuild` + +Confirm the installed release with `sqb --version`. + +## 1. Create the playground + +```bash +sqb playground waffle-shop +cd waffle-shop +``` + +This creates a self-contained waffle shop project with everything you need to explore SQLBuild. + +The playground creates the shared `sqlbuild_project.toml`; that is sufficient for this +quickstart. In a team project, you can additionally create a gitignored +`sqlbuild_local.toml` to select your usual target and override your personal development +schema or credentials without changing the committed project config. See +[Project Configuration](concepts/project-configuration.md#sqlbuild_local-toml). + +## 2. Plan + +Preview what SQLBuild will do: + +```bash +sqb plan +``` + +``` +Plan ready (13 selected) + +First run (12) + stg_customers view + stg_payments view + stg_orders view + daily_order_partitioned partition_tracked (custom) + daily_revenue table + dim_customers table + fact_orders table + customer_status_snapshot merge (timestamp) + hourly_order_activity delete_insert (timestamp, microbatch) + daily_activity_rollup delete_insert (timestamp, microbatch) + hourly_activity_with_daily_context delete_insert (timestamp, microbatch) + order_status_index delete_insert (integer) + +Seeds (1) + waffle_types +``` + +Source data is defined as inline expressions in `sources/raw.yml`, so SQLBuild resolves everything at compile time - no seeding step needed. + +## 3. Build + +Execute the full build: + +```bash +sqb --no-color build +``` + +SQLBuild materializes all models in DAG order. SQL unit tests run before their target models. Error-severity audits run against a staging table before promotion - if any fail, the production table is untouched: + +``` +Execution sqb build (concurrency: 1) + + 1/13 seed waffle_types OK 0.09s + 2/13 view stg_customers OK 0.05s + audit not_null (customer_id) PASS + audit unique (customer_id) PASS + audit not_null (email) PASS + 3/13 view stg_payments OK 0.02s + audit not_null (payment_id) PASS + audit unique (payment_id) PASS + audit not_null (order_id) PASS + 4/13 view stg_orders OK 0.03s + test test_stg_orders PASS + audit not_null (order_id) PASS + audit unique (order_id) PASS + audit not_null (customer_id) PASS + audit accepted_values (status) PASS + 5/13 custom daily_order_partitioned OK 0.17s + audit expression_is_true PASS 4/4 + audit not_null (order_date) PASS 4/4 + 6/13 table daily_revenue OK 0.05s + audit expression_is_true PASS + 7/13 table dim_customers OK 0.05s + audit not_null (customer_id) PASS + audit unique (customer_id) PASS + 8/13 table fact_orders OK 0.04s + audit not_null (order_id) PASS + 9/13 table customer_status_snapshot (merge) OK 0.04s + audit not_null (customer_id) PASS + audit unique (customer_id) PASS + 10/13 table hourly_order_activity (delete_insert) OK 0.16s + audit (d) expression_is_true PASS 4/4 + audit (d) not_null (activity_hour) PASS 4/4 + audit (f) expression_is_true PASS + audit (f) not_null (activity_hour) PASS + 11/13 table order_status_index (delete_insert) OK 0.03s + audit not_null (order_id) PASS + audit unique (order_id) PASS + 12/13 table daily_activity_rollup (delete_insert) OK 0.08s + audit (d) expression_is_true PASS 2/2 + audit (d) not_null (activity_day) PASS 2/2 + audit (f) expression_is_true PASS + audit (f) not_null (activity_day) PASS + 13/13 table hourly_activity_with_daily_context (delete_insert) OK 0.17s + audit (d) expression_is_true PASS 4/4 + audit (d) not_null (activity_hour) PASS 4/4 + audit (f) expression_is_true PASS + audit (f) not_null (activity_hour) PASS + +Completed successfully. +PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) +``` + +Notice `audit (d)` and `audit (f)` on incremental models - these are delta-phase and final-phase audits. Delta audits validate each batch before DML is applied; final audits run against the target after promotion. + +## 4. Verify + +Run the plan again to see the steady-state: + +```bash +sqb plan +``` + +``` +Plan ready (13 selected) + +Normal (12) + 3 view + 3 table + 3 delete_insert (timestamp, microbatch) + 1 partition_tracked (custom) + 1 merge (timestamp) + 1 delete_insert (integer) + +Seeds (1) + waffle_types +``` + +All models now show `Normal` instead of `First run`. + +## 5. Common operations + +```bash +# Rebuild a specific model +sqb build --select daily_revenue + +# Rebuild models under a path +sqb build --select path:models/marts + +# Full refresh of everything +sqb build --full-refresh + +# Run tests only +sqb test + +# Run audits only +sqb audit + +# Inspect lineage +sqb lineage fact_orders --direction both + +# Compile and check contracts +sqb compile +``` + +## What you just built + +### Model DAG + +``` +raw__customers ──> stg_customers ──> dim_customers +raw__orders ────> stg_orders ────> fact_orders ──> customer_status_snapshot +raw__payments ──> stg_payments ──> │ order_status_index + │ hourly_order_activity ──> daily_activity_rollup + │ hourly_activity_with_daily_context + └──> daily_revenue + └──> daily_order_partitioned +``` + +### Materialization types + +The project demonstrates several materialization strategies: + +- **Views** (`stg_customers`, `stg_orders`, `stg_payments`) - lightweight staging layer +- **Tables** (`fact_orders`, `dim_customers`, `daily_revenue`) - full table rebuilds +- **Merge incremental** (`customer_status_snapshot`) - upsert with timestamp cursor +- **Delete/insert incremental** (`hourly_order_activity`, `daily_activity_rollup`, `hourly_activity_with_daily_context`) - microbatch processing with configurable batch sizes +- **Integer cursor incremental** (`order_status_index`) - non-timestamp incremental +- **Custom materialization** (`daily_order_partitioned`) - Python-defined partition-tracked strategy + +See [Incremental](concepts/incremental.md) for details on cursor-based strategies and microbatch processing. + +### Project structure + +``` +waffle-shop/ + sqlbuild_project.toml # project config + sources/ + raw.yml # source declarations (inline expression data) + seeds/ + waffle_types.csv # seed data + lookups.yml # seed declarations + functions/ + sql/ + udf__is_completed_order.sql # scalar SQL UDF + table_fn__customer_orders.sql # table function + schemas/ + orders/ + order.sql # reusable model column schema + models/ + staging/ + stg_customers.sql # view (audits declared in MODEL() header) + stg_orders.sql # view + stg_payments.sql # view + marts/ + fact_orders.sql # table + dim_customers.sql # table + daily_revenue.sql # table + daily_order_partitioned.sql # custom materialization + hourly_order_activity.sql # microbatch incremental + hourly_activity_with_daily_context.sql # microbatch incremental + daily_activity_rollup.sql # microbatch incremental + intermediate/ + customer_status_snapshot.sql # merge incremental + order_status_index.sql # integer cursor incremental + audits/ + generic/ + expression_is_true.sql # custom generic audit + tests/ + unit/ + test_stg_orders.sql # model unit test + test_fact_orders.sql # multi-model test + test_daily_revenue_chain.sql # multi-model test + test_line_total_cents_macro.sql # macro test + scenarios/ + daily_revenue_minimal.sql # E2E scenario test + daily_revenue_multi_order.sql # E2E scenario test + macros/ + currency.py # cents_to_dollars macro + datetime.py # adapter-portable timestamp_trunc + materializations/ + partition_tracked.py # custom materialization +``` + +## Next steps + +- [Models](concepts/models.md) - understand `MODEL()` headers and materialization types +- [Rules](concepts/rules.md) - add compile-time SQL, architecture, and model-shape checks +- [Functions](concepts/functions.md) - SQL UDFs, Python UDFs, and table functions +- [Incremental](concepts/incremental.md) - learn about cursor-based incremental strategies +- [Audits](concepts/audits.md) - configure data quality checks +- [Testing](concepts/testing.md) - write SQL unit tests and multi-model tests with macro support +- [Column Lineage](concepts/column-lineage.md) - trace individual columns through your pipeline +- [CLI Reference](cli/build.md) - full command reference diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/exploring-a-project.md b/src/sqlbuild/.agents/skills/sqlbuild/references/exploring-a-project.md new file mode 100644 index 0000000000..4ddd5bed19 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/exploring-a-project.md @@ -0,0 +1,114 @@ +# Exploring a project + +Use SQLBuild's own introspection instead of grepping and guessing. `scope`, `lineage`, `dag`, +`compile` and `test --inspect` are offline; `query`, `plan` and `diff` read the warehouse. + +## Contents + +- Declaration scope: sqb scope +- Model and column lineage: sqb lineage +- Graph and plan as data +- Looking at data: sqb query +- Test boundaries: sqb test --inspect + +## Declaration scope: sqb scope + +Macros, enums and constants are visible by folder. `sqb scope` answers, for any resource: what can +it see, what does it actually use, where did each declaration come from, and what would break if +it moved. It is read-only and offline. + +Targets are kind-qualified; bare names are rejected: + +```bash +sqb scope model:stg_orders +sqb scope test:stg_orders__excludes_cancelled +sqb scope macro:normalize_order_status +sqb scope enum:order_status +sqb scope constant:minimum_order_value +sqb scope models/staging/orders/stg_orders.sql +``` + +Model-private declarations include the owner: `enum:model:stg_orders._state`. + +Common questions: + +| Question | Command | +|---|---| +| What does this model use, including declarations those use? | `sqb scope model:orders --used-only --dependency-depth 1` | +| Why is this macro visible or not visible here, and where should it live? | `sqb scope model:orders --explain macro:formatted_order_total` | +| Which declarations are close by but unavailable? | `sqb scope model:orders --include-nearby` | +| What could a new file at this path use? | `sqb scope --at models/commerce/new_model.sql` | +| Would moving this model break visibility? | `sqb scope model:orders --as-path models/finance/orders.sql` | +| Which folders hold declarations, and what is in one? | `sqb scope model:orders --browse models`, then `sqb scope model:orders --list models/commerce` | +| Only macros matching a pattern | `sqb scope model:orders --kind macro --match "format_*"` | + +Report sections: **Used**, **Scope chain** (owner folder, parents, project globals), **Available** +(globals collapsed; `--globals all` expands them), **Relationship grants** (tests and scenarios), +**Nearby unavailable**, **Diagnostics**. Use `--json` for machine-readable output. + +Placement rules the scope report reflects: + +| Location | Visible to | +|---|---| +| Top-level `macros/`, `enums/`, `constants/` | The whole project | +| `/_sqlbuild/macros/` (or `enums/`, `constants/`) | That folder and everything below | +| `/_sqlbuild/_macros/` (or `_enums/`, `_constants/`) | Files directly in that folder only | +| Underscored `_name` constant or enum inside `MODEL()` | That model only | + +A declaration must sit in the narrowest folder containing all of its consumers; a project-wide +declaration used by only one folder tree is rejected at compile. `--explain` shows the narrowest +required placement. + +## Model and column lineage: sqb lineage + +```bash +sqb lineage fact_orders # upstream tree (default) +sqb lineage fact_orders --direction both # upstream and downstream +sqb lineage fact_orders --direction downstream --depth 1 +sqb lineage --select tag:finance --format list # edges for a selection +sqb lineage fact_orders.total_cents # where a column comes from +sqb lineage fact_orders.order_id --direction downstream # who consumes a column +``` + +- Targets are resources (models, sources, seeds, functions). `model.column` switches to column + lineage. +- Formats: `tree` (default), `list` (edges), `json` (nodes, edges, `qualified_name`). +- Column edges carry a transform type (`direct`, `expression`, `aggregation`, `cast`, `star`, + `constant`) and a confidence level. `--mode fast` trades detail for speed on large projects. +- Before renaming, retyping or dropping a column, run a downstream column trace and update or + warn about every consumer. + +## Graph and plan as data + +- `sqb dag --json`: every node, edge and check. Good for scripting over the whole project. +- `sqb compile --json`: diagnostics, contract results and per-model lineage summaries. + `--select` focuses analysis while keeping whole-project reference checks. +- `sqb plan --json --select `: per model the action, reason, incremental strategy, + cursor bounds, backfill and `qualified_name` (the physical relation). + +Plan reasons: first run, query changed, config changed, schema changed, function changed, upstream +changed, run despite unchanged. Direct mode still builds the whole selected scope; the reasons +explain it. + +## Looking at data: sqb query + +```bash +sqb query "SELECT status, COUNT(*) FROM analytics.orders GROUP BY 1" --format table +sqb query --file /tmp/check.sql --limit 100 +``` + +Runs raw SQL on the active target (physical names). Formats: `long` (default), `table`, `json`, +`csv`. Default limit 20; `--no-limit` removes it. Profile inputs before writing joins and check +outputs after building. + +## Test boundaries: sqb test --inspect + +`sqb test --select fact_orders --inspect` lists mocked sources and refs, the real models a test +will execute in order, expected models and unsatisfied dependencies, without a warehouse. Use it +before writing or debugging a multi-model test. + +Full reference: [docs/cli/scope.md](docs/cli/scope.md), +[docs/concepts/declaration-scopes/explorer.md](docs/concepts/declaration-scopes/explorer.md), +[docs/cli/lineage.md](docs/cli/lineage.md), [docs/concepts/column-lineage.md](docs/concepts/column-lineage.md), +[docs/cli/dag.md](docs/cli/dag.md), [docs/cli/query.md](docs/cli/query.md), +[docs/concepts/planning.md](docs/concepts/planning.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md new file mode 100644 index 0000000000..644c78e5c5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -0,0 +1,146 @@ +# Incremental and microbatch models + +SQLBuild incremental models are cursor-driven. SQLBuild reads the target's current cursor maximum +and the listed inputs' extents, computes the replay window, filters inputs to it, and applies DML. +There is no separate checkpoint to manage in the default sequential path. + +## Contents + +- Strategies +- Cursors and the automatic input filter +- Microbatch execution +- Watermark roles and limits +- Replay, full refresh and schema changes +- Workflow and checks + +## Strategies + +| Strategy | Needs | Behaviour | +|---|---|---| +| `append` | optional cursor | Inserts new rows. `append_cursor_inclusive true` (default) uses `>=` at the boundary | +| `delete_insert` | `cursor` or `unique_key` | Deletes the target rows in the window (or matching keys), inserts the delta | +| `merge` | `unique_key` | Upserts by key; the cursor limits which upstream rows are scanned | + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs ( + fact_orders ordered_at, + ), +); + +SELECT customer_id, ordered_at AS activity_hour, line_total_cents +FROM __ref("fact_orders") +``` + +## Cursors and the automatic input filter + +| Field | Meaning | +|---|---| +| `cursor` | Output column tracking position | +| `cursor_type` | `timestamp` or `integer` | +| `cursor_grain` | `second`, `minute`, `hour`, `day`, `month`, `year` (timestamp cursors) | +| `cursor_start` | Floor; never replay before this value | +| `cursor_inputs` | Upstream ref/source names and their cursor columns; required with multiple inputs | +| `lookback` | Extend the window start backwards, for example `lookback 3d`, for late data | + +Behaviour to rely on: + +- **Listed inputs are filtered for you.** SQLBuild wraps each input listed in `cursor_inputs` in + `WHERE >= start AND < end`. Do not add the same filter by hand. +- **Unlisted inputs are read in full** and do not bound the window. Leave dimension and lookup + tables unlisted; list every input whose new data should drive reprocessing. +- With several listed inputs, the window end is the minimum of their maxima, so a fast input + cannot run ahead of a slow one. +- `__cursor_start()` (inclusive) and `__cursor_end()` (exclusive) expose the effective bounds when + SQL needs them explicitly, for example to filter an unlisted input or compute a derived range. + They are valid only in cursor incremental model SQL, not in tests, hooks, audits or functions. +- For `delete_insert` and `merge`, only target rows whose cursor falls inside the window are + rewritten. A change in an unlisted input does not rewrite older rows. + +## Microbatch execution + +`incremental_mode microbatch` splits the window into batches. Each batch creates its delta, runs +delta audits, applies DML and cleans up. Choose the strategy explicitly: + +| `microbatch_strategy` | Window source | +|---|---| +| `watermark` | Declared input availability (timestamp or integer cursors) | +| `rolling_window` | A timestamp window relative to the current run | + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + cursor activity_hour, + cursor_type timestamp, + cursor_grain hour, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor_inputs ( + fact_orders (column ordered_at, roles [filter, watermark]), + ), + batch_size 1d, +); +``` + +- `batch_size` is a duration (`1d`, `6h`, `1mo`) or an integer for integer cursors. +- Sequential batches are the default and use no state table. Concurrent batches are opt-in: + `[settings] microbatch_concurrency = true` in the project plus `batch_concurrency N` on the + model (`delete_insert` only). Raise concurrency deliberately; each batch uses warehouse compute. +- Mixed grains align automatically: an hourly model downstream of a daily one processes whole + days. + +## Watermark roles and limits + +Each watermark `cursor_inputs` entry declares `roles`: + +- `filter`: SQLBuild filters that input's rows to each batch window. +- `watermark`: the input's maximum contributes availability. A watermark-only input is not + filtered. +- `cursor_watermark_mode all` waits for the slowest watermark input; `any` lets any one advance. + +Batch limits protect ordinary runs from huge ranges: + +```sql +microbatch_limit ( + max_batches 7, + action cap_from_end, +), +``` + +| `action` | When the range exceeds `max_batches` | +|---|---| +| `error` | Fail before hooks or mutation | +| `warn` | Warn and run the full range | +| `cap_from_start` | Run the oldest N batches, defer the rest | +| `cap_from_end` | Run the newest N batches, defer older work | + +Project-wide `[microbatches.limits]` supports only `error` and `warn`. `--max-microbatches N` on +plan or build is a one-run hard ceiling and explicit authorization for an intentional backfill. + +## Replay, full refresh and schema changes + +- `replay_on_change` decides how much to reprocess when the model's identity changes: + `forward` (default), `full`, or `bounded-` such as `bounded-14d`. +- `--full-refresh` rebuilds selected models unless a model sets `full_refresh false`; + `full_refresh true` always rebuilds. +- `on_schema_change`: `append_new_columns` (default), `sync_all_columns`, `ignore`, `fail`. +- Cursor overrides for one run: `--start-cursor-ts/--end-cursor-ts` (ISO) or + `--start-cursor-int/--end-cursor-int`. + +## Workflow and checks + +1. `sqb compile --select `. +2. `sqb plan --select ` and read the action, reason, cursor bounds and backfill range. +3. `sqb build --select `. +4. Verify with `sqb diff prod:dev --bounded 14d --select ` or a query diff over the window. + +Full reference: [docs/concepts/incremental.md](docs/concepts/incremental.md), +[docs/concepts/planning/cascade-propagation.md](docs/concepts/planning/cascade-propagation.md), +[docs/concepts/snapshots.md](docs/concepts/snapshots.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md new file mode 100644 index 0000000000..b13b5282bd --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md @@ -0,0 +1,133 @@ +# Macros, enums, constants and where they live + +## Contents + +- Macros +- Constants and enums +- Model-private values +- Interpolation +- Placement and visibility +- Diagnosing visibility problems + +## Macros + +Macros are plain Python functions that return final SQL strings. Every public function in a macro +file is callable from SQL as `@name(...)`; prefix helpers, constants and classes with `_`. + +```python +# macros/currency.py +def cents_to_dollars(expression: str) -> str: + return f"ROUND(CAST({expression} AS DOUBLE) / 100, 2)" +``` + +```sql +SELECT order_id, @cents_to_dollars("amount_cents") AS amount_dollars +FROM __ref("stg_orders") +``` + +- Arguments are Python literals: strings, numbers, `True`/`False`, lists, dicts, `None`, or another + macro call. Pass SQL expressions and column names as quoted strings. Keyword arguments work: + `@mock_orders(count=5, status="completed")`. +- A macro used in SQL must return a string, and its output is final SQL. Never return text + containing another `@macro()` call; compose by calling the Python functions directly, including + functions imported from other visible macro files (`from macros.currency import add_tax`). +- Macros work in model SQL, SQL hooks, unit tests, scenarios, audits, SQL functions and inline + source expressions. They are not allowed in ordinary `MODEL()` configuration fields. +- Imported functions keep their original identity; they are not re-exported by the importing + file. Import cycles and imports from outside the file's scope are rejected. + +## Constants and enums + +Declared in `.sql` files; one file can hold several declarations. + +```sql +-- constants/commerce/thresholds.sql +CONSTANT (name min_items, value 7); +CONSTANT (name fallback_source, value "web"); + +-- enums/order_status.sql +ENUM ( + name order_status, + members [PLACED, COMPLETED, CANCELLED], +); +ENUM ( + name source, + members (WEB "web", PARTNER "partner"), +); +``` + +Use `@const("min_items")` and `@enum("order_status").COMPLETED` in SQL. SQLBuild validates names +and members at compile time and renders safe literals for the adapter. Constants are data (strings, +integers, booleans, floats, exact decimals, `null`, lists and objects), never raw SQL snippets; use a +macro when you need SQL. + +## Model-private values + +Values used by one model belong in its header, with underscore names: + +```sql +MODEL ( + enums ( + _state [OPEN, CLOSED], + ), + constants ( + _min_items 2, + ), +); + +SELECT * FROM __ref("order_batches") +WHERE state = @enum("_state").OPEN AND item_count > @const("_min_items") +``` + +They are visible only in that model's query and its inline SQL hooks, not in tests, scenarios, +other models or named hooks. + +## Interpolation + +| Syntax | Meaning | +|---|---| +| `@@name` | Project variable (`[vars]`, overridable with `--vars '{"name": ...}'`) | +| `@@ENV:NAME` | Environment variable at compile time | +| `@@CTX:name` | Destination relation, target or run ID; SQL hooks only | +| `${ENV:NAME}` | Environment variable inside TOML or YAML config values | + +## Placement and visibility + +| Location | Visible to | +|---|---| +| Top-level `macros/`, `enums/`, `constants/` | The whole project | +| `/_sqlbuild/macros/` (also `enums/`, `constants/`) | That folder and all folders below | +| `/_sqlbuild/_macros/` (also `_enums/`, `_constants/`) | Files directly in that folder | +| Underscored value in `MODEL()` | That model only | + +Declarations never flow upward or to siblings. `_sqlbuild/` may contain only those six role +folders and must sit below a concrete owner folder (not the project root or `models/` itself). +Older projects may use role folders such as `models/marts/_macros/` directly under the owner; they +remain supported. + +**Narrowest placement is enforced.** SQLBuild computes the lowest common owner folder of every +consumer. A declaration placed wider than needed, for example a top-level macro used only under +`models/commerce/`, is rejected at compile. When adding a declaration: + +1. Find every file that will use it. +2. Put it in the nearest folder containing all of them: `MODEL()` for one model, `_sqlbuild/_/` + for one folder, `_sqlbuild//` for one folder tree, top level only for separate trees such + as models and tests. +3. Run `sqb compile` and `sqb scope`. A misplaced declaration fails with `S024`, which names the + required folder, for example `Move it to 'models/marts/_sqlbuild/_constants/'`. + +## Diagnosing visibility problems + +```bash +sqb scope model:orders --explain macro:formatted_order_total # why visible or not; required placement +sqb scope model:orders --include-nearby # close declarations out of reach +sqb scope --at models/finance/revenue.sql # what a new file there could use +sqb scope model:orders --as-path models/finance/orders.sql # effect of moving a model +``` + +Full reference: [docs/concepts/macros.md](docs/concepts/macros.md), +[docs/concepts/macros/composition-and-context.md](docs/concepts/macros/composition-and-context.md), +[docs/concepts/constants.md](docs/concepts/constants.md), [docs/concepts/enums.md](docs/concepts/enums.md), +[docs/concepts/model-private-values.md](docs/concepts/model-private-values.md), +[docs/concepts/interpolation.md](docs/concepts/interpolation.md), +[docs/concepts/declaration-scopes.md](docs/concepts/declaration-scopes.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md new file mode 100644 index 0000000000..6a3bbb36cb --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md @@ -0,0 +1,157 @@ +# Testing: unit tests, scenarios, audits and Rules + +| Tool | Checks | Runs against | +|---|---|---| +| SQL unit test (`TEST`) | Model, macro or function logic on mocked inputs | One query; `sqb test`, and before each model in `sqb build` | +| Scenario (`SCENARIO`) | End-to-end logic across a real model graph | Isolated warehouse relations, or local DuckDB replay | +| Audit | Properties of built data (nulls, uniqueness, business rules) | Staging data or each incremental delta, before it is published | +| Rule | Project conventions and SQL quality | Compile time, offline | + +## Contents + +- Writing a unit test +- Multi-model tests +- Fixtures that fail early +- Repeated cases +- Scenarios +- Audits +- Rules and formatting + +## Writing a unit test + +Files live under `tests/unit/` (any subfolder). Each file is a `TEST();` header, CTEs, and a +required closing `SELECT 1`. + +```sql +-- tests/unit/test_stg_orders.sql +TEST(); + +WITH +__source__raw__orders AS ( + SELECT 1 AS id, 100 AS customer_id, 'completed' AS status, + CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at +), +__expected__stg_orders AS ( + SELECT 1 AS order_id, 100 AS customer_id, 'completed' AS status, + CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at +), +__assert__no_null_ids AS ( + SELECT * FROM __ref("stg_orders") WHERE order_id IS NULL +) +SELECT 1 +``` + +| CTE prefix | Meaning | +|---|---| +| `__source__` | Mock `__source("")` | +| `__ref__` | Mock `__ref("")`; that model's real SQL does not run | +| `__seed__` | Mock `__seed("")` | +| `__table_fn__` | Mock a table function invocation | +| `__macro__` | Replace every `@(...)` call | +| `__expected__` | Expected output of the model's real SQL (exact rows, compared both ways) | +| `__assert__` | Passes when the query returns zero rows | +| anything else | Helper CTE visible to mocks and model SQL | + +Macros work inside tests, so reusable mock generators such as `@mock_orders(count=5)` are normal. +Tests can also target a macro, UDF or table function directly; see +[docs/concepts/testing.md](docs/concepts/testing.md) for those modes. + +## Multi-model tests + +Mock only the sources, give `__expected__` for the model you care about, and every intermediate +model runs from its real SQL. Mock an intermediate with `__ref__` only when you deliberately +want to cut the chain there. + +```bash +sqb test --select fact_orders --inspect # offline: real models executed, mocks, gaps +sqb test --select fact_orders # run +``` + +## Fixtures that fail early + +With SQL analysis enabled, SQLBuild statically checks fixture shapes before connecting: missing +columns are reported with the test path and the models that read them. SQLBuild never invents +missing fixture values. Cast literal fixture values to the types the model expects +(`CAST('2026-04-01' AS DATE)`), and give every column the model reads. + +On failure, reports show unexpected and missing row counts with bounded samples, and the changed +columns when exactly one row differs each way. + +## Repeated cases + +| Pattern | Result | +|---|---| +| Several `TEST` blocks in one file | One result per block | +| A `VALUES` or macro case table | One aggregate result | +| `parameters (...)` plus named `cases (...)` in the header | One result per named case | + +```sql +TEST ( + name "order_status_maps_source_states", + parameters (source_status string, expected_status string), + cases ( + completed (source_status "completed", expected_status "completed"), + cancelled (source_status "cancelled", expected_status "excluded"), + ), +); +``` + +## Scenarios + +Use scenarios when a single test query becomes unwieldy: coherent customers, orders, payments and +refunds across many models. Files live under `tests/scenarios/` (one scenario per file; the stem +is its unique name), with a `SCENARIO (description "...", tags [...]);` header and the same CTE +conventions. + +```bash +sqb scenario test --select daily_revenue_minimal # build in isolated relations +sqb scenario capture --select daily_revenue_minimal # snapshot inputs as JSONL +sqb scenario test --local # replay locally in DuckDB +sqb scenario test --local --sync-snapshots # capture stale snapshots, then replay +``` + +Snapshots go to `tests/_scenario_snapshots//` and can be committed, so CI can run scenarios +without warehouse credentials. + +## Audits + +Built-ins: `not_null`, `unique`, `accepted_values (values [...])`, +`relationships (to "", field "")`. Attach them in the model's column schema: + +```sql +MODEL ( + materialized table, + columns ( + order_id (audits [not_null, unique]), + status (audits [accepted_values (values ["placed", "completed", "cancelled"])]), + ), +); +``` + +- `error` severity blocks publication (the staging table is not swapped in, or the delta is not + applied); `warn` reports and continues. Override per audit: `not_null (severity error)`. +- Custom generic audits live in `audits/generic/`; one-off singular audits in `audits/singular/` + with an `AUDIT (name "...", severity error);` header and a query returning failing rows. + Singular audits attach to the most downstream model they reference. +- Measurement audits compare metrics with warning and error thresholds. +- `sqb audit --select ` runs audits on their own. + +## Rules and formatting + +Rules are compile-time checks over SQL, models, dependencies, contracts, tests and paths. They run +during `sqb compile` and block artifacts when they fail. + +```bash +sqb rules list # catalogue +sqb rules show # what a rule checks and how to fix it +sqb rules run --select +sqb format --check # report formatting changes without writing +sqb format --select # rewrite deterministically +``` + +Fix the underlying issue rather than suppressing a finding; intentional exceptions are covered in +[docs/concepts/rules/findings-and-exceptions.md](docs/concepts/rules/findings-and-exceptions.md). + +Full reference: [docs/concepts/testing.md](docs/concepts/testing.md), +[docs/concepts/scenarios.md](docs/concepts/scenarios.md), [docs/concepts/audits.md](docs/concepts/audits.md), +[docs/concepts/rules.md](docs/concepts/rules.md), [docs/cli/test.md](docs/cli/test.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md b/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md new file mode 100644 index 0000000000..9d6fc76bf5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md @@ -0,0 +1,133 @@ +# Verifying changes with sqb diff + +A model that compiles, builds and passes tests can still return the wrong numbers. `sqb diff` +proves what changed. It has two modes: + +- **Model diff** compares built models between two targets (or two virtual environments). +- **Query diff** compares any two SQL queries on one connection. No build is required. + +## Contents + +- Choosing a mode +- Model diff +- Query diff +- Reading results +- Common failures + +## Choosing a mode + +| Situation | Use | +|---|---| +| You rebuilt models in a dev target and want to compare them with prod | Model diff `prod:dev` | +| You changed logic and want to see the effect before building anything | Query diff: current query vs proposed query | +| You fixed a bug and want to show exactly which rows changed | Query diff with `--key` and labels | +| Two tables or systems should agree (migration, reconciliation) | Query diff with `--key`, `--tolerance`, `--exclude-column` | +| You only care whether columns or types changed | Model diff `--schema-only` | + +## Model diff + +```bash +sqb diff : (--full | --schema-only | --bounded ) --select [flags] +``` + +- `FROM` and `TO` are configured target names (virtual environment names in virtual mode). The + `TO` target's connection runs the comparison and must be able to read both namespaces. +- `--select` is required. Exactly one mode is required. +- `--full` and `--bounded` join rows on the model's `unique_key`; models without one support only + `--schema-only`. +- `--bounded 14d` compares a recent cursor window (integer cursors take an integer). A model + without a cursor falls back to a full comparison. + +Typical dev-versus-prod check after changing a model: + +```bash +sqb build --select fact_orders --target dev --defer-to prod +sqb diff prod:dev --full --select fact_orders +``` + +Large models: + +- Deterministic key sampling caps the wide comparison: `--sample-rows 50000 --sample-seed 7`, + or the `row_diff_sample_rows` setting. `--exhaustive` disables sampling for one run. A sampled + pass means "no differences in the sampled keys", never full equality. +- `--max-models` and `--max-columns` fail loudly instead of truncating a broad selector. + +Model-level settings in the `MODEL()` header: + +```sql +MODEL ( + materialized table, + unique_key [order_id], + row_diff_exclude_columns [loaded_at], + row_diff_tolerances ( + by_column ( + total_revenue_cents (absolute 1), + ), + ), +); +``` + +## Query diff + +```bash +sqb diff \ + --left-query "" | --left-query-file \ + --right-query "" | --right-query-file \ + (--key [--key ...] | --unkeyed) \ + [--left-label ] [--right-label ] \ + [--exclude-column ] [--tolerance :absolute= | :relative=] \ + [--target ] [--json] [--verbose] +``` + +Rules that matter: + +- Give `--key` for the row identity (repeat for composite keys), or `--unkeyed` to compare exact + full-row multisets. One of them is required. +- Queries are raw warehouse SQL on the active target (or `--target`). `__ref()`, `__source()` and + macros are not expanded, so use physical names. Get them from `qualified_name` in + `sqb plan --json` or `sqb lineage --format json`. +- Both sides must produce the same column names and types. A type difference makes the result + `incomplete`. Cast both sides to a common type, for example `amount::DOUBLE`. +- SQLBuild materializes both queries as short-lived tables, compares them, and removes them. +- Labels make the report readable: `--left-label current --right-label proposed`. + +Recipes: + +```bash +# Effect of a logic change before building: current model output vs proposed SQL +sqb diff \ + --left-query "SELECT * FROM analytics.fact_orders" \ + --right-query-file /tmp/proposed_fact_orders.sql \ + --key order_id --left-label current --right-label proposed + +# Reconcile two tables, ignoring a load timestamp and float noise +sqb diff \ + --left-query "SELECT order_id, total::DOUBLE AS total, loaded_at FROM legacy.orders" \ + --right-query "SELECT order_id, total::DOUBLE AS total, loaded_at FROM analytics.orders" \ + --key order_id --exclude-column loaded_at --tolerance total:absolute=0.01 + +# No natural key: compare full rows +sqb diff --left-query "SELECT status FROM a" --right-query "SELECT status FROM b" --unkeyed +``` + +## Reading results + +- Exit code `0`: no differences. `1`: differences found. `2`: incomplete comparison or error. +- The summary shows schema differences, row counts per side, then equal, unequal, left-only and + right-only rows, followed by changed columns with match percentages and example values. +- `--verbose` shows more examples. `--json` (query diff) prints structured output to stdout. + `--json-output ` writes structured output while keeping the terminal summary. +- Report findings to the user as evidence: which keys changed, which columns, and why that is or + is not expected. + +## Common failures + +| Message | Fix | +|---|---| +| `raw-query diff requires one or more --key values or explicit --unkeyed` | Add `--key` or `--unkeyed` | +| `model diff requires FROM:TO` | Give `prod:dev`, or use `--left-query`/`--right-query` | +| Outcome `incomplete` with schema differences | Align column names and cast types on both sides | +| `Table Function with name __ref does not exist` | Replace `__ref()` with the physical relation name | +| Full/bounded diff rejected without `unique_key` | Add `unique_key` to the model or use `--schema-only` | + +Full reference: [docs/cli/diff.md](docs/cli/diff.md) and [docs/concepts/diff.md](docs/concepts/diff.md). diff --git a/src/sqlbuild/cli/commands/_helpers/skills/update.py b/src/sqlbuild/cli/commands/_helpers/skills/update.py index 210639fc9c..6f899e5441 100644 --- a/src/sqlbuild/cli/commands/_helpers/skills/update.py +++ b/src/sqlbuild/cli/commands/_helpers/skills/update.py @@ -23,7 +23,9 @@ generated_markers: tuple[str, ...] = (generated_marker, legacy_generated_marker) skill_name: str = "sqlbuild" skill_source_package: str = "sqlbuild" -skill_source_path: str = ".agents/skills/sqlbuild/SKILL.md" +skill_source_dir: str = ".agents/skills/sqlbuild" +skill_entry_file: str = "SKILL.md" +skill_file_suffix: str = ".md" valid_skill_targets: tuple[str, ...] = ("opencode", "claude", "agents") default_skill_targets: tuple[str, ...] = ("agents", "claude") opencode_skill_target: str = "opencode" @@ -50,11 +52,26 @@ def update_sqlbuild_skills( global_install=global_install, home_dir=home_dir, ) - source_content: str = load_packaged_skill_content() - generated_content: str = ensure_generated_marker(source_content) + expected_files: dict[str, str] = load_packaged_skill_files() + if not force: + collisions: list[Path] = [] + for install_target in install_targets: + collisions.extend( + _skill_collisions( + skill_dir=install_target.path.parent, expected_files=expected_files + ) + ) + if collisions: + raise CliUserError( + f"refusing to overwrite non-generated skill file: {collisions[0]}", + code="C806", + help="rerun with --force to replace it", + ) written_paths: list[Path] = [] for install_target in install_targets: - write_skill_file(path=install_target.path, content=generated_content, force=force) + _install_skill_directory( + skill_dir=install_target.path.parent, expected_files=expected_files, force=force + ) written_paths.append(install_target.path) return SkillUpdateResult(written_paths=tuple(written_paths)) @@ -167,25 +184,23 @@ def maintain_sqlbuild_skills(*, project_dir: Path) -> SkillMaintenanceResult: project_dir=project_dir, target_names=target_names, ) - expected_content: str = ensure_generated_marker(load_packaged_skill_content()) + expected_files: dict[str, str] = load_packaged_skill_files() stale_paths: list[Path] = [] collision_paths: list[Path] = [] for install_target in install_targets: - path: Path = install_target.path - if not path.exists(): - stale_paths.append(path) - continue - existing_content: str = path.read_text(encoding="utf-8") - if existing_content == expected_content: - continue - if not any(marker in existing_content for marker in generated_markers): - collision_paths.append(path) + skill_dir: Path = install_target.path.parent + collisions: tuple[Path, ...] = _skill_collisions( + skill_dir=skill_dir, expected_files=expected_files + ) + if collisions: + collision_paths.extend(collisions) continue - stale_paths.append(path) + if _skill_directory_is_stale(skill_dir=skill_dir, expected_files=expected_files): + stale_paths.append(install_target.path) if settings.auto_update and stale_paths: for path in stale_paths: - write_skill_file(path=path, content=expected_content) + _install_skill_directory(skill_dir=path.parent, expected_files=expected_files) message: str = "Updated stale SQLBuild skill files:\n" + "".join( f" {path}\n" for path in stale_paths ) @@ -253,7 +268,7 @@ def _find_git_root(*, project_dir: Path) -> Path | None: def load_packaged_skill_content() -> str: - skill_file: Traversable = files(skill_source_package).joinpath(*skill_source_path.split("/")) + skill_file: Traversable = _packaged_skill_root().joinpath(skill_entry_file) if not skill_file.is_file(): raise CliUserError( "packaged SQLBuild skill is missing", @@ -297,3 +312,85 @@ def write_skill_file(*, path: Path, content: str, force: bool = False) -> None: _ = temporary_file.write(content) temporary_path: Path = Path(temporary_file.name) replace(temporary_path, path) + + +def load_packaged_skill_files() -> dict[str, str]: + """Return every packaged skill file keyed by skill-relative path, with generated markers.""" + + entry_content: str = load_packaged_skill_content() + collected: dict[str, str] = _collect_skill_files(node=_packaged_skill_root(), prefix="") + collected[skill_entry_file] = entry_content + return { + relative_path: ensure_generated_marker(content) + for relative_path, content in sorted(collected.items()) + } + + +def _packaged_skill_root() -> Traversable: + return files(skill_source_package).joinpath(*skill_source_dir.split("/")) + + +def _collect_skill_files(*, node: Traversable, prefix: str) -> dict[str, str]: + collected: dict[str, str] = {} + for child in node.iterdir(): + if child.name.startswith((".", "__")): + continue + if child.is_dir(): + collected.update(_collect_skill_files(node=child, prefix=f"{prefix}{child.name}/")) + elif child.name.endswith(skill_file_suffix): + collected[f"{prefix}{child.name}"] = child.read_text(encoding="utf-8") + return collected + + +def _is_generated(path: Path) -> bool: + content: str = path.read_text(encoding="utf-8") + return any(marker in content for marker in generated_markers) + + +def _skill_collisions(*, skill_dir: Path, expected_files: dict[str, str]) -> tuple[Path, ...]: + """Return expected install paths occupied by files SQLBuild did not generate.""" + + return tuple( + skill_dir / relative_path + for relative_path in expected_files + if (skill_dir / relative_path).is_file() and not _is_generated(skill_dir / relative_path) + ) + + +def _stale_generated_files(*, skill_dir: Path, expected_files: dict[str, str]) -> tuple[Path, ...]: + if not skill_dir.is_dir(): + return () + return tuple( + path + for path in sorted(skill_dir.rglob(f"*{skill_file_suffix}")) + if path.relative_to(skill_dir).as_posix() not in expected_files and _is_generated(path) + ) + + +def _skill_directory_is_stale(*, skill_dir: Path, expected_files: dict[str, str]) -> bool: + for relative_path, content in expected_files.items(): + path: Path = skill_dir / relative_path + if not path.is_file() or path.read_text(encoding="utf-8") != content: + return True + return bool(_stale_generated_files(skill_dir=skill_dir, expected_files=expected_files)) + + +def _install_skill_directory( + *, skill_dir: Path, expected_files: dict[str, str], force: bool = False +) -> None: + """Write all packaged skill files and remove generated files no longer shipped.""" + + for relative_path, content in expected_files.items(): + path: Path = skill_dir / relative_path + if path.is_file() and path.read_text(encoding="utf-8") == content: + continue + write_skill_file(path=path, content=content, force=force) + for stale_path in _stale_generated_files(skill_dir=skill_dir, expected_files=expected_files): + stale_path.unlink() + for directory in sorted( + (path for path in skill_dir.rglob("*") if path.is_dir()), + key=lambda path: len(path.parts), + reverse=True, + ): + if not any(directory.iterdir()): + directory.rmdir() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/skills/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/skills/_test_types.py index 3e78d72576..97f0d4edc2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/skills/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/skills/_test_types.py @@ -9,7 +9,7 @@ class SkillsCliTestCase: expected_exit_code: int expected_files: tuple[Path, ...] unexpected_files: tuple[Path, ...] = () - expected_content_fragment: str = "# SQLBuild Skill" + expected_content_fragment: str = "## Reach for these tools" @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py b/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py index 099815b93b..e061ca7758 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py @@ -79,7 +79,7 @@ def test_given_skills_update_cli_when_running_then_writes_expected_skill_files( existing_file=Path(".opencode/skills/sqlbuild/SKILL.md"), existing_content="custom project instructions\n", expected_exit_code=0, - expected_content_fragment="# SQLBuild Skill", + expected_content_fragment="## Reach for these tools", ), ], ids=lambda case: case.description, diff --git a/tests/unit/scripts/docs_skill/_helpers/_test_types.py b/tests/unit/scripts/docs_skill/_helpers/_test_types.py index fd406569c1..135fb2e584 100644 --- a/tests/unit/scripts/docs_skill/_helpers/_test_types.py +++ b/tests/unit/scripts/docs_skill/_helpers/_test_types.py @@ -7,3 +7,13 @@ class OrderedNavigationPagesTestCase: docs_json: str page_paths: tuple[str, ...] expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class ReferencePagesTestCase: + description: str + docs_json: str + pages: dict[str, str] + stale_page: str + expected_files: tuple[str, ...] + expected_fragments: tuple[tuple[str, str], ...] diff --git a/tests/unit/scripts/docs_skill/_helpers/test_render.py b/tests/unit/scripts/docs_skill/_helpers/test_render.py new file mode 100644 index 0000000000..079fa6327b --- /dev/null +++ b/tests/unit/scripts/docs_skill/_helpers/test_render.py @@ -0,0 +1,84 @@ +from pathlib import Path + +import pytest + +from scripts.docs_skill._helpers.output import write_reference_pages +from scripts.docs_skill._helpers.render import build_reference_pages +from scripts.docs_skill.constants import GENERATED_MARKER +from tests.unit.scripts.docs_skill._helpers._test_types import ReferencePagesTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + ReferencePagesTestCase( + description="pages become files with relative links and a grouped index", + docs_json=( + '{"navigation":{"groups":[' + '{"group":"Concepts","pages":["concepts/diff","concepts/models"]},' + '{"group":"CLI Reference","pages":["cli/diff"]}]}}' + ), + pages={ + "concepts/diff.mdx": ( + '---\ntitle: "Data Diffs"\ndescription: "Compare data."\n---\n\n' + "See [models](/concepts/models#schemas), [the CLI](/cli/diff) and " + "[elsewhere](/missing/page).\n\n\nkept text\n\n" + ), + "concepts/models.mdx": '---\ntitle: "Models"\n---\n\n## Schemas\n\nBody.\n', + "cli/diff.mdx": '---\ntitle: "diff"\ndescription: "Diff command."\n---\n\nUsage.\n', + }, + stale_page="concepts/retired.md", + expected_files=( + "INDEX.md", + "cli/diff.md", + "concepts/diff.md", + "concepts/models.md", + ), + expected_fragments=( + ("concepts/diff.md", GENERATED_MARKER), + ("concepts/diff.md", "# Data Diffs"), + ("concepts/diff.md", "> Compare data."), + ("concepts/diff.md", "Online: https://docs.sqlbuild.com/concepts/diff"), + ("concepts/diff.md", "[models](models.md#schemas)"), + ("concepts/diff.md", "[the CLI](../cli/diff.md)"), + ("concepts/diff.md", "[elsewhere](https://docs.sqlbuild.com/missing/page)"), + ("concepts/diff.md", "kept text"), + ("INDEX.md", "## Concepts"), + ("INDEX.md", "- [Data Diffs](concepts/diff.md) (`concepts/diff`) - Compare data."), + ("INDEX.md", "## CLI Reference"), + ("INDEX.md", "- [diff](cli/diff.md) (`cli/diff`) - Diff command."), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_docs_pages_when_writing_references_then_each_page_is_a_linked_file( + test_case: ReferencePagesTestCase, + tmp_path: Path, +) -> None: + docs_root: Path = tmp_path / "docs" + output_dir: Path = tmp_path / "references" / "docs" + (docs_root).mkdir() + (docs_root / "docs.json").write_text(test_case.docs_json, encoding="utf-8") + for relative_path, contents in test_case.pages.items(): + (docs_root / relative_path).parent.mkdir(parents=True, exist_ok=True) + (docs_root / relative_path).write_text(contents, encoding="utf-8") + (output_dir / test_case.stale_page).parent.mkdir(parents=True, exist_ok=True) + (output_dir / test_case.stale_page).write_text(f"{GENERATED_MARKER}\nold\n", encoding="utf-8") + + write_reference_pages(output_dir=output_dir, pages=build_reference_pages(docs_root=docs_root)) + + written: dict[str, str] = { + path.relative_to(output_dir).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(output_dir.rglob("*.md")) + } + assert tuple(written) == test_case.expected_files + found: dict[tuple[str, str], bool] = { + (name, fragment): fragment in written[name] + for name, fragment in test_case.expected_fragments + } + assert found == dict.fromkeys(test_case.expected_fragments, True) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/_test_types.py index 66b52f30ab..6e20b095af 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/_test_types.py @@ -11,7 +11,7 @@ class SkillUpdateTestCase: global_install: bool = False force: bool = False expected_written_paths: tuple[Path, ...] = () - expected_content_fragment: str = "# SQLBuild Skill" + expected_content_fragment: str = "## Reach for these tools" project_path: Path = Path(".") git_marker_is_file: bool | None = None @@ -33,3 +33,36 @@ class SkillMaintenanceTestCase: expected_written_paths: tuple[Path, ...] = () project_path: Path = Path(".") git_marker_is_file: bool | None = None + + +@dataclass(frozen=True) +class SkillDirectoryInstallTestCase: + description: str + existing_files: dict[Path, str] = field(default_factory=dict) + expected_absent_paths: tuple[Path, ...] = () + expected_preserved_files: dict[Path, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class SkillDirectoryCollisionTestCase: + description: str + existing_files: dict[Path, str] + expected_error_fragment: str + expected_unwritten_path: Path + + +@dataclass(frozen=True) +class SkillDirectoryMaintenanceTestCase: + description: str + project_config: str + stale_relative_path: str + expected_message_fragment: str + expected_restored: bool + + +@dataclass(frozen=True) +class SkillContentTestCase: + description: str + max_entry_lines: int + expected_broken_links: frozenset[str] = frozenset() + expected_unknown_commands: frozenset[str] = frozenset() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py index b9fc53529e..2f8ccf6e4a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py @@ -1,6 +1,10 @@ from __future__ import annotations -from pathlib import Path +import re +from pathlib import Path, PurePosixPath + +from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills +from sqlbuild.cli.output.models import SkillUpdateResult def write_project_files(*, project_dir: Path, files: dict[Path, str]) -> None: @@ -45,3 +49,31 @@ def write_git_marker(*, repository_dir: Path, marker_is_file: bool | None) -> No {None: _skip_git_marker, False: _write_git_directory, True: _write_git_file}[marker_is_file]( repository_dir=repository_dir ) + + +def install_packaged_skill(*, project_dir: Path, target: str) -> Path: + """Install the packaged skill for one target and return its skill directory.""" + + result: SkillUpdateResult = update_sqlbuild_skills( + project_dir=project_dir, requested_targets=(target,) + ) + return result.written_paths[0].parent + + +def installed_skill_files(*, skill_dir: Path) -> dict[str, str]: + return { + path.relative_to(skill_dir).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(skill_dir.rglob("*.md")) + } + + +def relative_markdown_links(*, text: str, base: str) -> frozenset[str]: + """Return skill-relative targets of non-URL Markdown links in one skill file.""" + + return frozenset( + str(PurePosixPath(base, link)) for link in re.findall(r"\]\(((?!https?:)[^)#\s]+)", text) + ) + + +def mentioned_sqb_commands(*, text: str) -> frozenset[str]: + return frozenset(re.findall(r"(?:^|[`\s(])sqb ([a-z][a-z-]*)", text, flags=re.MULTILINE)) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py new file mode 100644 index 0000000000..f35fd27731 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import fnmatch +from pathlib import Path, PurePosixPath + +import pytest + +from sqlbuild.cli.commands._helpers.skills.update import ( + generated_marker, + load_packaged_skill_files, + maintain_sqlbuild_skills, + skill_entry_file, + update_sqlbuild_skills, +) +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.types import CliCommand +from sqlbuild.cli.output.models import SkillMaintenanceResult +from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( + SkillContentTestCase, + SkillDirectoryCollisionTestCase, + SkillDirectoryInstallTestCase, + SkillDirectoryMaintenanceTestCase, +) +from tests.unit.src.sqlbuild.cli.commands.main.skills.helpers import ( + install_packaged_skill, + installed_skill_files, + mentioned_sqb_commands, + relative_markdown_links, + write_project_files, +) + +_SKILL_DIR: Path = Path(".agents/skills/sqlbuild") + + +@pytest.mark.parametrize( + "test_case", + [ + SkillDirectoryInstallTestCase( + description="installs bundled guides and docs pages alongside SKILL.md", + ), + SkillDirectoryInstallTestCase( + description="removes generated files that are no longer shipped", + existing_files={ + _SKILL_DIR / "references/docs/retired_page.md": f"{generated_marker}\nold\n", + }, + expected_absent_paths=(_SKILL_DIR / "references/docs/retired_page.md",), + ), + SkillDirectoryInstallTestCase( + description="keeps user files that SQLBuild did not generate", + existing_files={_SKILL_DIR / "references/team-notes.md": "our conventions\n"}, + expected_preserved_files={_SKILL_DIR / "references/team-notes.md": "our conventions\n"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_skill_directory_when_updating_then_it_matches_the_package( + test_case: SkillDirectoryInstallTestCase, + tmp_path: Path, +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.existing_files) + + skill_dir: Path = install_packaged_skill(project_dir=tmp_path, target="agents") + + installed: dict[str, str] = installed_skill_files(skill_dir=skill_dir) + packaged: dict[str, str] = load_packaged_skill_files() + assert {path: installed[path] for path in packaged} == packaged + assert "references/docs/INDEX.md" in packaged + assert not any((tmp_path / path).exists() for path in test_case.expected_absent_paths) + assert { + path: (tmp_path / path).read_text(encoding="utf-8") + for path in test_case.expected_preserved_files + } == test_case.expected_preserved_files + + +@pytest.mark.parametrize( + "test_case", + [ + SkillDirectoryCollisionTestCase( + description="custom reference file blocks the whole install", + existing_files={_SKILL_DIR / "references/testing.md": "custom testing notes\n"}, + expected_error_fragment="references/testing.md", + expected_unwritten_path=_SKILL_DIR / skill_entry_file, + ), + ], + ids=lambda case: case.description, +) +def test_given_custom_file_in_skill_directory_when_updating_then_nothing_is_written( + test_case: SkillDirectoryCollisionTestCase, + tmp_path: Path, +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.existing_files) + + with pytest.raises(CliUserError, match=test_case.expected_error_fragment): + update_sqlbuild_skills(project_dir=tmp_path, requested_targets=("agents",)) + + assert not (tmp_path / test_case.expected_unwritten_path).exists() + assert { + path: (tmp_path / path).read_text(encoding="utf-8") for path in test_case.existing_files + } == test_case.existing_files + + +@pytest.mark.parametrize( + "test_case", + [ + SkillDirectoryMaintenanceTestCase( + description="stale bundled page is reported", + project_config='name = "demo"\nadapter = "duckdb"\n\n[skills]\ntargets = ["agents"]\n', + stale_relative_path="references/docs/INDEX.md", + expected_message_fragment="SQLBuild skill files are out of date", + expected_restored=False, + ), + SkillDirectoryMaintenanceTestCase( + description="stale bundled page is restored with auto update", + project_config=( + 'name = "demo"\nadapter = "duckdb"\n\n[skills]\n' + 'targets = ["agents"]\nauto_update = true\n' + ), + stale_relative_path="references/docs/INDEX.md", + expected_message_fragment="Updated stale SQLBuild skill files", + expected_restored=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_stale_reference_page_when_maintaining_then_directory_is_checked( + test_case: SkillDirectoryMaintenanceTestCase, + tmp_path: Path, +) -> None: + write_project_files( + project_dir=tmp_path, files={Path("sqlbuild_project.toml"): test_case.project_config} + ) + skill_dir: Path = install_packaged_skill(project_dir=tmp_path, target="agents") + stale_path: Path = skill_dir / test_case.stale_relative_path + stale_path.write_text(f"{generated_marker}\nold\n", encoding="utf-8") + + result: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=tmp_path) + + assert test_case.expected_message_fragment in result.message + assert ( + stale_path.read_text(encoding="utf-8") + == load_packaged_skill_files()[test_case.stale_relative_path] + ) is test_case.expected_restored + + +@pytest.mark.parametrize( + "test_case", + [ + SkillContentTestCase( + description="entry stays short and every link and command resolves", + max_entry_lines=500, + ), + ], + ids=lambda case: case.description, +) +def test_given_packaged_skill_when_reading_then_entry_is_short_and_references_resolve( + test_case: SkillContentTestCase, +) -> None: + packaged: dict[str, str] = load_packaged_skill_files() + authored_paths: set[str] = set(packaged) - set(fnmatch.filter(packaged, "references/docs/*")) + authored: dict[str, str] = {path: packaged[path] for path in authored_paths} + + links: frozenset[str] = frozenset().union( + *( + relative_markdown_links(text=text, base=str(PurePosixPath(path).parent)) + for path, text in authored.items() + ) + ) + commands: frozenset[str] = frozenset().union( + *(mentioned_sqb_commands(text=text) for text in authored.values()) + ) + + assert packaged[skill_entry_file].count("\n") <= test_case.max_entry_lines + assert links - packaged.keys() == test_case.expected_broken_links + assert commands - {command.value for command in CliCommand} == ( + test_case.expected_unknown_commands + ) From 31562125e82fd86c795f65479ab3ad3088da5ca9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:56:56 +0100 Subject: [PATCH 2/3] perf: plan Snowflake retention from the warehouse snapshot and batch build-time inspection --- .../contract/classes/retention_adapter.py | 19 +++ src/sqlbuild/adapter/contract/models.py | 1 + .../snowflake/classes/snowflake_adapter.py | 81 +++++++++++- .../planner/_helpers/planning/retention.py | 11 +- .../executor/build/_helpers/retention.py | 120 ++++++++++++++++-- .../adapters/snowflake/_test_types.py | 17 +++ .../adapters/snowflake/test_client.py | 2 + .../adapters/snowflake/test_retention.py | 93 ++++++++++++++ .../planner/_helpers/planning/_test_types.py | 9 ++ .../planner/_helpers/planning/helpers.py | 97 +++++++++----- .../_helpers/planning/test_retention.py | 78 ++++++++++-- .../executor/build/_helpers/_test_types.py | 11 ++ .../executor/build/_helpers/helpers.py | 26 ++++ .../executor/build/_helpers/test_retention.py | 117 +++++++++++++++-- 14 files changed, 606 insertions(+), 76 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/retention_adapter.py b/src/sqlbuild/adapter/contract/classes/retention_adapter.py index d23db19bc1..d2ed84ac26 100644 --- a/src/sqlbuild/adapter/contract/classes/retention_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/retention_adapter.py @@ -6,6 +6,7 @@ from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( + RelationInfo, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -23,6 +24,24 @@ def inspect_retention(self, *, connection: Any, request: RetentionRequest) -> Re message=f"adapter '{self.adapter_name}' does not support retention inspection" ) + def inspect_retentions( + self, *, connection: Any, requests: tuple[RetentionRequest, ...] + ) -> dict[str, RetentionState]: + """Inspect several requests; adapters may override with a batched implementation.""" + + return { + request.request_id: self.inspect_retention(connection=connection, request=request) + for request in requests + } + + def retention_state_from_relation( + self, *, request: RetentionRequest, relation: RelationInfo + ) -> RetentionState | None: + """Return retention state already carried by a listed relation, when available.""" + + del request, relation + return None + def render_retention_changes( self, *, request: RetentionRequest, state: RetentionState | None = None ) -> tuple[RenderedRetentionChange, ...]: diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 05235adcc8..cc9aead38d 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -80,6 +80,7 @@ class RelationInfo: created_at: datetime | None = None last_altered_at: datetime | None = None is_transient: bool | None = None + retention_days: int | None = None @property def identity(self) -> tuple[str | None, str | None, str]: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 7bd0f37264..cbf3550519 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -137,13 +137,84 @@ def inspect_retention(self, *, connection: Any, request: RetentionRequest) -> Re raise AdapterUserError( message=f"Snowflake retention metadata not found for {request.name}" ) - effective_days: int = int(row[0]) - is_transient: bool = str(row[1]).upper() == TRUE_METADATA_VALUE + return self._relation_retention_state( + request=request, retention_time=row[0], is_transient_value=row[1] + ) + + def inspect_retentions( + self, *, connection: Any, requests: tuple[RetentionRequest, ...] + ) -> dict[str, RetentionState]: + """Inspect relation retention with one metadata query per database and schema.""" + + grouped: dict[tuple[str | None, str], list[RetentionRequest]] = {} + for request in requests: + self._validate_relation_retention_request(request=request) + grouped.setdefault((request.database, request.schema), []).append(request) + states: dict[str, RetentionState] = {} + for (database, schema), group in grouped.items(): + names: tuple[str, ...] = tuple( + dict.fromkeys( + self._information_schema_identifier(str(request.name)) for request in group + ) + ) + clauses: list[str] = [ + f"table_name IN ({', '.join(['%s'] * len(names))})", + "table_schema = %s", + ] + params: list[str] = [*names, self._information_schema_identifier(schema)] + if database is not None: + clauses.append("table_catalog = %s") + params.append(self._information_schema_identifier(database)) + cursor: Any = connection.cursor() + try: + cursor.execute( + "SELECT table_name, retention_time, is_transient FROM " + + self._information_schema_relation(database=database, name="tables") + + " WHERE " + + " AND ".join(clauses), + tuple(params), + ) + rows: list[tuple[Any, ...]] = cursor.fetchall() + finally: + cursor.close() + rows_by_name: dict[str, tuple[Any, ...]] = {str(row[0]): row for row in rows} + for request in group: + row: tuple[Any, ...] | None = rows_by_name.get( + self._information_schema_identifier(str(request.name)) + ) + if row is None: + raise AdapterUserError( + message=f"Snowflake retention metadata not found for {request.name}" + ) + states[request.request_id] = self._relation_retention_state( + request=request, retention_time=row[1], is_transient_value=row[2] + ) + return states + + def retention_state_from_relation( + self, *, request: RetentionRequest, relation: RelationInfo + ) -> RetentionState | None: + if relation.retention_days is None or relation.is_transient is None: + return None + return RetentionState( + request_id=request.request_id, + scope=request.scope, + configured_days=None, + effective_days=relation.retention_days, + relation_kind="TRANSIENT" if relation.is_transient else "PERMANENT", + is_transient=relation.is_transient, + ) + + @staticmethod + def _relation_retention_state( + *, request: RetentionRequest, retention_time: Any, is_transient_value: Any + ) -> RetentionState: + is_transient: bool = str(is_transient_value).upper() == TRUE_METADATA_VALUE return RetentionState( request_id=request.request_id, scope=request.scope, configured_days=None, - effective_days=effective_days, + effective_days=int(retention_time), relation_kind="TRANSIENT" if is_transient else "PERMANENT", is_transient=is_transient, ) @@ -1706,7 +1777,8 @@ def list_relations( names: tuple[str, ...] | None = None, ) -> tuple[Any, ...]: query: str = ( - "SELECT table_name, table_schema, table_type, is_transient, created, last_altered " + "SELECT table_name, table_schema, table_type, is_transient, created, last_altered, " + "retention_time " f"FROM {self._information_schema_relation(database=database, name='tables')} WHERE 1=1" ) params: list[str] = [] @@ -1740,6 +1812,7 @@ def list_relations( ), created_at=None if row[4] is None else row[4], last_altered_at=None if row[5] is None else row[5], + retention_days=None if row[6] is None else int(row[6]), ) for row in rows ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 0b6e5ee402..f919a43a9b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -188,12 +188,11 @@ def _plan_relation_retention( statements=_flatten_changes(changes=changes), ), ) - relation_is_transient: bool | None = warehouse.snapshot.existing_relations[ - model.name - ].is_transient - state: RetentionState = runtime.adapter.inspect_retention( - connection=runtime.connection, request=request - ) + relation: RelationInfo = warehouse.snapshot.existing_relations[model.name] + relation_is_transient: bool | None = relation.is_transient + state: RetentionState = runtime.adapter.retention_state_from_relation( + request=request, relation=relation + ) or runtime.adapter.inspect_retention(connection=runtime.connection, request=request) if ( state.is_transient and desired_days > 1 diff --git a/src/sqlbuild/executor/build/_helpers/retention.py b/src/sqlbuild/executor/build/_helpers/retention.py index b93639fd0b..b5a4d06249 100644 --- a/src/sqlbuild/executor/build/_helpers/retention.py +++ b/src/sqlbuild/executor/build/_helpers/retention.py @@ -5,7 +5,12 @@ from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo, RenderedRetentionChange, RetentionState +from sqlbuild.adapter.contract.models import ( + RelationInfo, + RenderedRetentionChange, + RetentionRequest, + RetentionState, +) from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, @@ -178,13 +183,17 @@ def reconcile_model_retention( ) -> None: """Reconcile a successfully materialized relation before item completion.""" + entries: tuple[RetentionPlanEntry, ...] = tuple( + entry + for entry in plan.retention_entries + if model_name in entry.model_names and entry.request.scope == RetentionScope.RELATION + ) + states: _RetentionStates = _RetentionStates.inspect( + adapter=adapter, connection=connection, entries=entries + ) entry: RetentionPlanEntry - for entry in plan.retention_entries: - if model_name not in entry.model_names or entry.request.scope != RetentionScope.RELATION: - continue - state: RetentionState = _inspect_retention( - adapter=adapter, connection=connection, entry=entry - ) + for entry in entries: + state: RetentionState = states.current(entry=entry) if _state_matches(entry=entry, state=state): continue changes: tuple[RenderedRetentionChange, ...] = adapter.render_retention_changes( @@ -194,6 +203,7 @@ def reconcile_model_retention( for change in changes: if not _safe_before_build_success(entry=entry, state=state, change=change): continue + states.mark_changed(entry=entry, statements=change.statements) _apply_retention_statements( adapter=adapter, connection=connection, @@ -212,12 +222,14 @@ def reconcile_retention_after_build( for model_entry in plan.model_entries if materialization_recreates_relation(model_entry) ) - for entry in plan.retention_entries: - if entry.phase == RetentionPlanPhase.NONE: - continue - state: RetentionState = _inspect_retention( - adapter=adapter, connection=connection, entry=entry - ) + entries: tuple[RetentionPlanEntry, ...] = tuple( + entry for entry in plan.retention_entries if entry.phase != RetentionPlanPhase.NONE + ) + states: _RetentionStates = _RetentionStates.inspect( + adapter=adapter, connection=connection, entries=entries + ) + for entry in entries: + state: RetentionState = states.current(entry=entry) if _state_matches(entry=entry, state=state): continue changes: tuple[RenderedRetentionChange, ...] = adapter.render_retention_changes( @@ -230,6 +242,7 @@ def reconcile_retention_after_build( for change in changes: if not lowering_permitted and change.phase != RetentionChangePhase.PREPARE: continue + states.mark_changed(entry=entry, statements=change.statements) _apply_retention_statements( adapter=adapter, connection=connection, @@ -291,6 +304,87 @@ def _state_matches(*, entry: RetentionPlanEntry, state: RetentionState) -> bool: return all(value == desired_days for value in values) +class _RetentionStates: + """Batched retention states, re-inspected per request once this build has changed it.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + connection: Any, + states: dict[str, RetentionState], + ) -> None: + self._adapter: BaseAdapter = adapter + self._connection: Any = connection + self._states: dict[str, RetentionState] = states + self._changed: set[str] = set() + + @classmethod + def inspect( + cls, + *, + adapter: BaseAdapter, + connection: Any, + entries: tuple[RetentionPlanEntry, ...], + ) -> _RetentionStates: + return cls( + adapter=adapter, + connection=connection, + states=_inspect_retentions(adapter=adapter, connection=connection, entries=entries), + ) + + def current(self, *, entry: RetentionPlanEntry) -> RetentionState: + request_id: str = entry.request.request_id + if request_id in self._changed or request_id not in self._states: + self._states[request_id] = _inspect_retention( + adapter=self._adapter, connection=self._connection, entry=entry + ) + self._changed.discard(request_id) + return self._states[request_id] + + def mark_changed(self, *, entry: RetentionPlanEntry, statements: tuple[str, ...]) -> None: + if statements: + self._changed.add(entry.request.request_id) + + +def _inspect_retentions( + *, adapter: BaseAdapter, connection: Any, entries: tuple[RetentionPlanEntry, ...] +) -> dict[str, RetentionState]: + """Inspect every distinct request in one adapter call per retention scope.""" + + requests_by_scope: dict[RetentionScope, dict[str, RetentionRequest]] = {} + for entry in entries: + requests_by_scope.setdefault(entry.request.scope, {})[entry.request.request_id] = ( + entry.request + ) + states: dict[str, RetentionState] = {} + for scope, requests in requests_by_scope.items(): + with OperationLifecycle( + operation_kind="warehouse", + operation_name="retention_inspection", + attributes=OperationAttributes( + phase="inspect", + adapter=canonicalize_operation_adapter(adapter.adapter_name), + target_kind=scope.value, + ), + ) as lifecycle: + inspected: dict[str, RetentionState] = adapter.inspect_retentions( + connection=connection, requests=tuple(requests.values()) + ) + lifecycle.completed( + metadata={ + "item_count": len(inspected), + "changed_count": sum( + not _state_matches(entry=entry, state=inspected[entry.request.request_id]) + for entry in entries + if entry.request.request_id in inspected + ), + } + ) + states.update(inspected) + return states + + def _inspect_retention( *, adapter: BaseAdapter, connection: Any, entry: RetentionPlanEntry ) -> RetentionState: diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 167d1860f3..103623818c 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -177,3 +177,20 @@ class SnowflakeQualifiedColumnInspectionTestCase: description: str relation_count: int expected_statement_count: int + + +@dataclass(frozen=True) +class SnowflakeBatchedRetentionTestCase: + description: str + requests: tuple[tuple[str, str], ...] + schema_rows: tuple[list[tuple[object, ...]], ...] + expected_query_schemas: tuple[str, ...] + expected_days: dict[str, int] + + +@dataclass(frozen=True) +class SnowflakeBatchedRetentionErrorTestCase: + description: str + requests: tuple[tuple[str, str], ...] + rows: list[tuple[object, ...]] + expected_error: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 4014ff176d..54bcbf4ef5 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -379,6 +379,7 @@ def test_given_lowercase_schema_when_listing_relations_then_uppercases_filter_bi "YES", datetime(2026, 8, 1, tzinfo=UTC), datetime(2026, 8, 2, tzinfo=UTC), + 1, ) ] ) @@ -398,6 +399,7 @@ def test_given_lowercase_schema_when_listing_relations_then_uppercases_filter_bi assert len(relations) == 1 assert relations[0].schema == "staging" assert relations[0].name == "commerce__stg_order" + assert relations[0].retention_days == 1 assert relations[0].is_transient is True assert relations[0].created_at == datetime(2026, 8, 1, tzinfo=UTC) assert relations[0].last_altered_at == datetime(2026, 8, 2, tzinfo=UTC) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py index c0092e929d..1bee1c90e3 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py @@ -2,6 +2,7 @@ import pytest +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, RetentionRequest, @@ -10,11 +11,14 @@ from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeBatchedRetentionErrorTestCase, + SnowflakeBatchedRetentionTestCase, SnowflakeRetentionTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( FakeSnowflakeMetadataConnection, FakeSnowflakeMetadataCursor, + FakeSnowflakeMetadataSequenceConnection, ) @@ -59,3 +63,92 @@ def test_given_snowflake_relation_when_managing_retention_then_observes_and_rend assert changes[0].statements == (test_case.expected_sql,) assert cursor.executed_sql is not None assert "retention_time, is_transient" in cursor.executed_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeBatchedRetentionTestCase( + description="three tables in two schemas take one metadata query per schema", + requests=(("mart", "orders"), ("mart", "customers"), ("staging", "stg_orders")), + schema_rows=( + [("ORDERS", 7, "NO"), ("CUSTOMERS", 1, "NO")], + [("STG_ORDERS", 0, "YES")], + ), + expected_query_schemas=("MART", "STAGING"), + expected_days={"orders": 7, "customers": 1, "stg_orders": 0}, + ), + ], + ids=lambda case: case.description, +) +def test_given_many_relations_when_inspecting_retention_then_queries_once_per_schema( + test_case: SnowflakeBatchedRetentionTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursors: tuple[FakeSnowflakeMetadataCursor, ...] = tuple( + FakeSnowflakeMetadataCursor(rows=rows) for rows in test_case.schema_rows + ) + connection: FakeSnowflakeMetadataSequenceConnection = FakeSnowflakeMetadataSequenceConnection( + cursors + ) + requests: tuple[RetentionRequest, ...] = tuple( + RetentionRequest( + request_id=name, + scope=RetentionScope.RELATION, + database="analytics", + schema=schema, + name=name, + desired_days=7, + ) + for schema, name in test_case.requests + ) + + states: dict[str, RetentionState] = adapter.inspect_retentions( + connection=cast(Any, connection), requests=requests + ) + + assert ( + tuple( + cast(tuple[object, ...], cursor.executed_params)[-2] + for cursor in connection.returned_cursors + ) + == test_case.expected_query_schemas + ) + assert {request_id: state.effective_days for request_id, state in states.items()} == ( + test_case.expected_days + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeBatchedRetentionErrorTestCase( + description="a table missing from metadata fails like single inspection", + requests=(("mart", "orders"), ("mart", "customers")), + rows=[("ORDERS", 7, "NO")], + expected_error="Snowflake retention metadata not found for customers", + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_relation_when_batch_inspecting_retention_then_raises( + test_case: SnowflakeBatchedRetentionErrorTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: FakeSnowflakeMetadataConnection = FakeSnowflakeMetadataConnection( + FakeSnowflakeMetadataCursor(rows=test_case.rows) + ) + requests: tuple[RetentionRequest, ...] = tuple( + RetentionRequest( + request_id=name, + scope=RetentionScope.RELATION, + database="analytics", + schema=schema, + name=name, + desired_days=7, + ) + for schema, name in test_case.requests + ) + + with pytest.raises(AdapterUserError, match=test_case.expected_error): + adapter.inspect_retentions(connection=cast(Any, connection), requests=requests) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 60760dad2a..66e13a164d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -45,3 +45,12 @@ class ReusableExecutionModelNamesTestCase: stale_warning_hash: str query_change_tracking: bool expected_names: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotRetentionPlanningTestCase: + description: str + model_names: tuple[str, ...] + live_days: int + desired_days: int + expected_directions: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index aad04ebd3d..a47530c4a0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -2,7 +2,9 @@ from pathlib import Path from typing import Any +from unittest.mock import Mock +from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -40,47 +42,32 @@ def build_retention_planner_inputs( config_values: dict[str, object], table_type: ResolvedTableType | None = None, table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION, + model_names: tuple[str, ...] = ("orders",), ) -> tuple[PlannerRuntime, PlannerWarehouseState, PlannerScope]: - key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.MODEL, - name="orders", - ) - model: CompiledModel = CompiledModel( - key=key, - deps=(), - name="orders", - relative_path=Path("models/orders.sql"), - query_sql="SELECT 1 AS order_id", - config=CompileModelConfig( - values=config_values, - time_travel_retention=ResolvedTimeTravelRetention( - desired_days=desired_days, - unmanaged=False, - source=TimeTravelRetentionSource.MODEL, - ), - table_type=table_type or ResolvedTableType(), - ), - destination=CompiledRelationLocation( - database="warehouse", - schema="analytics", - name="orders", - qualified_name="warehouse.analytics.orders", - ), + models: tuple[CompiledModel, ...] = tuple( + _retention_model( + name=name, + desired_days=desired_days, + config_values=config_values, + table_type=table_type, + ) + for name in model_names ) + keys: tuple[CompiledObjectKey, ...] = tuple(model.key for model in models) project: CompiledProject = CompiledProject( run_id="run-1", effective_target_name="test", effective_connection={}, effective_vars={}, - models=(model,), + models=models, ) scope: PlannerScope = PlannerScope( - upstream_deps={key: ()}, - downstream_deps={key: ()}, - all_keys={model.name: key}, - models_by_name={model.name: model}, - selected_keys=frozenset({key}), - execution_order=(key,), + upstream_deps=dict.fromkeys(keys, ()), + downstream_deps=dict.fromkeys(keys, ()), + all_keys={model.name: model.key for model in models}, + models_by_name={model.name: model for model in models}, + selected_keys=frozenset(keys), + execution_order=keys, ) snapshot: WarehouseSnapshot = WarehouseSnapshot(existing_relations=existing_relations) warehouse: PlannerWarehouseState = PlannerWarehouseState( @@ -110,3 +97,49 @@ def build_retention_planner_inputs( warehouse, scope, ) + + +def retention_mock_adapter(**attributes: Any) -> Mock: + """Return a Mock adapter whose batched retention hooks use the real default implementations.""" + + adapter: Mock = Mock(**attributes) + adapter.inspect_retentions.side_effect = lambda *, connection, requests: ( + RetentionAdapterMixin.inspect_retentions(adapter, connection=connection, requests=requests) + ) + adapter.retention_state_from_relation.side_effect = lambda *, request, relation: ( + RetentionAdapterMixin.retention_state_from_relation( + adapter, request=request, relation=relation + ) + ) + return adapter + + +def _retention_model( + *, + name: str, + desired_days: int, + config_values: dict[str, object], + table_type: ResolvedTableType | None, +) -> CompiledModel: + return CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name), + deps=(), + name=name, + relative_path=Path(f"models/{name}.sql"), + query_sql="SELECT 1 AS order_id", + config=CompileModelConfig( + values=config_values, + time_travel_retention=ResolvedTimeTravelRetention( + desired_days=desired_days, + unmanaged=False, + source=TimeTravelRetentionSource.MODEL, + ), + table_type=table_type or ResolvedTableType(), + ), + destination=CompiledRelationLocation( + database="warehouse", + schema="analytics", + name=name, + qualified_name=f"warehouse.analytics.{name}", + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 1c090a87e4..fd28e8c99c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -16,6 +16,7 @@ RetentionChangePhase, RetentionScope, ) +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -42,10 +43,12 @@ from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( RetentionPlanningErrorTestCase, RetentionPlanningTestCase, + SnapshotRetentionPlanningTestCase, TableTypePlanningTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning.helpers import ( build_retention_planner_inputs, + retention_mock_adapter, ) _EXISTING_ORDERS: dict[str, RelationInfo] = { @@ -97,7 +100,7 @@ def test_given_table_type_drift_when_planning_then_independent_entry_tracks_actual_and_policy( test_case: TableTypePlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 relation: RelationInfo = RelationInfo( database="warehouse", @@ -148,7 +151,7 @@ def test_given_table_type_drift_when_planning_then_independent_entry_tracks_actu def test_given_matched_or_missing_table_when_planning_type_then_no_entry_is_created( test_case: TableTypePlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 runtime, warehouse, scope = build_retention_planner_inputs( adapter=adapter, @@ -195,7 +198,7 @@ def test_given_matched_or_missing_table_when_planning_type_then_no_entry_is_crea def test_given_missing_table_when_planning_type_then_no_entry_is_created( test_case: TableTypePlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 runtime, warehouse, scope = build_retention_planner_inputs( adapter=adapter, @@ -315,7 +318,7 @@ def test_given_missing_table_when_planning_type_then_no_entry_is_created( def test_given_materialization_family_when_planning_table_types_then_only_tables_have_drift( test_case: TableTypePlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 relation: RelationInfo = RelationInfo( database="warehouse", @@ -361,7 +364,7 @@ def test_given_materialization_family_when_planning_table_types_then_only_tables def test_given_declared_table_type_on_non_snowflake_when_planning_then_raises( test_case: RetentionPlanningErrorTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.DUCKDB.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.DUCKDB.value) runtime, warehouse, scope = build_retention_planner_inputs( adapter=adapter, desired_days=test_case.desired_days, @@ -397,7 +400,7 @@ def test_given_declared_table_type_on_non_snowflake_when_planning_then_raises( def test_given_changes_only_current_model_when_planning_type_then_selection_still_produces_entry( test_case: TableTypePlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 runtime: PlannerRuntime warehouse: PlannerWarehouseState @@ -497,7 +500,7 @@ def test_given_changes_only_current_model_when_planning_type_then_selection_stil def test_given_retention_policy_when_planning_then_orders_metadata_safely( test_case: RetentionPlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.inspect_retention.return_value = test_case.observed_state rendered_phase: RetentionChangePhase = { RetentionPlanPhase.NONE: RetentionChangePhase.ALTER, @@ -543,7 +546,7 @@ def test_given_retention_policy_when_planning_then_orders_metadata_safely( def test_given_missing_snowflake_relation_above_transient_limit_when_planning_then_fails_closed( test_case: RetentionPlanningErrorTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) runtime, warehouse, scope = build_retention_planner_inputs( adapter=adapter, desired_days=test_case.desired_days, @@ -569,7 +572,7 @@ def test_given_missing_snowflake_relation_above_transient_limit_when_planning_th def test_given_existing_transient_snowflake_relation_above_limit_when_planning_then_fails_closed( test_case: RetentionPlanningErrorTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.inspect_retention.return_value = RetentionState( request_id="orders", scope=RetentionScope.RELATION, @@ -610,7 +613,7 @@ def test_given_existing_transient_snowflake_relation_above_limit_when_planning_t def test_given_transient_live_table_and_permanent_effective_type_when_planning_then_converts_first( test_case: RetentionPlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 adapter.inspect_retention.return_value = test_case.observed_state adapter.render_retention_changes.return_value = ( @@ -673,7 +676,7 @@ def test_given_transient_live_table_and_permanent_effective_type_when_planning_t def test_given_permanent_live_table_with_retention_drift_when_planning_then_only_alters_retention( test_case: RetentionPlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.SNOWFLAKE.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.SNOWFLAKE.value) adapter.maximum_identifier_length.return_value = 255 adapter.inspect_retention.return_value = test_case.observed_state adapter.render_retention_changes.return_value = ( @@ -736,7 +739,7 @@ def test_given_permanent_live_table_with_retention_drift_when_planning_then_only def test_given_missing_bigquery_dataset_when_planning_then_defers_retention_until_after_create( test_case: RetentionPlanningTestCase, ) -> None: - adapter: Mock = Mock(adapter_name=BuiltinAdapter.BIGQUERY.value) + adapter: Mock = retention_mock_adapter(adapter_name=BuiltinAdapter.BIGQUERY.value) adapter.inspect_retention.return_value = test_case.observed_state adapter.render_retention_changes.return_value = ( RenderedRetentionChange( @@ -769,3 +772,54 @@ def test_given_missing_bigquery_dataset_when_planning_then_defers_retention_unti assert entries[0].direction == test_case.expected_direction assert entries[0].phase == test_case.expected_phase assert RetentionPlanPhase.PRE not in tuple(entry.phase for entry in entries) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRetentionPlanningTestCase( + description="many matching tables plan from the snapshot without metadata queries", + model_names=("orders", "customers", "products", "inventory"), + live_days=7, + desired_days=7, + expected_directions=("match", "match", "match", "match"), + ), + SnapshotRetentionPlanningTestCase( + description="drifted tables plan increases from the snapshot without metadata queries", + model_names=("orders", "customers"), + live_days=1, + desired_days=7, + expected_directions=("increase", "increase"), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_snapshot_retention_when_planning_then_no_table_is_inspected( + test_case: SnapshotRetentionPlanningTestCase, +) -> None: + runtime: PlannerRuntime + warehouse: PlannerWarehouseState + scope: PlannerScope + runtime, warehouse, scope = build_retention_planner_inputs( + adapter=SnowflakeAdapter(), + desired_days=test_case.desired_days, + existing_relations={ + name: RelationInfo( + database="warehouse", + schema="analytics", + name=name, + relation_type="base table", + is_transient=False, + retention_days=test_case.live_days, + ) + for name in test_case.model_names + }, + config_values={}, + model_names=test_case.model_names, + ) + + entries: tuple[RetentionPlanEntry, ...] = plan_retention( + runtime=runtime, warehouse=warehouse, scope=scope + ) + + assert tuple(entry.direction.value for entry in entries) == test_case.expected_directions diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index eda6e0c24b..efbd3a3413 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -180,3 +180,14 @@ class MaterializationRecreatesRelationTestCase: action: PlanAction incremental_mode: str | None expected_recreates: bool + + +@dataclass(frozen=True) +class BatchedRetentionReconciliationTestCase: + description: str + entry_request_ids: tuple[str, ...] + live_days: int + desired_days: int + expected_batch_request_ids: tuple[tuple[str, ...], ...] + expected_single_inspections: tuple[str, ...] + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index c8e5ebb08e..13286a522e 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -6,7 +6,9 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from unittest.mock import Mock +from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -236,3 +238,27 @@ def build_plan_output( model_entries=model_entries, seed_entries=seed_entries, ) + + +def retention_mock_adapter(**attributes: Any) -> Mock: + """Return a Mock adapter whose batched retention hooks use the real default implementations.""" + + adapter: Mock = Mock(**attributes) + adapter.inspect_retentions.side_effect = lambda *, connection, requests: ( + RetentionAdapterMixin.inspect_retentions(adapter, connection=connection, requests=requests) + ) + adapter.retention_state_from_relation.side_effect = lambda *, request, relation: ( + RetentionAdapterMixin.retention_state_from_relation( + adapter, request=request, relation=relation + ) + ) + return adapter + + +def batched_retention_request_ids(*, adapter: Mock) -> tuple[tuple[str, ...], ...]: + """Return the request IDs passed to each batched retention inspection call.""" + + batches: list[tuple[str, ...]] = [] + for batch in adapter.inspect_retentions.call_args_list: + batches.append(tuple(request.request_id for request in batch.kwargs["requests"])) + return tuple(batches) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py index 0fa70c0382..5c153f528b 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py @@ -36,6 +36,7 @@ ) from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( + BatchedRetentionReconciliationTestCase, BuildModelRetentionReconciliationTestCase, BuildRetentionPhaseTestCase, FinalRetentionReconciliationTestCase, @@ -44,7 +45,11 @@ TableTypeConversionErrorTestCase, TableTypeConversionTestCase, ) -from tests.unit.src.sqlbuild.executor.build._helpers.helpers import build_model_plan_entry +from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( + batched_retention_request_ids, + build_model_plan_entry, + retention_mock_adapter, +) _TARGET: RelationInfo = RelationInfo( database="warehouse", @@ -95,7 +100,7 @@ def test_given_retention_plan_when_applying_phase_then_executes_only_ordered_phase_statements( test_case: BuildRetentionPhaseTestCase, ) -> None: - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() connection: object = object() request: RetentionRequest = RetentionRequest( request_id="orders", @@ -186,7 +191,7 @@ def test_given_successful_model_when_reconciling_retention_then_defers_decreases ), ) ) - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.inspect_retention.return_value = RetentionState( request_id="orders", scope=RetentionScope.RELATION, @@ -280,7 +285,7 @@ def test_given_live_retention_after_build_when_reconciling_then_only_gated_decre ), model_entries=(build_model_plan_entry(name="orders", action=test_case.model_action),), ) - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.inspect_retention.return_value = RetentionState( request_id="orders", scope=RetentionScope.RELATION, @@ -348,7 +353,7 @@ def test_given_live_retention_after_build_when_reconciling_then_only_gated_decre def test_given_recoverable_table_type_state_when_converting_then_uses_inspection_only_recovery( test_case: TableTypeConversionTestCase, ) -> None: - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.render_qualified_name.side_effect = lambda *, database, schema, name: ".".join( (database, schema, name) ) @@ -400,7 +405,7 @@ def test_given_recoverable_table_type_state_when_converting_then_uses_inspection def test_given_unknown_live_table_type_when_converting_then_fails_closed( test_case: TableTypeConversionTestCase, ) -> None: - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.render_qualified_name.side_effect = lambda *, database, schema, name: ".".join( (database, schema, name) ) @@ -466,7 +471,7 @@ def test_given_unknown_live_table_type_when_converting_then_fails_closed( def test_given_unrecoverable_table_type_state_when_converting_then_fails_before_swap( test_case: TableTypeConversionErrorTestCase, ) -> None: - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.render_qualified_name.side_effect = lambda *, database, schema, name: ".".join( (database, schema, name) ) @@ -511,7 +516,7 @@ def test_given_retention_inspection_when_adapter_blocks_then_start_is_already_di barrier_events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() request: RetentionRequest = RetentionRequest( request_id="orders", scope=RetentionScope.RELATION, @@ -576,7 +581,7 @@ def test_given_table_type_conversion_when_ddl_blocks_then_start_is_already_dispa barrier_events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - adapter: Mock = Mock() + adapter: Mock = retention_mock_adapter() adapter.render_qualified_name.side_effect = lambda *, database, schema, name: ".".join( (database, schema, name) ) @@ -663,3 +668,97 @@ def test_given_model_action_when_checking_recreation_then_matches_materializatio if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + [ + BatchedRetentionReconciliationTestCase( + description="matching relations are inspected in one batch and left alone", + entry_request_ids=("orders", "customers", "products"), + live_days=7, + desired_days=7, + expected_batch_request_ids=(("orders", "customers", "products"),), + expected_single_inspections=(), + expected_statements=(), + ), + BatchedRetentionReconciliationTestCase( + description="a relation changed by an earlier entry is re-inspected before the next", + entry_request_ids=("orders", "orders", "customers"), + live_days=1, + desired_days=7, + expected_batch_request_ids=(("orders", "customers"),), + expected_single_inspections=("orders",), + expected_statements=("ALTER orders", "ALTER customers"), + ), + ], + ids=lambda case: case.description, +) +def test_given_many_retention_entries_after_build_when_reconciling_then_inspection_is_batched( + test_case: BatchedRetentionReconciliationTestCase, +) -> None: + requests: dict[str, RetentionRequest] = { + request_id: RetentionRequest( + request_id=request_id, + scope=RetentionScope.RELATION, + database=None, + schema="analytics", + name=request_id, + desired_days=test_case.desired_days, + ) + for request_id in test_case.entry_request_ids + } + plan: PlanOutput = PlanOutput( + retention_entries=tuple( + RetentionPlanEntry( + request=requests[request_id], + model_names=(request_id,), + actual_days=test_case.live_days, + effective_days=test_case.live_days, + source="target", + direction=RetentionDirection.INCREASE, + phase=RetentionPlanPhase.PRE, + ) + for request_id in test_case.entry_request_ids + ), + ) + live: dict[str, int] = dict.fromkeys(requests, test_case.live_days) + adapter: Mock = Mock() + adapter.inspect_retentions.side_effect = lambda *, connection, requests: { + request.request_id: RetentionState( + request_id=request.request_id, + scope=RetentionScope.RELATION, + configured_days=live[request.request_id], + effective_days=live[request.request_id], + ) + for request in requests + } + adapter.inspect_retention.side_effect = lambda *, connection, request: RetentionState( + request_id=request.request_id, + scope=RetentionScope.RELATION, + configured_days=live[request.request_id], + effective_days=live[request.request_id], + ) + adapter.render_retention_changes.side_effect = lambda *, request, state: ( + RenderedRetentionChange( + phase=RetentionChangePhase.ALTER, statements=(f"ALTER {request.request_id}",) + ), + ) + adapter.execute.side_effect = lambda *, connection, sql: live.update( + {sql.removeprefix("ALTER "): test_case.desired_days} + ) + connection: object = object() + + reconcile_retention_after_build(plan=plan, adapter=adapter, connection=connection) + + assert batched_retention_request_ids(adapter=adapter) == (test_case.expected_batch_request_ids) + assert ( + tuple( + single.kwargs["request"].request_id + for single in adapter.inspect_retention.call_args_list + ) + == test_case.expected_single_inspections + ) + assert tuple(executed.kwargs["sql"] for executed in adapter.execute.call_args_list) == ( + test_case.expected_statements + ) From b91e80907745db2f48b631b974f7d015595074d1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:14:28 +0100 Subject: [PATCH 3/3] fix: trust snapshot retention only for the destination relation; avoid case-colliding skill file names --- .github/workflows/publish.yml | 2 +- scripts/docs_skill/constants.py | 2 +- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 2 +- .../references/docs/{INDEX.md => CONTENTS.md} | 0 .../planner/_helpers/planning/retention.py | 14 +++- .../docs_skill/_helpers/test_render.py | 13 ++-- .../main/skills/test_skill_directory.py | 7 +- .../planner/_helpers/planning/_test_types.py | 11 ++++ .../_helpers/planning/test_retention.py | 65 +++++++++++++++++++ 9 files changed, 102 insertions(+), 14 deletions(-) rename src/sqlbuild/.agents/skills/sqlbuild/references/docs/{INDEX.md => CONTENTS.md} (100%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 53d7ca0b9f..5856e0bfee 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -80,7 +80,7 @@ jobs: exit 1 fi unzip -l dist/*.whl | grep 'scripts/dupscore/dupscore.toml' - unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md' + unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' python -m venv /tmp/sqlbuild-wheel-smoke /tmp/sqlbuild-wheel-smoke/bin/python -m pip install dist/*.whl /tmp/sqlbuild-wheel-smoke/bin/python -c "import sqlbuild._native as native; assert native.catalogue_json()" diff --git a/scripts/docs_skill/constants.py b/scripts/docs_skill/constants.py index 275592fe2c..a487a207c7 100644 --- a/scripts/docs_skill/constants.py +++ b/scripts/docs_skill/constants.py @@ -8,7 +8,7 @@ DEFAULT_OUTPUT_DIR: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/references/docs") DOCS_BASE_URL: str = "https://docs.sqlbuild.com" GENERATED_MARKER: str = "" -INDEX_FILENAME: str = "INDEX.md" +INDEX_FILENAME: str = "CONTENTS.md" FRONTMATTER_DELIMITER: str = "---" TABLE_OF_CONTENTS_MIN_LINES: int = 100 UNGROUPED_LABEL: str = "Other pages" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 196b14eea7..47cb70d9ba 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -157,7 +157,7 @@ diff. Rules. - [references/macros-and-declarations.md](references/macros-and-declarations.md): macros, enums, constants, placement and scopes. -- Full documentation for this SQLBuild version: [references/docs/INDEX.md](references/docs/INDEX.md) +- Full documentation for this SQLBuild version: [references/docs/CONTENTS.md](references/docs/CONTENTS.md) lists every page; open only the page you need, for example `references/docs/cli/build.md` or `references/docs/concepts/snapshots.md`. The same pages are online at https://docs.sqlbuild.com if you need a newer version. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md similarity index 100% rename from src/sqlbuild/.agents/skills/sqlbuild/references/docs/INDEX.md rename to src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index f919a43a9b..1b63c8c8c1 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -8,6 +8,7 @@ from sqlbuild.adapter.contract.models import ( RelationInfo, + RelationLookup, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -190,9 +191,16 @@ def _plan_relation_retention( ) relation: RelationInfo = warehouse.snapshot.existing_relations[model.name] relation_is_transient: bool | None = relation.is_transient - state: RetentionState = runtime.adapter.retention_state_from_relation( - request=request, relation=relation - ) or runtime.adapter.inspect_retention(connection=runtime.connection, request=request) + listed_state: RetentionState | None = None + if relation.identity == RelationLookup.key( + database=model.destination.database, schema=schema, name=model.destination.name + ): + listed_state = runtime.adapter.retention_state_from_relation( + request=request, relation=relation + ) + state: RetentionState = listed_state or runtime.adapter.inspect_retention( + connection=runtime.connection, request=request + ) if ( state.is_transient and desired_days > 1 diff --git a/tests/unit/scripts/docs_skill/_helpers/test_render.py b/tests/unit/scripts/docs_skill/_helpers/test_render.py index 079fa6327b..577a474981 100644 --- a/tests/unit/scripts/docs_skill/_helpers/test_render.py +++ b/tests/unit/scripts/docs_skill/_helpers/test_render.py @@ -29,7 +29,7 @@ }, stale_page="concepts/retired.md", expected_files=( - "INDEX.md", + "CONTENTS.md", "cli/diff.md", "concepts/diff.md", "concepts/models.md", @@ -43,10 +43,13 @@ ("concepts/diff.md", "[the CLI](../cli/diff.md)"), ("concepts/diff.md", "[elsewhere](https://docs.sqlbuild.com/missing/page)"), ("concepts/diff.md", "kept text"), - ("INDEX.md", "## Concepts"), - ("INDEX.md", "- [Data Diffs](concepts/diff.md) (`concepts/diff`) - Compare data."), - ("INDEX.md", "## CLI Reference"), - ("INDEX.md", "- [diff](cli/diff.md) (`cli/diff`) - Diff command."), + ("CONTENTS.md", "## Concepts"), + ( + "CONTENTS.md", + "- [Data Diffs](concepts/diff.md) (`concepts/diff`) - Compare data.", + ), + ("CONTENTS.md", "## CLI Reference"), + ("CONTENTS.md", "- [diff](cli/diff.md) (`cli/diff`) - Diff command."), ), ), ], diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py index f35fd27731..9444c490ca 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -64,7 +64,7 @@ def test_given_skill_directory_when_updating_then_it_matches_the_package( installed: dict[str, str] = installed_skill_files(skill_dir=skill_dir) packaged: dict[str, str] = load_packaged_skill_files() assert {path: installed[path] for path in packaged} == packaged - assert "references/docs/INDEX.md" in packaged + assert "references/docs/CONTENTS.md" in packaged assert not any((tmp_path / path).exists() for path in test_case.expected_absent_paths) assert { path: (tmp_path / path).read_text(encoding="utf-8") @@ -105,7 +105,7 @@ def test_given_custom_file_in_skill_directory_when_updating_then_nothing_is_writ SkillDirectoryMaintenanceTestCase( description="stale bundled page is reported", project_config='name = "demo"\nadapter = "duckdb"\n\n[skills]\ntargets = ["agents"]\n', - stale_relative_path="references/docs/INDEX.md", + stale_relative_path="references/docs/CONTENTS.md", expected_message_fragment="SQLBuild skill files are out of date", expected_restored=False, ), @@ -115,7 +115,7 @@ def test_given_custom_file_in_skill_directory_when_updating_then_nothing_is_writ 'name = "demo"\nadapter = "duckdb"\n\n[skills]\n' 'targets = ["agents"]\nauto_update = true\n' ), - stale_relative_path="references/docs/INDEX.md", + stale_relative_path="references/docs/CONTENTS.md", expected_message_fragment="Updated stale SQLBuild skill files", expected_restored=True, ), @@ -170,6 +170,7 @@ def test_given_packaged_skill_when_reading_then_entry_is_short_and_references_re ) assert packaged[skill_entry_file].count("\n") <= test_case.max_entry_lines + assert len({path.casefold() for path in packaged}) == len(packaged) assert links - packaged.keys() == test_case.expected_broken_links assert commands - {command.value for command in CliCommand} == ( test_case.expected_unknown_commands diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 66e13a164d..64a01b056a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -54,3 +54,14 @@ class SnapshotRetentionPlanningTestCase: live_days: int desired_days: int expected_directions: tuple[str, ...] + + +@dataclass(frozen=True) +class ForeignSnapshotRelationTestCase: + description: str + snapshot_schema: str + snapshot_days: int + live_days: int + desired_days: int + expected_direction: str + expected_inspections: int diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index fd28e8c99c..0e73a4f65a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -41,6 +41,7 @@ ) from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy, TableTypeSource from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( + ForeignSnapshotRelationTestCase, RetentionPlanningErrorTestCase, RetentionPlanningTestCase, SnapshotRetentionPlanningTestCase, @@ -823,3 +824,67 @@ def test_given_snowflake_snapshot_retention_when_planning_then_no_table_is_inspe ) assert tuple(entry.direction.value for entry in entries) == test_case.expected_directions + + +@pytest.mark.parametrize( + "test_case", + [ + ForeignSnapshotRelationTestCase( + description="same-named table in another schema is not trusted for retention", + snapshot_schema="staging", + snapshot_days=0, + live_days=7, + desired_days=1, + expected_direction="decrease", + expected_inspections=1, + ), + ForeignSnapshotRelationTestCase( + description="destination table in the snapshot is used without inspection", + snapshot_schema="analytics", + snapshot_days=7, + live_days=7, + desired_days=1, + expected_direction="decrease", + expected_inspections=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_snapshot_relation_identity_when_planning_retention_then_only_destination_is_trusted( + test_case: ForeignSnapshotRelationTestCase, +) -> None: + adapter: Mock = Mock(wraps=SnowflakeAdapter()) + adapter.adapter_name = BuiltinAdapter.SNOWFLAKE.value + adapter.inspect_retention.side_effect = lambda *, connection, request: RetentionState( + request_id=request.request_id, + scope=RetentionScope.RELATION, + configured_days=None, + effective_days=test_case.live_days, + relation_kind="PERMANENT", + is_transient=False, + ) + runtime: PlannerRuntime + warehouse: PlannerWarehouseState + scope: PlannerScope + runtime, warehouse, scope = build_retention_planner_inputs( + adapter=adapter, + desired_days=test_case.desired_days, + existing_relations={ + "orders": RelationInfo( + database="warehouse", + schema=test_case.snapshot_schema, + name="orders", + relation_type="base table", + is_transient=False, + retention_days=test_case.snapshot_days, + ) + }, + config_values={}, + ) + + entries: tuple[RetentionPlanEntry, ...] = plan_retention( + runtime=runtime, warehouse=warehouse, scope=scope + ) + + assert entries[0].direction.value == test_case.expected_direction + assert adapter.inspect_retention.call_count == test_case.expected_inspections