From 0dd61908950a7d3c239be715bb669063f78d1acc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:25:03 +0100 Subject: [PATCH 1/5] docs: document --debug and correct test-only macro placement Add global options, including --debug, to the build reference and explain which progress rows need --debug. Describe where macros used only by tests can live. Fix outdated flags, sample output and file layouts across CLI and concept pages. --- website/examples/README.md | 5 +- website/src/content/docs/docs/cli/build.mdx | 21 +++++- website/src/content/docs/docs/cli/clone.mdx | 4 +- website/src/content/docs/docs/cli/compile.mdx | 20 ++--- website/src/content/docs/docs/cli/dag.mdx | 2 +- website/src/content/docs/docs/cli/debug.mdx | 30 +++++--- .../src/content/docs/docs/cli/freshness.mdx | 27 ++++--- website/src/content/docs/docs/cli/init.mdx | 28 ++++--- website/src/content/docs/docs/cli/lineage.mdx | 27 +++---- website/src/content/docs/docs/cli/load.mdx | 22 +++--- website/src/content/docs/docs/cli/plan.mdx | 74 ++++++++++++------- website/src/content/docs/docs/cli/rename.mdx | 3 + .../src/content/docs/docs/cli/scenario.mdx | 8 +- .../src/content/docs/docs/concepts/audits.mdx | 6 +- .../declaration-scopes/visibility.mdx | 2 +- .../src/content/docs/docs/concepts/macros.mdx | 13 +++- .../content/docs/docs/concepts/planning.mdx | 4 +- .../planning/selection-and-staleness.mdx | 12 +++ .../content/docs/docs/concepts/testing.mdx | 6 ++ 19 files changed, 204 insertions(+), 110 deletions(-) diff --git a/website/examples/README.md b/website/examples/README.md index f5ad8b2a44..1223988629 100644 --- a/website/examples/README.md +++ b/website/examples/README.md @@ -12,8 +12,8 @@ DuckDB. ## Step 01: compile errors -Step 01 needs semantic compile checks for unknown columns and type mismatches, which are not -released yet; the output was produced with a development build. In a copy of `waffle-shop`: +Step 01 shows the semantic compile checks for unknown columns and type mismatches. In a copy of +`waffle-shop`: 1. In `sources/raw.yml`, add `contract: enforced` to `raw__customers` and `raw__orders`, so the columns and types that flow into the models are authoritative. @@ -30,6 +30,7 @@ first: cd tidy-shop cat > models/weekly_revenue.sql <<'SQL' MODEL ( + description "Weekly revenue totals", materialized table, ); diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 7dab358b73..123bb973ec 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -14,6 +14,18 @@ Compiles, plans, and executes the selected build lifecycle, running the full sel sqb --project-dir build [flags] ``` +## Global options + +These options apply to every `sqb` command and go before the command name, as in +`sqb --debug build`: + +| Option | Description | +|--------|-------------| +| `--project-dir ` | Run against the project in `` instead of the current directory | +| `--no-color` | Print without colour | +| `--debug` | Print detailed progress, the SQL each step runs, and internal diagnostics to stderr; implies `--verbose` | +| `--version`, `-V` | Print the SQLBuild version | + ## Flags | Flag | Description | @@ -37,7 +49,7 @@ sqb --project-dir build [flags] | `--load` | Explicitly load managed sources before building | | `--no-load` | Skip automatic source loading | | `--reload` | Reload managed sources (passes `is_reload=True` to loaders) | -| `--include-stale-upstreams` | Expand selection to include stale upstream models needed for coherence | +| `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](/docs/concepts/planning/selection-and-staleness) | | `--manifest` | Generate `target/manifest.json` with plan-aware project metadata | | `--select`, `-s` | Select specific models | | `--exclude` | Exclude specific models | @@ -94,6 +106,13 @@ Execution sqb build (concurrency: 1) ✓ Completed successfully PASS=99 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=99 (2.13s) ``` +Each resource prints one row when it finishes, followed by its tests and audits. Rows for +individual warehouse statements and for steps inside a resource, such as `Create staging relation`, +`Promote relation`, `loader callable`, or audit evaluation, appear only with `sqb --debug build`. +Without `--debug`, a statement that is still running after 30 seconds prints a `RUNNING` row every +30 seconds, with its elapsed time and, when the warehouse reports one, its query ID. A failed +statement or step still prints its `FAIL` row. + ## Deferred builds Use `--defer-to` to resolve unselected model references against another target. This lets you build a subset of models in dev while referencing production tables for everything else: diff --git a/website/src/content/docs/docs/cli/clone.mdx b/website/src/content/docs/docs/cli/clone.mdx index 1819bf15d1..446ca3b1fd 100644 --- a/website/src/content/docs/docs/cli/clone.mdx +++ b/website/src/content/docs/docs/cli/clone.mdx @@ -9,8 +9,8 @@ Copies selected relations from one target to another. It uses adapter-native clo No `manifest.json` generation or artifact management is required. Clone works directly against live targets. -When `--to` is omitted, the destination is the active target selected by `--target`, -`sqlbuild_local.toml`, or `default_target`, in that order. +When `--to` is omitted, the destination is the active target selected by `sqlbuild_local.toml` +or `default_target`, in that order. `--from` selects only the origin namespace and its clone-origin policy. `--to`, or the active target when it is omitted, selects the sole physical connection used by the operation. The diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 2650ac6231..6522084693 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -32,7 +32,7 @@ sqb --project-dir compile [flags] ## What compile does 1. **Discovery** - finds `sqlbuild_project.toml`, scans for models, sources, seeds, functions, audits, tests, and macros -2. **Graph resolution** - resolves `ref()` and `source()` calls, expands macros, orders models by dependency +2. **Graph resolution** - resolves `__ref()`, `__source()`, and `__seed()` references, expands macros, orders models by dependency 3. **SQL validation** - validates SQL syntax (when SQL analysis is enabled) 4. **Column lineage** - analyzes column-level dependencies across models (fast mode by default) 5. **Contract validation** - checks declared column contracts against inferred query output @@ -212,17 +212,17 @@ sqb compile ``` ``` -Compile ready (12 models) +Compile ready 12 models - stg_customers OK 3 columns - stg_orders OK 5 columns - stg_payments OK 4 columns - fact_orders OK 6 columns - dim_customers OK 4 columns - daily_revenue OK 3 columns - ... +├── daily_revenue OK 6 columns +├── dim_customers OK 7 columns +├── fact_orders OK 13 columns +├── stg_customers OK 5 columns +├── stg_orders OK 6 columns +└── stg_payments OK 6 columns +... - Compiled: 12 models, 1 seed, 5 functions, 0 errors, 0 warnings +✓ Project compiled 12 models, 1 seed, 2 functions, 0 errors, 0 warnings Wrote: target/compiled/ ``` diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index 57b1ed5aa3..aacd2cb357 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -87,7 +87,7 @@ Each node represents a source, seed, model, or function: | Field | Description | |-------|-------------| | `id` | Unique identifier (`{kind}:{name}`) | -| `kind` | `source`, `seed`, `model`, or `function` | +| `kind` | `source`, `seed`, `model`, `udf` (scalar function), or `table_fn` (table function) | | `name` | Resource name | | `asset_key` | Tuple used as the Dagster asset key (typically `[schema, name]` or `[database, schema, name]`) | | `target` | Warehouse identity (database, schema, name, qualified_name) | diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index d240abffd2..09c8138b78 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -5,6 +5,8 @@ description: "Validate project configuration and test the warehouse connection." Checks that your project config is valid, the adapter is resolvable, and the warehouse connection works. Useful for diagnosing setup issues. +For detailed progress from any other command, use the global [`--debug`](/docs/cli/build#global-options) option instead, as in `sqb --debug build`. + ## Usage ```bash @@ -13,11 +15,13 @@ sqb debug [flags] ## What it checks -The command runs three groups of checks: +The command reports four groups of checks: **Runtime** - SQLBuild version, Python version, Python path, OS info. -**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter and active target. For Snowflake it also shows the warehouse each [command group](/docs/concepts/project-configuration#command-group-warehouses) connects with and where that value came from. +**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](/docs/concepts/project-configuration#command-group-warehouses) connects with and where that value came from. + +**Providers** - Lists the [providers](/docs/concepts/python-nodes/providers) the project defines. **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -39,7 +43,7 @@ sqb debug SQLBuild Diagnostics Runtime: - sqlbuild version: 0.2.1 + sqlbuild version: 0.134.8 python version: 3.14.3 python path: /home/user/.venv/bin/python3 os info: Linux-6.6.87 @@ -50,18 +54,24 @@ Configuration: project: waffle_shop [OK loaded] adapter: snowflake [OK found] target: dev [OK resolved] + connection: snowflake_dev [OK resolved] + database: ANALYTICS [OK resolved] + schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] - query warehouse: SQB_WH [OK connection warehouse] + query warehouse: TRANSFORM_WH [OK connection warehouse] + +Providers: + providers: none discovered [OK none discovered] Connection: - account: FJMQFQV-OJ66172 + account: myorg-myaccount authenticator: programmatic_access_token - database: SQB_DB - role: role_sqb_test - schema: TEST + database: ANALYTICS + role: TRANSFORMER + schema: DEV token: **** - user: svc_sqb_test - warehouse: SQB_WH + user: SQLBUILD_USER + warehouse: TRANSFORM_WH connection test: [OK connected] query test: [OK SELECT 1] ``` diff --git a/website/src/content/docs/docs/cli/freshness.mdx b/website/src/content/docs/docs/cli/freshness.mdx index 36325597da..b23550c4c0 100644 --- a/website/src/content/docs/docs/cli/freshness.mdx +++ b/website/src/content/docs/docs/cli/freshness.mdx @@ -54,11 +54,11 @@ sqb freshness Source freshness Observed (3) - raw_customers timestamp 2026-06-05T14:30:00 adapter - raw_orders timestamp 2026-06-05T15:45:00 column tolerance 15m - raw_payments integer 42871 column +├── raw_customers value 2026-06-05T14:30:00+00:00 kind timestamp via column +├── raw_orders value 2026-06-05T12:00:00+00:00 kind timestamp via column tolerance 15m +└── raw_payments value 2026-06-05T14:28:00+00:00 kind timestamp via column tolerance 15m -Summary: observed=3 changed=0 unchanged=0 tolerated=0 unknown=0 errors=0 +OBSERVED=3 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0 ``` Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata. Sources that can't be observed (expression sources, managed sources without freshness config on unsupported adapters) show as `unknown`. @@ -81,15 +81,15 @@ sqb freshness --state Source freshness Changed (1) - raw_orders previous 2026-06-05T12:00:00 current 2026-06-05T15:45:00 tolerance 15m +└── raw_orders previous 2026-06-05T12:00:00+00:00 current 2026-06-05T15:45:00+00:00 tolerance 15m Unchanged (1) - raw_customers previous 2026-06-05T14:30:00 current 2026-06-05T14:30:00 +└── raw_customers previous 2026-06-05T14:30:00+00:00 current 2026-06-05T14:30:00+00:00 Tolerated (1) - raw_payments previous 2026-06-05T14:28:00 current 2026-06-05T14:30:00 tolerance 15m +└── raw_payments previous 2026-06-05T14:28:00+00:00 current 2026-06-05T14:30:00+00:00 tolerance 15m -Summary: observed=0 changed=1 unchanged=1 tolerated=1 unknown=0 errors=0 +OBSERVED=0 CHANGED=1 UNCHANGED=1 TOLERATED=1 UNKNOWN=0 ERROR=0 ``` ### Statuses @@ -133,7 +133,7 @@ sqb freshness --json "status": "observed", "strategy": "column", "value_kind": "timestamp", - "current_data_version": "2026-06-05T15:45:00", + "current_data_version": "2026-06-05T15:45:00+00:00", "previous_data_version": null, "lag_tolerance": "15m", "target": { @@ -141,7 +141,8 @@ sqb freshness --json "schema": "raw", "name": "orders" }, - "message": null + "message": null, + "age_status": null } ], "summary": { @@ -150,7 +151,11 @@ sqb freshness --json "unchanged": 0, "tolerated": 0, "unknown": 0, - "errors": 0 + "errors": 0, + "age_pass": 0, + "age_warn": 0, + "age_error": 0, + "age_unknown": 0 } } ``` diff --git a/website/src/content/docs/docs/cli/init.mdx b/website/src/content/docs/docs/cli/init.mdx index 86f5dcf05a..2165a5f20b 100644 --- a/website/src/content/docs/docs/cli/init.mdx +++ b/website/src/content/docs/docs/cli/init.mdx @@ -17,12 +17,14 @@ No flags. Run in the directory where you want to create the project. ## Project layout -`sqb init` creates the configuration, linter settings, and empty resource directories needed for a standalone project: +`sqb init` creates the configuration, empty resource directories needed for a standalone project, +and the SQLBuild agent skill (see [`sqb skills`](/docs/cli/skills)): ```text my-project/ sqlbuild_project.toml - .sqruff + .agents/skills/sqlbuild/ + .claude/skills/sqlbuild/ models/ staging/ marts/ @@ -54,23 +56,31 @@ These top-level roles are project-wide. Once an audit, schema, hook, or macro is one folder, such as `models/marts/`, SQLBuild requires it to move into that folder's `_sqlbuild/` directory; see [Where to Put Declarations](/docs/concepts/declaration-scopes/placement). -The generated project uses DuckDB, creates a named `local` connection shared by `dev` and -`prod`, and defaults models to table materialization. Its configuration follows this shape: +The generated project uses DuckDB, creates a named `developer` connection shared by `dev` and +`prod`, defaults models to table materialization, and makes audits warn by default. Its +configuration follows this shape: ```toml +name = "my_project" adapter = "duckdb" default_target = "dev" -[connections.local] +[connections.developer] database = "my_project.duckdb" -[targets.dev] -connection = "local" -schema = "dev" +[settings] +default_audit_severity = "warn" + +[defaults] +materialized = "table" [targets.prod] -connection = "local" +connection = "developer" schema = "prod" + +[targets.dev] +connection = "developer" +schema = "dev" ``` The project name is derived from the current directory name, with hyphens converted to diff --git a/website/src/content/docs/docs/cli/lineage.mdx b/website/src/content/docs/docs/cli/lineage.mdx index f1e59e5c79..e8ffc17e76 100644 --- a/website/src/content/docs/docs/cli/lineage.mdx +++ b/website/src/content/docs/docs/cli/lineage.mdx @@ -54,15 +54,15 @@ upstream ├── model stg_payments models/staging/stg_payments.sql │ └── source raw__payments sources/raw.yml ├── seed waffle_types seeds/waffle_types.csv -└── function udf__is_completed_order functions/sql/udf__is_completed_order.sql +└── udf udf__is_completed_order downstream ├── model customer_status_snapshot models/intermediate/customer_status_snapshot.sql ├── model hourly_order_activity models/marts/hourly_order_activity.sql │ ├── model daily_activity_rollup models/marts/daily_activity_rollup.sql │ │ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql -│ │ └── model hourly_order_activity (already shown) -│ └── model hourly_activity_with_daily_context (already shown) -└── model order_status_index models/intermediate/order_status_index.sql +│ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql (already shown) +├── model order_status_index models/intermediate/order_status_index.sql +└── table_fn table_fn__customer_orders ``` Cycles and repeated nodes are annotated with "(already shown)" to avoid infinite recursion. @@ -76,11 +76,12 @@ sqb lineage fact_orders --format list ``` ``` -source:raw__orders -> model:stg_orders -source:raw__payments -> model:stg_payments -model:stg_orders -> model:fact_orders -model:stg_payments -> model:fact_orders -seed:waffle_types -> model:fact_orders +udf:udf__is_completed_order -> model:fact_orders +model:stg_orders -> model:fact_orders +seed:waffle_types -> model:fact_orders +model:stg_payments -> model:fact_orders +source:raw__orders -> model:stg_orders +source:raw__payments -> model:stg_payments ``` ### JSON @@ -140,8 +141,8 @@ sqb lineage fact_orders.total_cents ``` Column trace fact_orders.total_cents upstream - <- stg_payments.amount_cents (expression) - <- raw__payments.amount_cents (direct) +└── stg_payments.amount_cents (expression) + └── raw__payments.amount_cents (direct) ``` ### List @@ -153,8 +154,8 @@ sqb lineage fact_orders.total_cents --format list ``` Column dependencies -stg_payments.amount_cents -> fact_orders.total_cents expression -raw__payments.amount_cents -> stg_payments.amount_cents direct +stg_payments.amount_cents -> fact_orders.total_cents expression +raw__payments.amount_cents -> stg_payments.amount_cents direct ``` ### JSON diff --git a/website/src/content/docs/docs/cli/load.mdx b/website/src/content/docs/docs/cli/load.mdx index b2303c114e..94745f706e 100644 --- a/website/src/content/docs/docs/cli/load.mdx +++ b/website/src/content/docs/docs/cli/load.mdx @@ -27,7 +27,7 @@ sqb --project-dir load [flags] | `--end-cursor-int` | Override end cursor for integer-based loaders | | `--json` | Output results as JSON | | `--json-output` | Write JSON results to a file path | -| `--var` | Set project variables (`--var key=value`) | +| `--vars` | Override project variables with a JSON object (`--vars '{"key": "value"}'`) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `build` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | ## Examples @@ -58,21 +58,21 @@ Intermediate loaders (those referenced only via `depends_on` without a direct so ## Output ``` -Load ready (3 selected) +Load ready 2 selected -Sources (3) - raw_customers - raw_orders - raw_payments +Sources (2) +├── raw__customers +└── raw__orders Execution sqb load (concurrency: 1) - 1/3 source raw_customers OK 0.05s rows=5 - 2/3 source raw_orders OK 0.03s rows=10 - 3/3 source raw_payments OK 0.02s rows=8 +Connecting to duckdb... +✓ Warehouse connected duckdb (0.05s) -Completed successfully. -PASS=3 WARN=0 FAIL=0 SKIP=0 TOTAL=3 (0.12s) + 1/2 source raw__customers OK 0.04s rows=5 + 2/2 source raw__orders OK 0.03s rows=10 + +✓ Completed successfully PASS=2 WARN=0 FAIL=0 SKIP=0 TOTAL=2 (0.15s) ``` ## Auto-load during builds diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index b2321f692b..aa66b93b67 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -23,6 +23,7 @@ sqb --project-dir plan [flags] | `--defer-to` | Resolve unselected model references against another target | | `--as` | Compile for another target and show its plan, including [model migration](/docs/concepts/models/migrations) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | +| `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](/docs/concepts/planning/selection-and-staleness) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | | `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | | `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | @@ -37,45 +38,68 @@ sqb --project-dir plan [flags] First run: ``` -Plan ready (13 selected) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Changed functions (2) +├── table_fn__customer_orders table function + └── reason first run +└── udf__is_completed_order sql udf + └── reason first run First run (12) - stg_customers view - stg_payments view - stg_orders view - daily_order_partitioned partition_tracked (custom) - daily_revenue table - dim_customers table - fact_orders table - customer_status_snapshot merge (timestamp) - hourly_order_activity delete_insert (timestamp, microbatch) - daily_activity_rollup delete_insert (timestamp, microbatch) - hourly_activity_with_daily_context delete_insert (timestamp, microbatch) - order_status_index delete_insert (integer) +├── stg_customers view +├── stg_payments view +├── stg_orders view +├── daily_order_partitioned partition_tracked (custom) +├── daily_revenue table +├── dim_customers table +├── fact_orders table +├── customer_status_snapshot merge (timestamp) + ├── cursor last_ordered_at (timestamp) + └── bounds computed at run time, once model fact_orders is up to date +├── hourly_order_activity delete_insert (timestamp, microbatch) + ├── cursor activity_hour (timestamp) + ├── mode microbatch + ... +└── order_status_index delete_insert (integer) + ├── cursor order_id (integer) + └── bounds computed at run time, once model fact_orders is up to date Seeds (1) - waffle_types +└── waffle_types (first_run) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` Steady state: ``` -Plan ready (13 selected) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Models (12) +├── stg_customers view +├── stg_payments view +... +└── order_status_index delete_insert (integer) + ├── cursor order_id (integer) + └── bounds computed at run time, once model fact_orders is up to date -Normal (12) - 3 view - 3 table - 3 delete_insert (timestamp, microbatch) - 1 partition_tracked (custom) - 1 merge (timestamp) - 1 delete_insert (integer) +Functions (2) +├── table_fn__customer_orders table function +└── udf__is_completed_order sql udf Seeds (1) - waffle_types +└── waffle_types (current) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function changed`; models further downstream continue forward. See [Replay decisions](/docs/concepts/planning/replay-decisions). diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index 6b41ffb884..f0c6b7dd6c 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -94,8 +94,11 @@ Rename model stg_orders -> stg_order_lines ├── models/marts/fact_orders.sql │ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") ├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders └── tests/unit/test_fact_orders.sql └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Migrations +└── stg_order_lines migrate_from stg_orders keeps the relation's history and its old name working Compiled: ok, 3 files changed ``` diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 8849f86183..c29d81dcae 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -32,7 +32,6 @@ sqb scenario test [flags] | `--max-snapshot-total-rows` | Override total row limit for capture | | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | ### Selectors @@ -88,8 +87,8 @@ Remote scenarios report per-scenario PASS/FAIL with nested check rows: ``` daily_revenue_minimal PASS - check expected daily_revenue PASS - check assertion no_negative_revenue PASS + expect expected daily_revenue PASS + expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 ``` @@ -123,7 +122,6 @@ sqb scenario capture [flags] | `--max-snapshot-total-rows` | Override total row limit | | `--max-snapshot-bytes` | Override per-relation byte limit | | `--max-snapshot-total-bytes` | Override total byte limit | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | ### Examples @@ -160,7 +158,7 @@ target/run/scenarios/daily_revenue_minimal/ ref__stg_payments.sql models/ marts/daily_revenue.sql - checks/ + expectations/ expected__daily_revenue.sql assertion__no_negative_revenue.sql ``` diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 9b2cad7b76..352decd440 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -156,9 +156,9 @@ audit factory that attaches it. A top-level generic audit used only by models un `models/marts/` fails to compile and names the folder it belongs in: ``` -error[P001]: Invalid declaration scope index: -[S024] Declaration 'audit:non_negative' is currently project at 'audits/generic' ... -Move it to 'models/marts/_sqlbuild/_audits/generic/' +error[S024]: Declaration 'audit:non_negative' is currently project at 'audits/generic' (audits/generic/non_negative.sql); required exact-owner-private at 'models/marts'. Consumers: model:daily_revenue. Move it to 'models/marts/_sqlbuild/_audits/generic/' + audit: non_negative + --> audits/generic/non_negative.sql:1:1 ``` Attaching a scoped generic audit from a folder that cannot see it fails with `S006`. Built-in diff --git a/website/src/content/docs/docs/concepts/declaration-scopes/visibility.mdx b/website/src/content/docs/docs/concepts/declaration-scopes/visibility.mdx index cc3294d6cb..12eff374db 100644 --- a/website/src/content/docs/docs/concepts/declaration-scopes/visibility.mdx +++ b/website/src/content/docs/docs/concepts/declaration-scopes/visibility.mdx @@ -141,7 +141,7 @@ test can also use helper macros scoped to its own test directory: ```text tests/unit/orders/ ├── _sqlbuild/ -│ └── macros/ +│ └── _macros/ │ └── fixtures.py └── test_orders__calculates_total.sql ``` diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 9eccc3bc64..b25b8753fe 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -16,8 +16,7 @@ Put project-wide macros in Python files under the top-level `macros/` directory: ```text my_project/ ├── macros/ -│ ├── currency.py -│ └── test_helpers.py +│ └── currency.py ├── models/ └── sqlbuild_project.toml ``` @@ -147,10 +146,16 @@ Imported functions are not re-exported as new macros from the importing file. ## Use macros in tests -Tests are SQL, so they can use macros as reusable fixture generators: +Tests are SQL, so they can use macros as reusable fixture generators. A macro that only tests use +follows the usual [placement](/docs/concepts/declaration-scopes/placement) rule: it must be private +to the narrowest folder that contains its uses, so the project-wide `macros/` directory rejects it +with `S024`. Put it in `_sqlbuild/_macros/` in the folder of the tests that call it, such as +`tests/unit/orders/_sqlbuild/_macros/`, or, when those tests define `__expected__` output for a +model, in that model's `_sqlbuild/_macros/`. The `S024` error names the folder to use. +`tests/unit/_sqlbuild/` itself is rejected with `D013`. ```python -# macros/test_helpers.py +# tests/unit/orders/_sqlbuild/_macros/order_fixtures.py def mock_orders(count: int = 1) -> str: rows = [ f"SELECT {i} AS id, {i * 100} AS customer_id, 'completed' AS status" diff --git a/website/src/content/docs/docs/concepts/planning.mdx b/website/src/content/docs/docs/concepts/planning.mdx index adb21a5f7f..c7d6a8baba 100644 --- a/website/src/content/docs/docs/concepts/planning.mdx +++ b/website/src/content/docs/docs/concepts/planning.mdx @@ -26,7 +26,7 @@ Run the plan command with the same target and selectors you intend to build: ```bash sqb plan -sqb --target dev plan --select +fact_orders +sqb plan --target dev --select +fact_orders ``` A plan answers four questions: @@ -84,7 +84,7 @@ The plan assigns reasons to explain detected work: | Config changed | Version-relevant configuration differs | | Schema changed | A relevant warehouse or upstream schema changed | | Function changed | A function the model calls directly has a different definition | -| Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | +| Renamed | The model's relation moves from an old name through a [model migration](/docs/concepts/models/migrations) | These reasons explain the plan; they are not a promise that a build will skip entries labelled current. Builds execute the selected scope. diff --git a/website/src/content/docs/docs/concepts/planning/selection-and-staleness.mdx b/website/src/content/docs/docs/concepts/planning/selection-and-staleness.mdx index b3f6a086ac..d8f49682a9 100644 --- a/website/src/content/docs/docs/concepts/planning/selection-and-staleness.mdx +++ b/website/src/content/docs/docs/concepts/planning/selection-and-staleness.mdx @@ -39,6 +39,18 @@ Warnings (1) The model stays current rather than being rebuilt on a stale input, and the warning tells you exactly which upstreams changed and how to pull them in. +`sqb plan` and `sqb build` work differently: they run every selected model, even when an +unselected upstream changed. Pass `--selection-diagnostics` to list those stale upstreams. After +changing `stg_orders`, `sqb plan --select fact_orders --selection-diagnostics` shows: + +``` +Warnings (1) +└── fact_orders + ├── selected model 'fact_orders' will build on 1 stale upstream(s) not selected for rebuild: + ├── - stg_orders + └── rebuild the closure to refresh them: --select +fact_orders +``` + ## Incorporating the changed upstreams Use a closure selector to include the changed upstreams in the run. With `+agg_daily_revenue`, SQLBuild pulls in the changed `stg_orders`, cascades the change through the intermediate models, and rebuilds: diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 74c2470ec1..34ebfda398 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -322,6 +322,12 @@ Test SQL uses macros, enums, and constants available from the test file's direct defines `__expected__model_name` output for that model. Model-private values are not available to tests. See [How Visibility Works](/docs/concepts/declaration-scopes/visibility#tests-and-expected-output). +A macro used only by tests must be private to them. Put it in `_sqlbuild/_macros/` in the tests' +folder, such as `tests/unit/orders/_sqlbuild/_macros/`, or in the `_sqlbuild/_macros/` of a model +the tests define `__expected__` output for. The project-wide `macros/` directory rejects a +test-only macro with `S024`, which names the folder to use, and `tests/unit/_sqlbuild/` is rejected +with `D013`. + ## Macro mocking When a model uses macros that you want to control in tests (e.g. target-specific logic, dynamic SQL generation), you can override their output with `__macro__` CTEs: From 5cd7099202b9bfb071f9fc01d98bdfed3f9e0f4e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:30:56 +0100 Subject: [PATCH 2/5] docs: correct audit defaults, seed loading and selector behaviour Audits default to error severity and the delta_and_final run scope, seeds load on every build, path-between selectors include every path, and project concurrency is no longer serial-only. Refresh selector error samples. --- website/src/content/docs/docs/cli/build.mdx | 2 +- website/src/content/docs/docs/concepts/audits.mdx | 6 +++--- .../docs/docs/concepts/project-configuration.mdx | 10 +++++----- .../src/content/docs/docs/concepts/selectors.mdx | 14 +++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 123bb973ec..feb7f7190f 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -68,7 +68,7 @@ This replaces the former `sqb run` command. The full lifecycle (tests + audits) ## Execution order 1. Managed sources are loaded (unless `--no-load`) -2. Seeds are loaded (if changed) +2. Selected seeds are loaded 3. Source audits run before their dependent models (unless `--no-audits`) 4. SQL unit tests run before their target model (unless `--no-tests`) 5. Models are materialized in DAG topological order diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 352decd440..a764da23ca 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -358,7 +358,7 @@ An audit whose query raises a warehouse error, for example because a relation it fails whatever its severity: the build reports the error message and fails, and everything downstream of the audit's target is blocked, the same as when a model audit raises. -Set the default severity in `sqlbuild_project.toml`: +Audits are `error` by default. Set a different default severity in `sqlbuild_project.toml`: ```toml [settings] @@ -379,8 +379,8 @@ Audits on incremental models can run at different lifecycle phases: | Scope | Behavior | |-------|----------| -| `final` | Run once against the staged table before promotion (default). | -| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete. | +| `final` | Run once against the staged table before promotion. | +| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete (default). | ```sql MODEL ( diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index c0fedb7f13..833a3ab52e 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -508,8 +508,8 @@ concurrency = 1 microbatch_concurrency = false auto_load_sources = true table_promotion_mode = "staged" -default_audit_severity = "warn" -default_audit_run_scope = "final" +default_audit_severity = "error" +default_audit_run_scope = "delta_and_final" ``` | Field | Default | Description | @@ -518,12 +518,12 @@ default_audit_run_scope = "final" | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | -| `concurrency` | `1` | Maximum parallel model execution (currently serial only) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/docs/concepts/python-nodes/loaders). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | -| `default_audit_severity` | `warn` | Default severity for audits: `warn` or `error` | -| `default_audit_run_scope` | `final` | Default run scope for audits: `final` or `delta_and_final` | +| `default_audit_severity` | `error` | Default severity for audits: `warn` or `error` | +| `default_audit_run_scope` | `delta_and_final` | Default run scope for audits: `final` or `delta_and_final` | ### Table promotion mode diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index 6b8db58d05..9789836817 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -99,7 +99,7 @@ sqb build --select path:models/staging+ ## Path-between selectors -Select all models on the shortest path between two nodes: +Select all models on every path between two nodes: ```bash sqb build --select fact_orders~daily_activity_rollup @@ -144,15 +144,15 @@ Unknown resource names, name patterns with no matches, empty paths, and malforme clear error messages: ``` -unknown selector name 'nonexistent_model' -unknown selector pattern 'missing_*' -no models found under path 'models/nonexistent'. -no models found with tag 'nonexistent_tag' -path selector 'fact_orders~' requires names on both sides of '~' +error[S007]: unknown selector name 'nonexistent_model' +error[S007]: unknown selector name 'missing_*' +error[S009]: no models found under path 'models/nonexistent'. +error[S000]: no SQL resources or Python nodes found with tag 'nonexistent_tag' +error[S003]: path selector 'fact_orders~' requires names on both sides of '~' ``` If a path selector omits the root directory, SQLBuild asks for the explicit form: ``` -path selectors require an explicit root: use 'models/' or 'python/' +error[S012]: path selectors require an explicit root: use 'models/' or 'python/' ``` From 7ae62fb1050edc7ddfefe59eb6953fcd0b4dde3a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:33:28 +0100 Subject: [PATCH 3/5] docs: fix custom adapter imports and dbt bootstrap sample Custom adapter examples now import from the real module paths and call execute with keyword arguments. The generated dbt connection is named _dbt_profile. The quickstart lists the loader and generic audit where the playground puts them. --- .../src/content/docs/docs/concepts/adapters.mdx | 15 ++++++++------- .../docs/concepts/dbt-compatibility/overview.mdx | 4 ++-- website/src/content/docs/docs/quickstart.mdx | 8 ++++---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/website/src/content/docs/docs/concepts/adapters.mdx b/website/src/content/docs/docs/concepts/adapters.mdx index 53da54eb6f..bdbd328538 100644 --- a/website/src/content/docs/docs/concepts/adapters.mdx +++ b/website/src/content/docs/docs/concepts/adapters.mdx @@ -46,7 +46,7 @@ The most common case is extending an existing adapter with custom behavior. Subc ```python # adapters/duckdb_plus.py -from sqlbuild.integrations.duckdb.client import DuckDbAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter class DuckDbPlusAdapter(DuckDbAdapter): adapter_name = "duckdb_plus" @@ -54,7 +54,7 @@ class DuckDbPlusAdapter(DuckDbAdapter): def connect(self, config): connection = super().connect(config) # custom setup - load extensions, set pragmas, etc. - self.execute(connection, "SET enable_progress_bar = true") + self.execute(connection=connection, sql="SET enable_progress_bar = true") return connection ``` @@ -74,7 +74,7 @@ For a database engine with no built-in support, subclass `BaseAdapter`. It provi ```python # adapters/my_database.py -from sqlbuild.adapter.base.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter class MyDatabaseAdapter(BaseAdapter): adapter_name = "my_database" @@ -83,7 +83,7 @@ class MyDatabaseAdapter(BaseAdapter): def connect(self, config): ... - def execute(self, connection, sql): + def execute(self, *, connection, sql): ... def close(self, connection): @@ -99,7 +99,7 @@ For full control with no inherited defaults, subclass `StrictAdapter` instead. E - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) ### Adapter class hierarchy @@ -107,6 +107,7 @@ For full control with no inherited defaults, subclass `StrictAdapter` instead. E StrictAdapter (fully abstract - all methods must be implemented) └── BaseAdapter (ANSI SQL defaults - override only what differs) ├── DuckDbAdapter + ├── MotherDuckAdapter ├── SnowflakeAdapter ├── BigQueryAdapter ├── DatabricksAdapter @@ -117,6 +118,6 @@ StrictAdapter (fully abstract - all methods must be implemented) `StrictAdapter` composes four mixins that define the full adapter contract: - **ConnectionMixin** - `connect`, `close`, `begin`, `commit`, `rollback` -- **SchemaMixin** - `relation_exists`, `list_relations`, `describe_relation`, `ensure_schema` -- **MaterializationMixin** - `create_table_as`, `create_view`, `drop`, `rename`, `swap`, `load_seed` +- **SchemaMixin** - `relation_exists`, `list_relations`, `get_columns`, `ensure_schema` +- **MaterializationMixin** - `create_table_as`, `create_view_as`, `drop`, `rename`, `swap`, `load_seed` - **DiffMixin** - `diff_schema`, `diff_rows`, `sample_unequal_rows`, `sample_side_only_rows` diff --git a/website/src/content/docs/docs/concepts/dbt-compatibility/overview.mdx b/website/src/content/docs/docs/concepts/dbt-compatibility/overview.mdx index ae81b97805..dace6c4460 100644 --- a/website/src/content/docs/docs/concepts/dbt-compatibility/overview.mdx +++ b/website/src/content/docs/docs/concepts/dbt-compatibility/overview.mdx @@ -45,13 +45,13 @@ profiles_dir = "/Users/you/.dbt" target_path = "../analytics/target" target = "dev" -[connections.dbt_dev] +[connections.dev_dbt_profile] source = "dbt_profile" profile = "analytics" target = "dev" [targets.dev] -connection = "dbt_dev" +connection = "dev_dbt_profile" database = "ANALYTICS" schema = "analytics" ``` diff --git a/website/src/content/docs/docs/quickstart.mdx b/website/src/content/docs/docs/quickstart.mdx index 3d6cfffde1..3a63e6543e 100644 --- a/website/src/content/docs/docs/quickstart.mdx +++ b/website/src/content/docs/docs/quickstart.mdx @@ -66,7 +66,7 @@ Seeds (1) ✓ Plan complete 15 selected, 2 sources to load ``` -`raw__customers` and `raw__orders` are loaded by a Python loader (`sources/loaders/waffle_sources.py`) at the start of the build; `raw__payments` is an inline expression in `sources/raw.yml`. Incremental models also list their cursor and window under their row; they are left out here. +`raw__customers` and `raw__orders` are loaded by a Python loader (`python/loaders/waffle_sources.py`) at the start of the build; `raw__payments` is an inline expression in `sources/raw.yml`. Incremental models also list their cursor and window under their row; they are left out here. ## 3. Build @@ -263,15 +263,15 @@ waffle-shop/ hourly_activity_with_daily_context.sql # microbatch incremental daily_activity_rollup.sql # microbatch incremental _sqlbuild/ + _audits/ + generic/ + expression_is_true.sql # custom generic audit _macros/ # visible only to models directly in marts/ currency.py # cents_to_dollars macro datetime.py # adapter-portable timestamp_trunc intermediate/ customer_status_snapshot.sql # merge incremental order_status_index.sql # integer cursor incremental - audits/ - generic/ - expression_is_true.sql # custom generic audit tests/ unit/ test_stg_orders.sql # model unit test From d6458aac5ad652e77309f62bd91d594ff359df9e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:40:14 +0100 Subject: [PATCH 4/5] docs: refresh the P006 macro reference sample --- website/src/content/docs/docs/concepts/macros.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index b25b8753fe..48fc2d971f 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -111,9 +111,12 @@ it arrived as a quoted string argument such as `@wrap('__ref("stg_orders")')`: ``` error[P006]: model:order_summary depends on model:stg_orders through macro orders_base() + model: order_summary --> models/marts/_sqlbuild/_macros/orders.py = help: write the reference in the model, or pass it in: @orders_base(__ref("stg_orders")) - = help: while migrating a project, allow macro-generated references with [references] enforce_explicit = false in sqlbuild_project.toml + = help: the current value is [references] enforce_explicit = true (from sqlbuild_project.toml or its default); while migrating a project, to allow macro-generated references, set this in sqlbuild_project.toml: + [references] + enforce_explicit = false ``` A dependency hidden inside a macro is invisible to reviewers of the model file and bypasses From 0531ad121b05eff51b3d06d405328c1eaa0113cc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:40:14 +0100 Subject: [PATCH 5/5] docs: regenerate the SQLBuild skill references --- .../sqlbuild/references/docs/cli/build.md | 51 ++++-- .../sqlbuild/references/docs/cli/clone.md | 4 +- .../sqlbuild/references/docs/cli/compile.md | 20 +-- .../sqlbuild/references/docs/cli/dag.md | 2 +- .../sqlbuild/references/docs/cli/debug.md | 30 ++-- .../sqlbuild/references/docs/cli/freshness.md | 27 +-- .../sqlbuild/references/docs/cli/init.md | 28 +++- .../sqlbuild/references/docs/cli/lineage.md | 27 +-- .../sqlbuild/references/docs/cli/load.md | 22 +-- .../sqlbuild/references/docs/cli/plan.md | 74 ++++++--- .../sqlbuild/references/docs/cli/rename.md | 3 + .../sqlbuild/references/docs/cli/scenario.md | 8 +- .../references/docs/concepts/adapters.md | 15 +- .../references/docs/concepts/audits.md | 12 +- .../concepts/dbt-compatibility/overview.md | 4 +- .../concepts/declaration-scopes/visibility.md | 2 +- .../references/docs/concepts/macros.md | 18 +- .../references/docs/concepts/planning.md | 4 +- .../planning/selection-and-staleness.md | 12 ++ .../docs/concepts/project-configuration.md | 10 +- .../references/docs/concepts/selectors.md | 14 +- .../references/docs/concepts/testing.md | 6 + .../sqlbuild/references/docs/quickstart.md | 157 ++++++++++-------- 23 files changed, 340 insertions(+), 210 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 635b22ac6b..6290b31341 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -9,6 +9,7 @@ Online: https://sqlbuild.com/docs/cli/build/ ## Contents - Usage +- Global options - Flags - Fast iteration - Execution order @@ -27,6 +28,18 @@ Compiles, plans, and executes the selected build lifecycle, running the full sel sqb --project-dir build [flags] ``` +## Global options + +These options apply to every `sqb` command and go before the command name, as in +`sqb --debug build`: + +| Option | Description | +|--------|-------------| +| `--project-dir ` | Run against the project in `` instead of the current directory | +| `--no-color` | Print without colour | +| `--debug` | Print detailed progress, the SQL each step runs, and internal diagnostics to stderr; implies `--verbose` | +| `--version`, `-V` | Print the SQLBuild version | + ## Flags | Flag | Description | @@ -50,7 +63,7 @@ sqb --project-dir build [flags] | `--load` | Explicitly load managed sources before building | | `--no-load` | Skip automatic source loading | | `--reload` | Reload managed sources (passes `is_reload=True` to loaders) | -| `--include-stale-upstreams` | Expand selection to include stale upstream models needed for coherence | +| `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--manifest` | Generate `target/manifest.json` with plan-aware project metadata | | `--select`, `-s` | Select specific models | | `--exclude` | Exclude specific models | @@ -69,7 +82,7 @@ This replaces the former `sqb run` command. The full lifecycle (tests + audits) ## Execution order 1. Managed sources are loaded (unless `--no-load`) -2. Seeds are loaded (if changed) +2. Selected seeds are loaded 3. Source audits run before their dependent models (unless `--no-audits`) 4. SQL unit tests run before their target model (unless `--no-tests`) 5. Models are materialized in DAG topological order @@ -84,24 +97,36 @@ audit reads, so the audit can gate the resource like its other audits. See ``` Execution sqb build (concurrency: 1) - 1/13 seed waffle_types OK 0.09s - 2/13 view stg_customers OK 0.05s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - 3/13 view stg_orders OK 0.03s + 1/17 source raw__customers OK 0.05s rows=5 + 2/17 source raw__orders OK 0.03s rows=10 + + 3/17 seed waffle_types OK 0.03s + 7/17 view stg_orders OK 0.04s test test_stg_orders PASS + expect expected stg_orders PASS + expect assertion order_ids_are_not_null PASS audit not_null (order_id) PASS audit unique (order_id) PASS - 10/13 table hourly_order_activity (delete_insert) OK 0.16s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS + audit not_null (customer_id) PASS + audit relationships (waffle_type_id) PASS + audit accepted_values (status) PASS + 13/17 table hourly_order_activity (delete_insert) OK 0.24s + 5 batches (1d) range 2026-03-31T09:00:00 → 2026-04-04T14:00:00 8 rows + audit (d) orders_placed_is_non_negative PASS 5/5 + audit (d) not_null (activity_hour) PASS 5/5 + audit (f) orders_placed_is_non_negative PASS audit (f) not_null (activity_hour) PASS -Completed successfully. -PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) +✓ Completed successfully PASS=99 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=99 (2.13s) ``` +Each resource prints one row when it finishes, followed by its tests and audits. Rows for +individual warehouse statements and for steps inside a resource, such as `Create staging relation`, +`Promote relation`, `loader callable`, or audit evaluation, appear only with `sqb --debug build`. +Without `--debug`, a statement that is still running after 30 seconds prints a `RUNNING` row every +30 seconds, with its elapsed time and, when the warehouse reports one, its query ID. A failed +statement or step still prints its `FAIL` row. + ## Deferred builds Use `--defer-to` to resolve unselected model references against another target. This lets you build a subset of models in dev while referencing production tables for everything else: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md index 1d2a61c9f6..8dd961fa5b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/clone.md @@ -10,8 +10,8 @@ Copies selected relations from one target to another. It uses adapter-native clo No `manifest.json` generation or artifact management is required. Clone works directly against live targets. -When `--to` is omitted, the destination is the active target selected by `--target`, -`sqlbuild_local.toml`, or `default_target`, in that order. +When `--to` is omitted, the destination is the active target selected by `sqlbuild_local.toml` +or `default_target`, in that order. `--from` selects only the origin namespace and its clone-origin policy. `--to`, or the active target when it is omitted, selects the sole physical connection used by the operation. The diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 5761be4184..966b45dbd6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -43,7 +43,7 @@ sqb --project-dir compile [flags] ## What compile does 1. **Discovery** - finds `sqlbuild_project.toml`, scans for models, sources, seeds, functions, audits, tests, and macros -2. **Graph resolution** - resolves `ref()` and `source()` calls, expands macros, orders models by dependency +2. **Graph resolution** - resolves `__ref()`, `__source()`, and `__seed()` references, expands macros, orders models by dependency 3. **SQL validation** - validates SQL syntax (when SQL analysis is enabled) 4. **Column lineage** - analyzes column-level dependencies across models (fast mode by default) 5. **Contract validation** - checks declared column contracts against inferred query output @@ -217,17 +217,17 @@ sqb compile ``` ``` -Compile ready (12 models) +Compile ready 12 models - stg_customers OK 3 columns - stg_orders OK 5 columns - stg_payments OK 4 columns - fact_orders OK 6 columns - dim_customers OK 4 columns - daily_revenue OK 3 columns - ... +├── daily_revenue OK 6 columns +├── dim_customers OK 7 columns +├── fact_orders OK 13 columns +├── stg_customers OK 5 columns +├── stg_orders OK 6 columns +└── stg_payments OK 6 columns +... - Compiled: 12 models, 1 seed, 5 functions, 0 errors, 0 warnings +✓ Project compiled 12 models, 1 seed, 2 functions, 0 errors, 0 warnings Wrote: target/compiled/ ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index 7aa68e393a..7e126a40c0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -96,7 +96,7 @@ Each node represents a source, seed, model, or function: | Field | Description | |-------|-------------| | `id` | Unique identifier (`{kind}:{name}`) | -| `kind` | `source`, `seed`, `model`, or `function` | +| `kind` | `source`, `seed`, `model`, `udf` (scalar function), or `table_fn` (table function) | | `name` | Resource name | | `asset_key` | Tuple used as the Dagster asset key (typically `[schema, name]` or `[database, schema, name]`) | | `target` | Warehouse identity (database, schema, name, qualified_name) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index 1cbd7b53a0..08d13112a7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -8,6 +8,8 @@ Online: https://sqlbuild.com/docs/cli/debug/ Checks that your project config is valid, the adapter is resolvable, and the warehouse connection works. Useful for diagnosing setup issues. +For detailed progress from any other command, use the global [`--debug`](build.md#global-options) option instead, as in `sqb --debug build`. + ## Usage ```bash @@ -16,11 +18,13 @@ sqb debug [flags] ## What it checks -The command runs three groups of checks: +The command reports four groups of checks: **Runtime** - SQLBuild version, Python version, Python path, OS info. -**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter and active target. For Snowflake it also shows the warehouse each [command group](../concepts/project-configuration.md#command-group-warehouses) connects with and where that value came from. +**Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](../concepts/project-configuration.md#command-group-warehouses) connects with and where that value came from. + +**Providers** - Lists the [providers](../concepts/python-nodes/providers.md) the project defines. **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -42,7 +46,7 @@ sqb debug SQLBuild Diagnostics Runtime: - sqlbuild version: 0.2.1 + sqlbuild version: 0.134.8 python version: 3.14.3 python path: /home/user/.venv/bin/python3 os info: Linux-6.6.87 @@ -53,18 +57,24 @@ Configuration: project: waffle_shop [OK loaded] adapter: snowflake [OK found] target: dev [OK resolved] + connection: snowflake_dev [OK resolved] + database: ANALYTICS [OK resolved] + schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] - query warehouse: SQB_WH [OK connection warehouse] + query warehouse: TRANSFORM_WH [OK connection warehouse] + +Providers: + providers: none discovered [OK none discovered] Connection: - account: FJMQFQV-OJ66172 + account: myorg-myaccount authenticator: programmatic_access_token - database: SQB_DB - role: role_sqb_test - schema: TEST + database: ANALYTICS + role: TRANSFORMER + schema: DEV token: **** - user: svc_sqb_test - warehouse: SQB_WH + user: SQLBUILD_USER + warehouse: TRANSFORM_WH connection test: [OK connected] query test: [OK SELECT 1] ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md index b46c677134..97ae4958ac 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/freshness.md @@ -66,11 +66,11 @@ sqb freshness Source freshness Observed (3) - raw_customers timestamp 2026-06-05T14:30:00 adapter - raw_orders timestamp 2026-06-05T15:45:00 column tolerance 15m - raw_payments integer 42871 column +├── raw_customers value 2026-06-05T14:30:00+00:00 kind timestamp via column +├── raw_orders value 2026-06-05T12:00:00+00:00 kind timestamp via column tolerance 15m +└── raw_payments value 2026-06-05T14:28:00+00:00 kind timestamp via column tolerance 15m -Summary: observed=3 changed=0 unchanged=0 tolerated=0 unknown=0 errors=0 +OBSERVED=3 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0 ``` Sources without explicit `freshness:` config are auto-observed using the `adapter` strategy if the adapter supports table metadata. Sources that can't be observed (expression sources, managed sources without freshness config on unsupported adapters) show as `unknown`. @@ -93,15 +93,15 @@ sqb freshness --state Source freshness Changed (1) - raw_orders previous 2026-06-05T12:00:00 current 2026-06-05T15:45:00 tolerance 15m +└── raw_orders previous 2026-06-05T12:00:00+00:00 current 2026-06-05T15:45:00+00:00 tolerance 15m Unchanged (1) - raw_customers previous 2026-06-05T14:30:00 current 2026-06-05T14:30:00 +└── raw_customers previous 2026-06-05T14:30:00+00:00 current 2026-06-05T14:30:00+00:00 Tolerated (1) - raw_payments previous 2026-06-05T14:28:00 current 2026-06-05T14:30:00 tolerance 15m +└── raw_payments previous 2026-06-05T14:28:00+00:00 current 2026-06-05T14:30:00+00:00 tolerance 15m -Summary: observed=0 changed=1 unchanged=1 tolerated=1 unknown=0 errors=0 +OBSERVED=0 CHANGED=1 UNCHANGED=1 TOLERATED=1 UNKNOWN=0 ERROR=0 ``` ### Statuses @@ -145,7 +145,7 @@ sqb freshness --json "status": "observed", "strategy": "column", "value_kind": "timestamp", - "current_data_version": "2026-06-05T15:45:00", + "current_data_version": "2026-06-05T15:45:00+00:00", "previous_data_version": null, "lag_tolerance": "15m", "target": { @@ -153,7 +153,8 @@ sqb freshness --json "schema": "raw", "name": "orders" }, - "message": null + "message": null, + "age_status": null } ], "summary": { @@ -162,7 +163,11 @@ sqb freshness --json "unchanged": 0, "tolerated": 0, "unknown": 0, - "errors": 0 + "errors": 0, + "age_pass": 0, + "age_warn": 0, + "age_error": 0, + "age_unknown": 0 } } ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md index ef2de139f0..7929be2bf4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md @@ -18,12 +18,14 @@ No flags. Run in the directory where you want to create the project. ## Project layout -`sqb init` creates the configuration, linter settings, and empty resource directories needed for a standalone project: +`sqb init` creates the configuration, empty resource directories needed for a standalone project, +and the SQLBuild agent skill (see [`sqb skills`](skills.md)): ```text my-project/ sqlbuild_project.toml - .sqruff + .agents/skills/sqlbuild/ + .claude/skills/sqlbuild/ models/ staging/ marts/ @@ -55,23 +57,31 @@ These top-level roles are project-wide. Once an audit, schema, hook, or macro is one folder, such as `models/marts/`, SQLBuild requires it to move into that folder's `_sqlbuild/` directory; see [Where to Put Declarations](../concepts/declaration-scopes/placement.md). -The generated project uses DuckDB, creates a named `local` connection shared by `dev` and -`prod`, and defaults models to table materialization. Its configuration follows this shape: +The generated project uses DuckDB, creates a named `developer` connection shared by `dev` and +`prod`, defaults models to table materialization, and makes audits warn by default. Its +configuration follows this shape: ```toml +name = "my_project" adapter = "duckdb" default_target = "dev" -[connections.local] +[connections.developer] database = "my_project.duckdb" -[targets.dev] -connection = "local" -schema = "dev" +[settings] +default_audit_severity = "warn" + +[defaults] +materialized = "table" [targets.prod] -connection = "local" +connection = "developer" schema = "prod" + +[targets.dev] +connection = "developer" +schema = "dev" ``` The project name is derived from the current directory name, with hyphens converted to diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md index 2014ce6a70..5efed973e4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md @@ -66,15 +66,15 @@ upstream ├── model stg_payments models/staging/stg_payments.sql │ └── source raw__payments sources/raw.yml ├── seed waffle_types seeds/waffle_types.csv -└── function udf__is_completed_order functions/sql/udf__is_completed_order.sql +└── udf udf__is_completed_order downstream ├── model customer_status_snapshot models/intermediate/customer_status_snapshot.sql ├── model hourly_order_activity models/marts/hourly_order_activity.sql │ ├── model daily_activity_rollup models/marts/daily_activity_rollup.sql │ │ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql -│ │ └── model hourly_order_activity (already shown) -│ └── model hourly_activity_with_daily_context (already shown) -└── model order_status_index models/intermediate/order_status_index.sql +│ └── model hourly_activity_with_daily_context models/marts/hourly_activity_with_daily_context.sql (already shown) +├── model order_status_index models/intermediate/order_status_index.sql +└── table_fn table_fn__customer_orders ``` Cycles and repeated nodes are annotated with "(already shown)" to avoid infinite recursion. @@ -88,11 +88,12 @@ sqb lineage fact_orders --format list ``` ``` -source:raw__orders -> model:stg_orders -source:raw__payments -> model:stg_payments -model:stg_orders -> model:fact_orders -model:stg_payments -> model:fact_orders -seed:waffle_types -> model:fact_orders +udf:udf__is_completed_order -> model:fact_orders +model:stg_orders -> model:fact_orders +seed:waffle_types -> model:fact_orders +model:stg_payments -> model:fact_orders +source:raw__orders -> model:stg_orders +source:raw__payments -> model:stg_payments ``` ### JSON @@ -152,8 +153,8 @@ sqb lineage fact_orders.total_cents ``` Column trace fact_orders.total_cents upstream - <- stg_payments.amount_cents (expression) - <- raw__payments.amount_cents (direct) +└── stg_payments.amount_cents (expression) + └── raw__payments.amount_cents (direct) ``` ### List @@ -165,8 +166,8 @@ sqb lineage fact_orders.total_cents --format list ``` Column dependencies -stg_payments.amount_cents -> fact_orders.total_cents expression -raw__payments.amount_cents -> stg_payments.amount_cents direct +stg_payments.amount_cents -> fact_orders.total_cents expression +raw__payments.amount_cents -> stg_payments.amount_cents direct ``` ### JSON diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md index 770802380a..4f7f514996 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/load.md @@ -28,7 +28,7 @@ sqb --project-dir load [flags] | `--end-cursor-int` | Override end cursor for integer-based loaders | | `--json` | Output results as JSON | | `--json-output` | Write JSON results to a file path | -| `--var` | Set project variables (`--var key=value`) | +| `--vars` | Override project variables with a JSON object (`--vars '{"key": "value"}'`) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `build` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | ## Examples @@ -59,21 +59,21 @@ Intermediate loaders (those referenced only via `depends_on` without a direct so ## Output ``` -Load ready (3 selected) +Load ready 2 selected -Sources (3) - raw_customers - raw_orders - raw_payments +Sources (2) +├── raw__customers +└── raw__orders Execution sqb load (concurrency: 1) - 1/3 source raw_customers OK 0.05s rows=5 - 2/3 source raw_orders OK 0.03s rows=10 - 3/3 source raw_payments OK 0.02s rows=8 +Connecting to duckdb... +✓ Warehouse connected duckdb (0.05s) -Completed successfully. -PASS=3 WARN=0 FAIL=0 SKIP=0 TOTAL=3 (0.12s) + 1/2 source raw__customers OK 0.04s rows=5 + 2/2 source raw__orders OK 0.03s rows=10 + +✓ Completed successfully PASS=2 WARN=0 FAIL=0 SKIP=0 TOTAL=2 (0.15s) ``` ## Auto-load during builds diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 0e2c4cf31a..47640cc24e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -32,6 +32,7 @@ sqb --project-dir plan [flags] | `--defer-to` | Resolve unselected model references against another target | | `--as` | Compile for another target and show its plan, including [model migration](../concepts/models/migrations.md) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | +| `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | | `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | | `--end-cursor-ts` | Override end cursor for timestamp incremental models (ISO format) | @@ -46,45 +47,68 @@ sqb --project-dir plan [flags] First run: ``` -Plan ready (13 selected) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Changed functions (2) +├── table_fn__customer_orders table function + └── reason first run +└── udf__is_completed_order sql udf + └── reason first run First run (12) - stg_customers view - stg_payments view - stg_orders view - daily_order_partitioned partition_tracked (custom) - daily_revenue table - dim_customers table - fact_orders table - customer_status_snapshot merge (timestamp) - hourly_order_activity delete_insert (timestamp, microbatch) - daily_activity_rollup delete_insert (timestamp, microbatch) - hourly_activity_with_daily_context delete_insert (timestamp, microbatch) - order_status_index delete_insert (integer) +├── stg_customers view +├── stg_payments view +├── stg_orders view +├── daily_order_partitioned partition_tracked (custom) +├── daily_revenue table +├── dim_customers table +├── fact_orders table +├── customer_status_snapshot merge (timestamp) + ├── cursor last_ordered_at (timestamp) + └── bounds computed at run time, once model fact_orders is up to date +├── hourly_order_activity delete_insert (timestamp, microbatch) + ├── cursor activity_hour (timestamp) + ├── mode microbatch + ... +└── order_status_index delete_insert (integer) + ├── cursor order_id (integer) + └── bounds computed at run time, once model fact_orders is up to date Seeds (1) - waffle_types +└── waffle_types (first_run) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` Steady state: ``` -Plan ready (13 selected) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Models (12) +├── stg_customers view +├── stg_payments view +... +└── order_status_index delete_insert (integer) + ├── cursor order_id (integer) + └── bounds computed at run time, once model fact_orders is up to date -Normal (12) - 3 view - 3 table - 3 delete_insert (timestamp, microbatch) - 1 partition_tracked (custom) - 1 merge (timestamp) - 1 delete_insert (integer) +Functions (2) +├── table_fn__customer_orders table function +└── udf__is_completed_order sql udf Seeds (1) - waffle_types +└── waffle_types (current) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function changed`; models further downstream continue forward. See [Replay decisions](../concepts/planning/replay-decisions.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index e5081706ac..a5abad2a50 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -108,8 +108,11 @@ Rename model stg_orders -> stg_order_lines ├── models/marts/fact_orders.sql │ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") ├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders └── tests/unit/test_fact_orders.sql └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Migrations +└── stg_order_lines migrate_from stg_orders keeps the relation's history and its old name working Compiled: ok, 3 files changed ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 14eb08b544..27155865a1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -39,7 +39,6 @@ sqb scenario test [flags] | `--max-snapshot-total-rows` | Override total row limit for capture | | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | ### Selectors @@ -95,8 +94,8 @@ Remote scenarios report per-scenario PASS/FAIL with nested check rows: ``` daily_revenue_minimal PASS - check expected daily_revenue PASS - check assertion no_negative_revenue PASS + expect expected daily_revenue PASS + expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 ``` @@ -130,7 +129,6 @@ sqb scenario capture [flags] | `--max-snapshot-total-rows` | Override total row limit | | `--max-snapshot-bytes` | Override per-relation byte limit | | `--max-snapshot-total-bytes` | Override total byte limit | -| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | ### Examples @@ -167,7 +165,7 @@ target/run/scenarios/daily_revenue_minimal/ ref__stg_payments.sql models/ marts/daily_revenue.sql - checks/ + expectations/ expected__daily_revenue.sql assertion__no_negative_revenue.sql ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md index 3bea08f0ab..b471f7fe13 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -53,7 +53,7 @@ The most common case is extending an existing adapter with custom behavior. Subc ```python # adapters/duckdb_plus.py -from sqlbuild.integrations.duckdb.client import DuckDbAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter class DuckDbPlusAdapter(DuckDbAdapter): adapter_name = "duckdb_plus" @@ -61,7 +61,7 @@ class DuckDbPlusAdapter(DuckDbAdapter): def connect(self, config): connection = super().connect(config) # custom setup - load extensions, set pragmas, etc. - self.execute(connection, "SET enable_progress_bar = true") + self.execute(connection=connection, sql="SET enable_progress_bar = true") return connection ``` @@ -81,7 +81,7 @@ For a database engine with no built-in support, subclass `BaseAdapter`. It provi ```python # adapters/my_database.py -from sqlbuild.adapter.base.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter class MyDatabaseAdapter(BaseAdapter): adapter_name = "my_database" @@ -90,7 +90,7 @@ class MyDatabaseAdapter(BaseAdapter): def connect(self, config): ... - def execute(self, connection, sql): + def execute(self, *, connection, sql): ... def close(self, connection): @@ -106,7 +106,7 @@ For full control with no inherited defaults, subclass `StrictAdapter` instead. E - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) ### Adapter class hierarchy @@ -114,6 +114,7 @@ For full control with no inherited defaults, subclass `StrictAdapter` instead. E StrictAdapter (fully abstract - all methods must be implemented) └── BaseAdapter (ANSI SQL defaults - override only what differs) ├── DuckDbAdapter + ├── MotherDuckAdapter ├── SnowflakeAdapter ├── BigQueryAdapter ├── DatabricksAdapter @@ -124,6 +125,6 @@ StrictAdapter (fully abstract - all methods must be implemented) `StrictAdapter` composes four mixins that define the full adapter contract: - **ConnectionMixin** - `connect`, `close`, `begin`, `commit`, `rollback` -- **SchemaMixin** - `relation_exists`, `list_relations`, `describe_relation`, `ensure_schema` -- **MaterializationMixin** - `create_table_as`, `create_view`, `drop`, `rename`, `swap`, `load_seed` +- **SchemaMixin** - `relation_exists`, `list_relations`, `get_columns`, `ensure_schema` +- **MaterializationMixin** - `create_table_as`, `create_view_as`, `drop`, `rename`, `swap`, `load_seed` - **DiffMixin** - `diff_schema`, `diff_rows`, `sample_unequal_rows`, `sample_side_only_rows` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 833b3dbd9e..6486305610 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -174,9 +174,9 @@ audit factory that attaches it. A top-level generic audit used only by models un `models/marts/` fails to compile and names the folder it belongs in: ``` -error[P001]: Invalid declaration scope index: -[S024] Declaration 'audit:non_negative' is currently project at 'audits/generic' ... -Move it to 'models/marts/_sqlbuild/_audits/generic/' +error[S024]: Declaration 'audit:non_negative' is currently project at 'audits/generic' (audits/generic/non_negative.sql); required exact-owner-private at 'models/marts'. Consumers: model:daily_revenue. Move it to 'models/marts/_sqlbuild/_audits/generic/' + audit: non_negative + --> audits/generic/non_negative.sql:1:1 ``` Attaching a scoped generic audit from a folder that cannot see it fails with `S006`. Built-in @@ -376,7 +376,7 @@ An audit whose query raises a warehouse error, for example because a relation it fails whatever its severity: the build reports the error message and fails, and everything downstream of the audit's target is blocked, the same as when a model audit raises. -Set the default severity in `sqlbuild_project.toml`: +Audits are `error` by default. Set a different default severity in `sqlbuild_project.toml`: ```toml [settings] @@ -397,8 +397,8 @@ Audits on incremental models can run at different lifecycle phases: | Scope | Behavior | |-------|----------| -| `final` | Run once against the staged table before promotion (default). | -| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete. | +| `final` | Run once against the staged table before promotion. | +| `delta_and_final` | Run against each delta batch before DML, then again against the target after all batches complete (default). | ```sql MODEL ( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md index faad954af2..e0ff7236bb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/overview.md @@ -53,13 +53,13 @@ profiles_dir = "/Users/you/.dbt" target_path = "../analytics/target" target = "dev" -[connections.dbt_dev] +[connections.dev_dbt_profile] source = "dbt_profile" profile = "analytics" target = "dev" [targets.dev] -connection = "dbt_dev" +connection = "dev_dbt_profile" database = "ANALYTICS" schema = "analytics" ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md index e5f7420ec1..a8cadb1a59 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/declaration-scopes/visibility.md @@ -151,7 +151,7 @@ test can also use helper macros scoped to its own test directory: ```text tests/unit/orders/ ├── _sqlbuild/ -│ └── macros/ +│ └── _macros/ │ └── fixtures.py └── test_orders__calculates_total.sql ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 7cf958daae..8457efecc1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -28,8 +28,7 @@ Put project-wide macros in Python files under the top-level `macros/` directory: ```text my_project/ ├── macros/ -│ ├── currency.py -│ └── test_helpers.py +│ └── currency.py ├── models/ └── sqlbuild_project.toml ``` @@ -123,9 +122,12 @@ it arrived as a quoted string argument such as `@wrap('__ref("stg_orders")')`: ``` error[P006]: model:order_summary depends on model:stg_orders through macro orders_base() + model: order_summary --> models/marts/_sqlbuild/_macros/orders.py = help: write the reference in the model, or pass it in: @orders_base(__ref("stg_orders")) - = help: while migrating a project, allow macro-generated references with [references] enforce_explicit = false in sqlbuild_project.toml + = help: the current value is [references] enforce_explicit = true (from sqlbuild_project.toml or its default); while migrating a project, to allow macro-generated references, set this in sqlbuild_project.toml: + [references] + enforce_explicit = false ``` A dependency hidden inside a macro is invisible to reviewers of the model file and bypasses @@ -156,10 +158,16 @@ Imported functions are not re-exported as new macros from the importing file. ## Use macros in tests -Tests are SQL, so they can use macros as reusable fixture generators: +Tests are SQL, so they can use macros as reusable fixture generators. A macro that only tests use +follows the usual [placement](declaration-scopes/placement.md) rule: it must be private +to the narrowest folder that contains its uses, so the project-wide `macros/` directory rejects it +with `S024`. Put it in `_sqlbuild/_macros/` in the folder of the tests that call it, such as +`tests/unit/orders/_sqlbuild/_macros/`, or, when those tests define `__expected__` output for a +model, in that model's `_sqlbuild/_macros/`. The `S024` error names the folder to use. +`tests/unit/_sqlbuild/` itself is rejected with `D013`. ```python -# macros/test_helpers.py +# tests/unit/orders/_sqlbuild/_macros/order_fixtures.py def mock_orders(count: int = 1) -> str: rows = [ f"SELECT {i} AS id, {i * 100} AS customer_id, 'completed' AS status" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md index e44318b6ba..417c91ac0c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md @@ -37,7 +37,7 @@ Run the plan command with the same target and selectors you intend to build: ```bash sqb plan -sqb --target dev plan --select +fact_orders +sqb plan --target dev --select +fact_orders ``` A plan answers four questions: @@ -95,7 +95,7 @@ The plan assigns reasons to explain detected work: | Config changed | Version-relevant configuration differs | | Schema changed | A relevant warehouse or upstream schema changed | | Function changed | A function the model calls directly has a different definition | -| Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | +| Renamed | The model's relation moves from an old name through a [model migration](models/migrations.md) | These reasons explain the plan; they are not a promise that a build will skip entries labelled current. Builds execute the selected scope. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md index 38f2ad6d21..309f5c2af5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/selection-and-staleness.md @@ -42,6 +42,18 @@ Warnings (1) The model stays current rather than being rebuilt on a stale input, and the warning tells you exactly which upstreams changed and how to pull them in. +`sqb plan` and `sqb build` work differently: they run every selected model, even when an +unselected upstream changed. Pass `--selection-diagnostics` to list those stale upstreams. After +changing `stg_orders`, `sqb plan --select fact_orders --selection-diagnostics` shows: + +``` +Warnings (1) +└── fact_orders + ├── selected model 'fact_orders' will build on 1 stale upstream(s) not selected for rebuild: + ├── - stg_orders + └── rebuild the closure to refresh them: --select +fact_orders +``` + ## Incorporating the changed upstreams Use a closure selector to include the changed upstreams in the run. With `+agg_daily_revenue`, SQLBuild pulls in the changed `stg_orders`, cascades the change through the intermediate models, and rebuilds: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index ed614b0cca..882e718ff3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -531,8 +531,8 @@ concurrency = 1 microbatch_concurrency = false auto_load_sources = true table_promotion_mode = "staged" -default_audit_severity = "warn" -default_audit_run_scope = "final" +default_audit_severity = "error" +default_audit_run_scope = "delta_and_final" ``` | Field | Default | Description | @@ -541,12 +541,12 @@ default_audit_run_scope = "final" | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | -| `concurrency` | `1` | Maximum parallel model execution (currently serial only) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | -| `default_audit_severity` | `warn` | Default severity for audits: `warn` or `error` | -| `default_audit_run_scope` | `final` | Default run scope for audits: `final` or `delta_and_final` | +| `default_audit_severity` | `error` | Default severity for audits: `warn` or `error` | +| `default_audit_run_scope` | `delta_and_final` | Default run scope for audits: `final` or `delta_and_final` | ### Table promotion mode diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index b625635644..e96f34019b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -112,7 +112,7 @@ sqb build --select path:models/staging+ ## Path-between selectors -Select all models on the shortest path between two nodes: +Select all models on every path between two nodes: ```bash sqb build --select fact_orders~daily_activity_rollup @@ -157,15 +157,15 @@ Unknown resource names, name patterns with no matches, empty paths, and malforme clear error messages: ``` -unknown selector name 'nonexistent_model' -unknown selector pattern 'missing_*' -no models found under path 'models/nonexistent'. -no models found with tag 'nonexistent_tag' -path selector 'fact_orders~' requires names on both sides of '~' +error[S007]: unknown selector name 'nonexistent_model' +error[S007]: unknown selector name 'missing_*' +error[S009]: no models found under path 'models/nonexistent'. +error[S000]: no SQL resources or Python nodes found with tag 'nonexistent_tag' +error[S003]: path selector 'fact_orders~' requires names on both sides of '~' ``` If a path selector omits the root directory, SQLBuild asks for the explicit form: ``` -path selectors require an explicit root: use 'models/' or 'python/' +error[S012]: path selectors require an explicit root: use 'models/' or 'python/' ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 8bd778c8c2..13ee48ccc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -346,6 +346,12 @@ Test SQL uses macros, enums, and constants available from the test file's direct defines `__expected__model_name` output for that model. Model-private values are not available to tests. See [How Visibility Works](declaration-scopes/visibility.md#tests-and-expected-output). +A macro used only by tests must be private to them. Put it in `_sqlbuild/_macros/` in the tests' +folder, such as `tests/unit/orders/_sqlbuild/_macros/`, or in the `_sqlbuild/_macros/` of a model +the tests define `__expected__` output for. The project-wide `macros/` directory rejects a +test-only macro with `S024`, which names the folder to use, and `tests/unit/_sqlbuild/` is rejected +with `D013`. + ## Macro mocking When a model uses macros that you want to control in tests (e.g. target-specific logic, dynamic SQL generation), you can override their output with `__macro__` CTEs: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md index 1dcebfc5d5..5b7f08be63 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md @@ -50,29 +50,37 @@ sqb plan ``` ``` -Plan ready (13 selected) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Changed functions (2) +├── table_fn__customer_orders table function +└── udf__is_completed_order sql udf First run (12) - stg_customers view - stg_payments view - stg_orders view - daily_order_partitioned partition_tracked (custom) - daily_revenue table - dim_customers table - fact_orders table - customer_status_snapshot merge (timestamp) - hourly_order_activity delete_insert (timestamp, microbatch) - daily_activity_rollup delete_insert (timestamp, microbatch) - hourly_activity_with_daily_context delete_insert (timestamp, microbatch) - order_status_index delete_insert (integer) +├── stg_customers view +├── stg_payments view +├── stg_orders view +├── daily_order_partitioned partition_tracked (custom) +├── daily_revenue table +├── dim_customers table +├── fact_orders table +├── customer_status_snapshot merge (timestamp) +├── hourly_order_activity delete_insert (timestamp, microbatch) +├── daily_activity_rollup delete_insert (timestamp, microbatch) +├── hourly_activity_with_daily_context delete_insert (timestamp, microbatch) +└── order_status_index delete_insert (integer) Seeds (1) - waffle_types +└── waffle_types (first_run) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` -Source data is defined as inline expressions in `sources/raw.yml`, so SQLBuild resolves everything at compile time - no seeding step needed. +`raw__customers` and `raw__orders` are loaded by a Python loader (`python/loaders/waffle_sources.py`) at the start of the build; `raw__payments` is an inline expression in `sources/raw.yml`. Incremental models also list their cursor and window under their row; they are left out here. ## 3. Build @@ -87,55 +95,60 @@ SQLBuild materializes all models in DAG order. SQL unit tests run before their t ``` Execution sqb build (concurrency: 1) - 1/13 seed waffle_types OK 0.09s - 2/13 view stg_customers OK 0.05s + 1/17 source raw__customers OK 0.05s rows=5 + 2/17 source raw__orders OK 0.03s rows=10 + + 3/17 seed waffle_types OK 0.03s + 4/17 view stg_customers OK 0.02s audit not_null (customer_id) PASS audit unique (customer_id) PASS audit not_null (email) PASS - 3/13 view stg_payments OK 0.02s - audit not_null (payment_id) PASS - audit unique (payment_id) PASS - audit not_null (order_id) PASS - 4/13 view stg_orders OK 0.03s + 7/17 view stg_orders OK 0.04s + test test_daily_revenue_chain PASS + expect expected stg_orders PASS + expect expected stg_payments PASS + expect expected daily_revenue PASS + expect assertion daily_revenue_is_non_negative PASS test test_stg_orders PASS + expect expected stg_orders PASS + expect assertion order_ids_are_not_null PASS audit not_null (order_id) PASS audit unique (order_id) PASS audit not_null (customer_id) PASS + audit relationships (waffle_type_id) PASS audit accepted_values (status) PASS - 5/13 custom daily_order_partitioned OK 0.17s - audit expression_is_true PASS 4/4 + 8/17 custom daily_order_partitioned OK 0.21s + audit waffles_ordered_is_positive PASS 4/4 audit not_null (order_date) PASS 4/4 - 6/13 table daily_revenue OK 0.05s - audit expression_is_true PASS - 7/13 table dim_customers OK 0.05s - audit not_null (customer_id) PASS - audit unique (customer_id) PASS - 8/13 table fact_orders OK 0.04s + 9/17 table daily_revenue OK 0.07s + test test_daily_revenue_chain PASS + expect expected stg_orders PASS + expect expected stg_payments PASS + expect expected daily_revenue PASS + expect assertion daily_revenue_is_non_negative PASS + audit revenue_is_non_negative PASS + 11/17 table fact_orders OK 0.05s + test test_fact_orders PASS + expect expected fact_orders PASS + expect assertion line_totals_are_non_negative PASS audit not_null (order_id) PASS - 9/13 table customer_status_snapshot (merge) OK 0.04s + 12/17 table customer_status_snapshot (merge) OK 0.05s audit not_null (customer_id) PASS audit unique (customer_id) PASS - 10/13 table hourly_order_activity (delete_insert) OK 0.16s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS + 13/17 table hourly_order_activity (delete_insert) OK 0.24s + 5 batches (1d) range 2026-03-31T09:00:00 → 2026-04-04T14:00:00 8 rows + audit (d) orders_placed_is_non_negative PASS 5/5 + audit (d) not_null (activity_hour) PASS 5/5 + audit (f) orders_placed_is_non_negative PASS audit (f) not_null (activity_hour) PASS - 11/13 table order_status_index (delete_insert) OK 0.03s - audit not_null (order_id) PASS - audit unique (order_id) PASS - 12/13 table daily_activity_rollup (delete_insert) OK 0.08s - audit (d) expression_is_true PASS 2/2 - audit (d) not_null (activity_day) PASS 2/2 - audit (f) expression_is_true PASS - audit (f) not_null (activity_day) PASS - 13/13 table hourly_activity_with_daily_context (delete_insert) OK 0.17s - audit (d) expression_is_true PASS 4/4 - audit (d) not_null (activity_hour) PASS 4/4 - audit (f) expression_is_true PASS + 17/17 table hourly_activity_with_daily_context (delete_insert) OK 0.63s + 13 batches (6h) range 2026-03-31T18:00:00 → 2026-04-03T23:00:00 0 rows + audit (d) day_orders_cover_hourly_orders PASS 13/13 + audit (d) not_null (activity_hour) PASS 13/13 + audit (f) day_orders_cover_hourly_orders PASS audit (f) not_null (activity_hour) PASS -Completed successfully. -PASS=66 WARN=0 FAIL=0 SKIP=0 TOTAL=66 (1.09s) +✓ Completed successfully PASS=99 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=99 (2.13s) ``` Notice `audit (d)` and `audit (f)` on incremental models - these are delta-phase and final-phase audits. Delta audits validate each batch before DML is applied; final audits run against the target after promotion. @@ -149,23 +162,37 @@ sqb plan ``` ``` -Plan ready (13 selected) - -Normal (12) - 3 view - 3 table - 3 delete_insert (timestamp, microbatch) - 1 partition_tracked (custom) - 1 merge (timestamp) - 1 delete_insert (integer) +Plan ready 15 selected, 2 sources to load + +Sources to load (2) +├── raw__customers table +└── raw__orders delete_insert (cursor: ordered_at) + +Models (12) +├── stg_customers view +├── stg_payments view +├── stg_orders view +├── daily_order_partitioned partition_tracked (custom) +├── daily_revenue table +├── dim_customers table +├── fact_orders table +├── customer_status_snapshot merge (timestamp) +├── hourly_order_activity delete_insert (timestamp, microbatch) +├── daily_activity_rollup delete_insert (timestamp, microbatch) +├── hourly_activity_with_daily_context delete_insert (timestamp, microbatch) +└── order_status_index delete_insert (integer) + +Functions (2) +├── table_fn__customer_orders table function +└── udf__is_completed_order sql udf Seeds (1) - waffle_types +└── waffle_types (current) -✓ Plan complete 13 selected +✓ Plan complete 15 selected, 2 sources to load ``` -All models now show `Normal` instead of `First run`. +The models and functions are no longer listed as first run, and the seed is current. The two loader-managed sources load on every build. ## 5. Common operations @@ -250,15 +277,15 @@ waffle-shop/ hourly_activity_with_daily_context.sql # microbatch incremental daily_activity_rollup.sql # microbatch incremental _sqlbuild/ + _audits/ + generic/ + expression_is_true.sql # custom generic audit _macros/ # visible only to models directly in marts/ currency.py # cents_to_dollars macro datetime.py # adapter-portable timestamp_trunc intermediate/ customer_status_snapshot.sql # merge incremental order_status_index.sql # integer cursor incremental - audits/ - generic/ - expression_is_true.sql # custom generic audit tests/ unit/ test_stg_orders.sql # model unit test