diff --git a/.github/workflows/benchmark-dbt.yml b/.github/workflows/benchmark-dbt.yml index 01203854ee..fde3196e6b 100644 --- a/.github/workflows/benchmark-dbt.yml +++ b/.github/workflows/benchmark-dbt.yml @@ -1,6 +1,6 @@ name: Benchmark dbt comparison -# Manual only. Times SQLBuild, dbt v2 and dbt Core on the same generated dense DuckDB project with +# Manual only. Times SQLBuild and dbt v2 (dbt Core on request) on the same generated dense DuckDB project with # released packages from PyPI, and publishes the tables as an artifact and in the job summary. # See benchmarks/dbt-compare/README.md. on: @@ -17,12 +17,12 @@ on: default: "2.0.9" type: string dbt_core_version: - description: "dbt-core version from PyPI" + description: "dbt-core version from PyPI (installed only for core_* variants)" required: false default: "1.12.6" type: string dbt_duckdb_version: - description: "dbt-duckdb version from PyPI" + description: "dbt-duckdb version from PyPI (installed only for core_* variants)" required: false default: "1.11.0" type: string @@ -39,7 +39,7 @@ on: variants: description: "Space-separated bench.sh variants" required: false - default: "sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse core_compile" + default: "sqb_norules sqb_norules_full sqb_rules sqb_rules_full v2_compile_cached v2_compile_strict_cached v2_compile_dirty v2_compile_strict_dirty v2_compile v2_compile_strict v2_compile_strict_remote" type: string permissions: @@ -87,7 +87,6 @@ jobs: echo "RESULTS=$RUNNER_TEMP/dbt-compare-results" echo "SQB=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/sqb" echo "DBT_V2=$RUNNER_TEMP/dbt-compare/venv-dbt2/bin/dbt" - echo "DBT_CORE=$RUNNER_TEMP/dbt-compare/venv-dbt/bin/dbt" echo "PYTHON=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/python" } >>"$GITHUB_ENV" - name: Install released packages @@ -99,9 +98,13 @@ jobs: uv pip install -q -p "$WORK/venv-sqb/bin/python" "$sqlbuild_spec" pytest duckdb uv venv -q -p 3.12 "$WORK/venv-dbt2" uv pip install -q -p "$WORK/venv-dbt2/bin/python" "dbt==$DBT_VERSION" - uv venv -q -p 3.12 "$WORK/venv-dbt" - uv pip install -q -p "$WORK/venv-dbt/bin/python" \ - "dbt-core==$DBT_CORE_VERSION" "dbt-duckdb==$DBT_DUCKDB_VERSION" + # dbt Core is a reference only; install it when a core_* variant is requested. + if [[ " $VARIANTS " == *" core_"* ]]; then + uv venv -q -p 3.12 "$WORK/venv-dbt" + uv pip install -q -p "$WORK/venv-dbt/bin/python" \ + "dbt-core==$DBT_CORE_VERSION" "dbt-duckdb==$DBT_DUCKDB_VERSION" + echo "DBT_CORE=$WORK/venv-dbt/bin/dbt" >>"$GITHUB_ENV" + fi - name: Record versions and runner run: | { @@ -113,8 +116,10 @@ jobs: echo "- Python: $("$WORK/venv-sqb/bin/python" -c 'import platform; print(platform.python_version())')" echo "- sqlbuild: $("$WORK/venv-sqb/bin/python" -c 'import importlib.metadata as m; print(m.version("sqlbuild"))')" echo "- dbt (v2): $("$WORK/venv-dbt2/bin/dbt" --version 2>&1 | head -1)" - echo "- dbt-core: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-core"))')" - echo "- dbt-duckdb: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-duckdb"))')" + if [[ -n ${DBT_CORE:-} ]]; then + echo "- dbt-core: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-core"))')" + echo "- dbt-duckdb: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-duckdb"))')" + fi echo "- Models: ${MODELS}, rounds: ${ROUNDS}, variants: ${VARIANTS}" echo } | tee "$RESULTS/environment.md" @@ -126,10 +131,17 @@ jobs: run: | "$BENCH/correctness.sh" "$WORK/correctness" | tee "$RESULTS/correctness.md" cp -r "$WORK/correctness/logs" "$RESULTS/correctness-logs" + # Strict analysis is only in the `dbt` distribution and can fall back silently; stop before + # the long benchmark unless it demonstrably ran. + - name: Prove dbt v2 strict analysis ran (correctness) + run: python3 "$BENCH/check_strict.py" correctness "$WORK/correctness" | tee "$RESULTS/strict-check.md" - name: Benchmark run: | read -ra variants <<<"$VARIANTS" "$BENCH/bench.sh" "$WORK/project" "$RESULTS/runs" "$ROUNDS" "${variants[@]}" + - name: Prove dbt v2 strict analysis ran (benchmark) + if: ${{ contains(inputs.variants, 'strict') }} + run: python3 "$BENCH/check_strict.py" runs "$RESULTS/runs" | tee -a "$RESULTS/strict-check.md" - name: Summarize if: ${{ !cancelled() }} run: | @@ -139,6 +151,8 @@ jobs: echo cat "$RESULTS/correctness.md" 2>/dev/null || echo "Not run." echo + cat "$RESULTS/strict-check.md" 2>/dev/null || echo "Strict check not run." + echo echo "## Benchmark (${MODELS} models, medians of ${ROUNDS} rounds)" echo python3 "$BENCH/summarize.py" "$RESULTS/runs" 2>&1 || echo "No benchmark results." diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md index 683e3f9be2..f0c437c0e0 100644 --- a/benchmarks/dbt-compare/README.md +++ b/benchmarks/dbt-compare/README.md @@ -1,19 +1,20 @@ # SQLBuild vs dbt: dense project compile benchmark This harness generates one dense synthetic project in both SQLBuild and dbt form, then times the -same edit loop in each tool. It compares SQLBuild with dbt v2 (the `dbt` package on PyPI), with -dbt Core as a reference. Published results are on the +same edit loop in each tool. The comparison is with dbt v2 (the `dbt` package on PyPI); dbt Core +variants are available as a reference but don't run by default. Published results are on the [Benchmarks](https://sqlbuild.com/docs/benchmarks) page. ## Files | File | Purpose | |---|---| -| `write_dbt_dense.py` | Writes `sqlbuild/`, `sqlbuild-norules/`, `dbt/`, `warehouse.duckdb` and `profiles/` for N models | +| `write_dbt_dense.py` | Writes `sqlbuild/`, `sqlbuild-norules/`, `dbt/`, `dbt-remote/`, `warehouse.duckdb` and `profiles/` for N models | | `edit_models.py` | Changes one constant in the SQL body of N evenly spaced models, in either project | | `bench.sh` | Times cold, warm, one-model edit and 100-model edit runs per variant, in alternating order | | `summarize.py` | Prints median wall time, CPU time and peak RSS per variant and scenario | | `correctness.sh` | Runs every tool on 50-model projects: clean, a missing `ref`, and a missing column | +| `check_strict.py` | Fails unless dbt v2 strict static analysis demonstrably ran (see below) | ## The project @@ -38,12 +39,26 @@ average. `sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules; column binding, type inference and contract checks still run. +In `dbt/`, the source declares `config: {schema_origin: local}`, so dbt v2 takes source columns +from their declared `data_type`s, as SQLBuild takes them from its source declarations, without +reading the warehouse. `dbt-remote/` is the same project without that setting: dbt's default, which +downloads every source's schema from the warehouse. Only `v2_compile_strict_remote` uses it. + +`warehouse.duckdb` looks like the warehouse after `dbt build`: every source table and every model +relation exists, empty, with the declared columns, types and `not null` constraints. Each model is a +table or a view to match its materialization (all dense models are tables). dbt reads it through +`profiles/profiles.yml`; every dbt variant uses the same file. dbt v2 reads columns of unselected +upstream models from it in `--static-analysis strict --dirty` runs. `sqb compile` doesn't open it: +compile is offline, and its output is identical with the file present or removed. + ## Running it on GitHub Actions The `Benchmark dbt comparison` workflow (`.github/workflows/benchmark-dbt.yml`) runs only when -started by hand. It installs the requested released versions from PyPI, runs `correctness.sh`, then -`bench.sh`, and uploads the results directory with a `summary.md` (environment, correctness table, -benchmark table). The same summary appears on the run page. +started by hand. It installs the requested released versions from PyPI (dbt Core only when a +`core_*` variant is requested), runs `correctness.sh` and `check_strict.py correctness`, then +`bench.sh` and `check_strict.py runs`, and uploads the results directory with a `summary.md` +(environment, correctness table, strict check, benchmark table). The same summary appears on the +run page. ```bash gh workflow run benchmark-dbt.yml -f sqlbuild_version=0.164.2 -f rounds=3 @@ -53,11 +68,11 @@ gh workflow run benchmark-dbt.yml -f sqlbuild_version=0.164.2 -f rounds=3 |---|---| | `sqlbuild_version` | empty: the latest release | | `dbt_version` | `2.0.9` | -| `dbt_core_version` | `1.12.6` | -| `dbt_duckdb_version` | `1.11.0` | +| `dbt_core_version` | `1.12.6` (only for `core_*` variants) | +| `dbt_duckdb_version` | `1.11.0` (only for `core_*` variants) | | `models` | `10000` | | `rounds` | `3` | -| `variants` | all eight variants below | +| `variants` | `sqb_norules sqb_norules_full sqb_rules sqb_rules_full v2_compile_cached v2_compile_strict_cached v2_compile_dirty v2_compile_strict_dirty v2_compile v2_compile_strict v2_compile_strict_remote` | ## Running it locally @@ -66,17 +81,22 @@ Use the released packages in fresh virtual environments. From the repository roo ```bash uv venv -p 3.12 /path/to/venv-sqb && uv pip install -p /path/to/venv-sqb/bin/python sqlbuild pytest duckdb uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt -uv venv -p 3.12 /path/to/venv-dbt && uv pip install -p /path/to/venv-dbt/bin/python dbt-core dbt-duckdb export SQB=/path/to/venv-sqb/bin/sqb DBT_V2=/path/to/venv-dbt2/bin/dbt \ - DBT_CORE=/path/to/venv-dbt/bin/dbt PYTHON=/path/to/venv-sqb/bin/python + PYTHON=/path/to/venv-sqb/bin/python "$PYTHON" benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k benchmarks/dbt-compare/correctness.sh /path/to/correctness +python3 benchmarks/dbt-compare/check_strict.py correctness /path/to/correctness benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3 +python3 benchmarks/dbt-compare/check_strict.py runs /path/to/results python3 benchmarks/dbt-compare/summarize.py /path/to/results ``` +For the dbt Core reference rows, install `dbt-core dbt-duckdb` in another environment, set +`DBT_CORE` to its `dbt`, and pass `core_parse core_compile` to `bench.sh`. `correctness.sh` adds dbt +Core rows when `DBT_CORE` is set. + The generator imports the CI helpers from the checkout, so it needs a Python with `sqlbuild`, `duckdb` and `pytest` installed, and works best with a sqlbuild release close to the checkout. Nothing is built from source. Run on an otherwise idle machine: other load changes wall times far @@ -88,7 +108,8 @@ For every variant and round, `bench.sh` copies the pristine project into `work/V five seconds, then runs: 1. **cold**: the command with no `target/` and no caches. -2. **warm**: the same command again with nothing changed. +2. **warm**: the same command again with nothing changed. SQLBuild's default replays the stored + result of the previous compile; the summary labels these rows "warm (replay)". 3. **edit1**: one model's SQL body edited, then the command. 4. **edit100**: 100 models' SQL bodies edited (1% of 10,000), then the command. @@ -96,33 +117,97 @@ Each edit changes one constant in a projection, so the query changes but its col references don't. Rounds alternate the variant order. `SCENARIOS` limits which of warm, edit1 and edit100 run. -Commands, run from the project copy: +Commands, run from the project copy. Every dbt command also gets +`--profiles-dir ../profiles --no-version-check`. | Variant | Command | |---|---| -| `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` | -| `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` | -| `sqb_rules_full` | `sqb_rules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile that uses the per-model caches | -| `v2_parse` / `v2_compile` | dbt v2 `parse` / `compile` with `--profiles-dir ../profiles --no-version-check` | -| `v2_compile_strict` | dbt v2 `compile --static-analysis strict`, which checks columns like SQLBuild does | -| `core_parse` / `core_compile` | the same `parse` / `compile` commands with dbt Core | - -Both tools use their defaults otherwise: SQLBuild's compile cache, reuse and delta compile, and -dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets -`threads: 4`. +| `sqb_norules` | SQLBuild default (compile with column and type checks): `sqb --no-color compile` in `sqlbuild-norules/` | +| `sqb_norules_full` | `sqb_norules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile with per-model caches | +| `sqb_rules` | SQLBuild with every built-in rule and a custom Python rule: `sqb --no-color compile` in `sqlbuild/` | +| `sqb_rules_full` | `sqb_rules` as a full compile with per-model caches, as for `sqb_norules_full` | +| `v2_compile_cached` | dbt v2 with parse cache (opt-in; compiles the whole project): `compile --partial-parse --partial-load` | +| `v2_compile_strict_cached` | the same with `--static-analysis strict` | +| `v2_compile_dirty` | dbt v2 `--dirty` (changed models and descendants only). Cold: `compile --partial-parse --partial-load`; then `compile --partial-load --dirty` | +| `v2_compile_strict_dirty` | the same with `--static-analysis strict` on every run | +| `v2_compile` | dbt v2 default: `compile` | +| `v2_compile_strict` | dbt v2 default with `--static-analysis strict`, which binds columns and types; contract and nullability checks differ | +| `v2_compile_strict_remote` | `v2_compile_strict` in `dbt-remote/`: source schemas downloaded from the warehouse on every run. Default mode, because the parse cache doesn't skip the download either, so the difference from `v2_compile_strict` is the introspection cost | +| `v2_parse` | dbt v2 `parse` (not run by default) | +| `core_parse` / `core_compile` | dbt Core `parse` / `compile` (not run by default) | + +SQLBuild uses its defaults: the compile cache, replaying an unchanged compile, and compiling only +changed files after an edit. In SQLBuild's default, configurable rules are off (`sqb_norules`). dbt Core uses its default partial parsing. +`DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets `threads: 4`. + +### dbt v2 modes + +dbt v2 2.0.9 has three ways to run `compile`. The benchmark times each, with and without +`--static-analysis strict`, in this order: + +- **With parse cache** (`*_cached`): `--partial-parse` keeps a parse cache under + `target/private/metadata/` and `--partial-load` loads it faster. The cache is opt-in, and + `--partial-parse` is hidden from `dbt --help` in 2.0.9. The cold run builds the cache. Every later + run still compiles the whole project. +- **`--dirty`** (`*_dirty`): selects only models whose files changed, by modification time, since the + last `--partial-parse` run, plus all their downstream dependents. It needs that cache, so the cold + run is a `--partial-parse --partial-load` compile and is timed as cold. With nothing changed it + reports `NoNodesSelected` ("Nothing to do"). Each run refreshes the cache, so each edit is compared + with the run before it. Dense models form long `ref` chains, so one edited model can select dozens + of descendants. +- In both cached modes, when more than 100 files changed since the cache was written, dbt 2.0.9 logs + `[partial-parse] ... falling back to full parse`. The summary marks those runs "full parse", and + `check_strict.py` ignores `[partial-parse]` lines. +- **Default** (`v2_compile`, `v2_compile_strict`): no cache between runs. Every run parses and + compiles the whole project. + +### Proving strict analysis ran + +Strict static analysis is only in the `dbt` distribution, and a run without it can fall back to +baseline analysis. `check_strict.py` fails the workflow unless: + +- `correctness`: in the `column` project, strict exits non-zero with `UnresolvedIdentifier (dbt0227)` + in default mode, with sources from the warehouse, and on a warm run after the error is introduced + in the cached and `--dirty` modes. The same project exits 0 with plain `dbt compile`, and the + `clean` project exits 0 under strict in every mode. +- `runs`: every timed strict run exits 0. +- Both: no strict log or output line matches any of these patterns (case-insensitive): + `did not run`, `fall back` / `falling back` / `fallback`, `baseline`, `login` / `log in`, + `authenticat`, `does not include the static analysis`, `requires the full dbt distribution`, + `static_analysis' to off`, `dbt1014`, `Skipping analysis`. `/usr/bin/time -f '%e %U %S %M'` records wall time, user and system CPU, and the peak RSS of the largest process. `summarize.py` also reports what each tool said it did: SQLBuild prints whether it -reused the previous compile or compiled only the changed files, and dbt Core logs when it falls back -to a full parse. The 1-minute load average before and after each run is recorded too. +replayed the previous compile or compiled only the changed files, dbt Core logs when it falls back +to a full parse, and dbt v2 prints how many models it processed. For dbt v2 the column also counts +models it skipped static analysis for. The 1-minute load average before and after each run is +recorded too. ## What isn't equivalent -- SQLBuild compile binds every column, infers types and checks contracts. dbt v2 does this with - `--static-analysis strict`; its default compile doesn't report unknown columns. dbt Core compile - renders Jinja and resolves references without analysing the SQL. +- SQLBuild compile binds every column, infers types and checks contracts. dbt v2 binds columns and + types with `--static-analysis strict`; its default compile doesn't report unknown columns. dbt + Core compile renders Jinja and resolves references without analysing the SQL. +- SQLBuild compile reports contract, type and nullability problems as errors. dbt v2 strict warns on + contract type mismatches and checks enforced contracts at run time. +- SQLBuild compile also plans and binds its 10,000 SQL tests. dbt `compile` excludes unit tests; they + are rendered in `dbt test` or `dbt build`. +- SQLBuild compile computes column lineage by default. +- The measured dbt is the `dbt` distribution from PyPI, under dbt Labs' licence. Strict static + analysis isn't in the Apache-2.0 build. +- `dbt/` declares source columns locally, as SQLBuild does. `v2_compile_strict_remote` shows dbt's + default of downloading source schemas from the warehouse. +- `--dirty` compiles the changed nodes and all their descendants, and leaves the rest of `target/` + from earlier runs; its run summary covers only the selected nodes. SQLBuild's delta compile + re-analyses the changed models plus any model whose inputs' columns, types or nullability changed, + re-runs project-wide Rules, and reports the same result as a full compile. +- With `--static-analysis strict --dirty`, dbt v2 checks the selected models against the columns of + unselected upstream relations in the warehouse, so it depends on the warehouse matching the + project. When an upstream relation is missing, dbt warns `RemoteError (dbt1014)` and turns static + analysis off for that model; the summary counts such models, and with this warehouse it should + report none. +- dbt v2's parse cache and `--dirty` are opt-in; SQLBuild's caching is on by default. SQLBuild's warm + run replays the stored result. - SQLBuild compile also runs configurable Rules (`sqb_rules`). dbt has no built-in counterpart. -- SQLBuild compile prepares its 10,000 SQL tests. dbt `compile` parses the unit tests but doesn't - render them; that happens in `dbt test` or `dbt build`. - dbt renders every model through Jinja; SQLBuild expands references and macros in its own compiler. That cost is part of each tool and is included. diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh index 0a77ec6c3e..62f989709b 100755 --- a/benchmarks/dbt-compare/bench.sh +++ b/benchmarks/dbt-compare/bench.sh @@ -13,13 +13,14 @@ # # Environment: # SQB sqb executable (default: sqb) -# DBT_CORE dbt Core executable (default: dbt) # DBT_V2 dbt v2 executable (default: dbt2) +# DBT_CORE dbt Core executable, for the core_* variants (default: dbt) # PYTHON Python used for the edit helper (default: python3) # SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three) # -# Variants: sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse -# core_compile. With no VARIANT arguments, all of them run. +# Default variants: sqb_norules sqb_norules_full sqb_rules sqb_rules_full v2_compile_cached +# v2_compile_strict_cached v2_compile_dirty v2_compile_strict_dirty v2_compile v2_compile_strict +# v2_compile_strict_remote. Also available: v2_parse core_parse core_compile. set -u ROOT=$(realpath "$1") OUT=$(realpath -m "$2") @@ -35,13 +36,16 @@ SCENARIOS=" ${SCENARIOS:-warm edit1 edit100} " MODELS=$(find "$ROOT/sqlbuild/models" -name '*.sql' | wc -l) export DBT_SEND_ANONYMOUS_USAGE_STATS=false DBTFLAGS="--profiles-dir $ROOT/profiles --no-version-check" -declare -A SRC CMD +# CMD_COLD overrides CMD for the cold scenario. +declare -A SRC CMD CMD_COLD SRC[sqb_rules]=sqlbuild CMD[sqb_rules]="$SQB --no-color compile" SRC[sqb_norules]=sqlbuild-norules CMD[sqb_norules]="$SQB --no-color compile" SRC[sqb_rules_full]=sqlbuild CMD[sqb_rules_full]="env SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1 $SQB --no-color compile" +SRC[sqb_norules_full]=sqlbuild-norules +CMD[sqb_norules_full]="env SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1 $SQB --no-color compile" SRC[core_parse]=dbt CMD[core_parse]="$DBT_CORE parse $DBTFLAGS" SRC[core_compile]=dbt @@ -52,11 +56,30 @@ SRC[v2_compile]=dbt CMD[v2_compile]="$DBT_V2 compile $DBTFLAGS" SRC[v2_compile_strict]=dbt CMD[v2_compile_strict]="$DBT_V2 compile $DBTFLAGS --static-analysis strict" +# dbt/ declares source columns locally (schema_origin: local). dbt-remote/ reads them from the +# warehouse, dbt's default; v2_compile_strict_remote shows that introspection cost in default mode, +# which re-reads source schemas on every run. +SRC[v2_compile_strict_remote]=dbt-remote +CMD[v2_compile_strict_remote]="$DBT_V2 compile $DBTFLAGS --static-analysis strict" +# dbt v2's parse cache is off by default. --partial-parse keeps it under target/private/metadata/; +# --partial-load loads it faster. --dirty selects only nodes whose files changed since the last +# --partial-parse run, plus their descendants, so its cold run builds the cache without it. +SRC[v2_compile_cached]=dbt +CMD[v2_compile_cached]="$DBT_V2 compile $DBTFLAGS --partial-parse --partial-load" +SRC[v2_compile_strict_cached]=dbt +CMD[v2_compile_strict_cached]="$DBT_V2 compile $DBTFLAGS --static-analysis strict --partial-parse --partial-load" +SRC[v2_compile_dirty]=dbt +CMD_COLD[v2_compile_dirty]="$DBT_V2 compile $DBTFLAGS --partial-parse --partial-load" +CMD[v2_compile_dirty]="$DBT_V2 compile $DBTFLAGS --partial-load --dirty" +SRC[v2_compile_strict_dirty]=dbt +CMD_COLD[v2_compile_strict_dirty]="$DBT_V2 compile $DBTFLAGS --static-analysis strict --partial-parse --partial-load" +CMD[v2_compile_strict_dirty]="$DBT_V2 compile $DBTFLAGS --static-analysis strict --partial-load --dirty" VARIANTS=("$@") if [ ${#VARIANTS[@]} -eq 0 ]; then - VARIANTS=(sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict - core_parse core_compile) + VARIANTS=(sqb_norules sqb_norules_full sqb_rules sqb_rules_full v2_compile_cached + v2_compile_strict_cached v2_compile_dirty v2_compile_strict_dirty v2_compile v2_compile_strict + v2_compile_strict_remote) fi for v in "${VARIANTS[@]}"; do if [ -z "${CMD[$v]+set}" ]; then @@ -69,12 +92,15 @@ mkdir -p "$OUT" "$WORK" load1() { cut -d' ' -f1 /proc/loadavg; } measure() { - local v=$1 scenario=$2 round=$3 base + local v=$1 scenario=$2 round=$3 base cmd base="$OUT/$v.$scenario.r$round" + cmd=${CMD[$v]} + if [ "$scenario" = cold ] && [ -n "${CMD_COLD[$v]+set}" ]; then cmd=${CMD_COLD[$v]}; fi + echo "$cmd" >"$base.cmd" local before after before=$(load1) /usr/bin/time -f '%e %U %S %M' -o "$base.time" \ - bash -c "cd '$WORK/$v' && exec ${CMD[$v]}" >"$base.out" 2>"$base.err" + bash -c "cd '$WORK/$v' && exec $cmd" >"$base.out" 2>"$base.err" echo $? >"$base.rc" after=$(load1) echo "$v $scenario r$round load $before $after" >>"$OUT/uptime.log" diff --git a/benchmarks/dbt-compare/check_strict.py b/benchmarks/dbt-compare/check_strict.py new file mode 100644 index 0000000000..73e152cfbb --- /dev/null +++ b/benchmarks/dbt-compare/check_strict.py @@ -0,0 +1,108 @@ +"""Fail unless dbt v2 strict static analysis really ran. + +Usage: + python check_strict.py correctness CORRECTNESS_DIR + python check_strict.py runs RESULTS_RUNS_DIR + +`correctness` checks correctness.sh logs: strict reports the seeded unknown column (dbt0227) in +default, cached and dirty modes, the same project compiles with exit 0 without strict, and the +clean project passes strict. `runs` checks bench.sh output: every strict run exits 0. Both fail if +any strict output matches FALLBACK_PATTERNS, which would mean strict analysis fell back or was off. +""" + +import re +import sys +from pathlib import Path + +FALLBACK_PATTERNS = ( + r"did not run", + r"fall(?:s|ing)? ?back", + r"\bbaseline\b", + r"\blog ?in\b", + r"authenticat", + r"does not include the static analysis", + r"requires the full dbt distribution", + r"static_analysis'? to off", + r"dbt1014", + r"Skipping analysis", +) +FALLBACK_RE = re.compile("|".join(FALLBACK_PATTERNS), re.IGNORECASE) +# Parse-cache messages, such as a partial parse falling back to a full parse, aren't about analysis. +IGNORED_RE = re.compile(r"^\[partial-parse\]") +UNKNOWN_COLUMN = "UnresolvedIdentifier (dbt0227)" +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def read(path: Path) -> str: + return ANSI_RE.sub("", path.read_text(errors="replace")) + + +def fallback_lines(text: str) -> list[str]: + lines = (line.strip() for line in text.splitlines()) + return [line for line in lines if FALLBACK_RE.search(line) and not IGNORED_RE.match(line)] + + +def check_correctness(root: Path) -> list[str]: + logs = root / "logs" + errors: list[str] = [] + + def result(name: str) -> tuple[int, str]: + log = logs / f"{name}.log" + if not log.exists(): + errors.append(f"{name}: missing log") + return -1, "" + return int((logs / f"{name}.rc").read_text()), read(log) + + for name in ( + "column.v2_strict", + "column.v2_strict_remote", + "column.v2_strict_cached_warm", + "column.v2_strict_dirty_warm", + ): + rc, text = result(name) + if rc == 0 or UNKNOWN_COLUMN not in text: + errors.append(f"{name}: expected a non-zero exit with {UNKNOWN_COLUMN}, got rc={rc}") + for name in ( + "column.v2_compile", + "clean.v2_strict", + "clean.v2_strict_remote", + "clean.v2_strict_cached_cold", + "clean.v2_strict_dirty_cold", + ): + rc, _text = result(name) + if rc != 0: + errors.append(f"{name}: expected exit 0, got rc={rc}") + for log in sorted(logs.glob("*strict*.log")): + errors += [f"{log.name}: {line}" for line in fallback_lines(read(log))] + return errors + + +def check_runs(root: Path) -> list[str]: + errors: list[str] = [] + runs = sorted(root.glob("*strict*.rc")) + if not runs: + errors.append("no strict runs found") + for rc_path in runs: + stem = str(rc_path)[: -len(".rc")] + rc = int(rc_path.read_text()) + if rc != 0: + errors.append(f"{Path(stem).name}: exit code {rc}") + text = read(Path(stem + ".out")) + read(Path(stem + ".err")) + errors += [f"{Path(stem).name}: {line}" for line in fallback_lines(text)] + return errors + + +def main() -> None: + if len(sys.argv) != 3 or sys.argv[1] not in ("correctness", "runs"): + sys.exit(__doc__) + mode, root = sys.argv[1], Path(sys.argv[2]) + errors = check_correctness(root) if mode == "correctness" else check_runs(root) + if errors: + print("dbt v2 strict static analysis check failed:") + print("\n".join(f"- {error}" for error in errors)) + sys.exit(1) + print(f"dbt v2 strict static analysis check passed ({mode}).") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/dbt-compare/correctness.sh b/benchmarks/dbt-compare/correctness.sh index 235b8fde2a..57f8f52d56 100755 --- a/benchmarks/dbt-compare/correctness.sh +++ b/benchmarks/dbt-compare/correctness.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash # Correctness matrix on 50-model projects: clean, a reference to a missing model, and a column the -# upstream model does not have. Run from the repository root. +# upstream model does not have. Run from the repository root. check_strict.py reads the logs. # # Usage: correctness.sh OUTROOT -# Environment: SQB, DBT_V2, DBT_CORE and PYTHON, as for bench.sh. PYTHON must be able to -# import duckdb, pytest and the installed sqlbuild package (the generator uses the CI helpers). +# Environment: SQB, DBT_V2 and PYTHON, as for bench.sh; set DBT_CORE to add dbt Core rows. PYTHON +# must be able to import duckdb, pytest and the installed sqlbuild package (the generator uses the +# CI helpers). set -u OUT=$(realpath -m "$1") HERE=$(cd "$(dirname "$0")" && pwd) SQB=${SQB:-sqb} -DBT_CORE=${DBT_CORE:-dbt} +DBT_CORE=${DBT_CORE:-} DBT_V2=${DBT_V2:-dbt2} PYTHON=${PYTHON:-python3} export DBT_SEND_ANONYMOUS_USAGE_STATS=false @@ -26,13 +27,16 @@ first_problem() { sed -E 's/^ +//' | cut -c1-140 } +# run KIND ID LABEL DIR COMMAND...: logs to logs/KIND.ID.log and the exit code to logs/KIND.ID.rc. +# Set KEEP_TARGET=1 to keep DIR's target/ from an earlier run. run() { - local kind=$1 label=$2 dir=$3 - shift 3 - local log="$OUT/logs/$kind.$label.log" - rm -rf "$dir/target" "$dir/logs" + local kind=$1 id=$2 label=$3 dir=$4 + shift 4 + local log="$OUT/logs/$kind.$id.log" + [ "${KEEP_TARGET:-0}" = 1 ] || rm -rf "$dir/target" "$dir/logs" (cd "$dir" && "$@" >"$log" 2>&1) local rc=$? + echo "$rc" >"$OUT/logs/$kind.$id.rc" printf '| %s | %s | %s | %s |\n' "$kind" "$label" "$rc" "$(first_problem "$log")" } @@ -41,12 +45,38 @@ echo "|---|---|---|---|" for kind in clean ref column; do P=$OUT/p50-$kind DBTFLAGS=(--profiles-dir "$P/profiles" --no-version-check) - run "$kind" "sqb compile (rules on)" "$P/sqlbuild" "$SQB" --no-color compile - run "$kind" "sqb compile (rules off)" "$P/sqlbuild-norules" "$SQB" --no-color compile - run "$kind" "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" - run "$kind" "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" - run "$kind" "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ - "${DBTFLAGS[@]}" --static-analysis strict - run "$kind" "dbt Core parse" "$P/dbt" "$DBT_CORE" parse "${DBTFLAGS[@]}" - run "$kind" "dbt Core compile" "$P/dbt" "$DBT_CORE" compile "${DBTFLAGS[@]}" + STRICT=(--static-analysis strict) + run "$kind" sqb_rules "sqb compile (rules on)" "$P/sqlbuild" "$SQB" --no-color compile + run "$kind" sqb_norules "sqb compile (rules off)" "$P/sqlbuild-norules" "$SQB" --no-color compile + run "$kind" v2_parse "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" + run "$kind" v2_compile "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" + run "$kind" v2_strict "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ + "${DBTFLAGS[@]}" "${STRICT[@]}" + run "$kind" v2_strict_remote "dbt v2 compile --static-analysis strict, sources from warehouse" \ + "$P/dbt-remote" "$DBT_V2" compile "${DBTFLAGS[@]}" "${STRICT[@]}" + if [ -n "$DBT_CORE" ]; then + run "$kind" core_parse "dbt Core parse" "$P/dbt" "$DBT_CORE" parse "${DBTFLAGS[@]}" + run "$kind" core_compile "dbt Core compile" "$P/dbt" "$DBT_CORE" compile "${DBTFLAGS[@]}" + fi +done + +# The cached and dirty strict modes must catch the column error on a warm run: build the cache on +# the clean project, copy in the seeded model, then compile again. +P=$OUT/p50-clean +DBTFLAGS=(--profiles-dir "$P/profiles" --no-version-check) +seeded=$(cd "$OUT/p50-column/dbt" && find models -name '*.sql' -exec grep -l missing_col {} +) +for mode in cached dirty; do + W=$OUT/warm-$mode + rm -rf "$W" + cp -a "$P/dbt" "$W" + run clean "v2_strict_${mode}_cold" "dbt v2 strict $mode: build the cache" "$W" "$DBT_V2" compile \ + "${DBTFLAGS[@]}" --static-analysis strict --partial-parse --partial-load + cp "$OUT/p50-column/dbt/$seeded" "$W/$seeded" + if [ "$mode" = cached ]; then + flags=(--partial-parse --partial-load) + else + flags=(--partial-load --dirty) + fi + KEEP_TARGET=1 run column "v2_strict_${mode}_warm" "dbt v2 strict $mode: after seeding the error" \ + "$W" "$DBT_V2" compile "${DBTFLAGS[@]}" --static-analysis strict "${flags[@]}" done diff --git a/benchmarks/dbt-compare/summarize.py b/benchmarks/dbt-compare/summarize.py index fb9fd89983..d5b406b697 100644 --- a/benchmarks/dbt-compare/summarize.py +++ b/benchmarks/dbt-compare/summarize.py @@ -11,24 +11,60 @@ SCENARIOS = ("cold", "warm", "edit1", "edit100") VARIANT_ORDER = ( - "sqb_rules", "sqb_norules", + "sqb_norules_full", + "sqb_rules", "sqb_rules_full", - "v2_parse", + "v2_compile_cached", + "v2_compile_strict_cached", + "v2_compile_dirty", + "v2_compile_strict_dirty", "v2_compile", "v2_compile_strict", + "v2_compile_strict_remote", + "v2_parse", "core_parse", "core_compile", ) +SQB_DEFAULT = "SQLBuild (default: compile with column and type checks)" +SQB_RULES = "SQLBuild with every built-in rule and a custom Python rule" +FULL = ", full compile with per-model caches" +LABELS = { + "sqb_norules": SQB_DEFAULT, + "sqb_norules_full": SQB_DEFAULT + FULL, + "sqb_rules": SQB_RULES, + "sqb_rules_full": SQB_RULES + FULL, + "v2_compile_cached": "dbt v2 with parse cache (opt-in; compiles the whole project)", + "v2_compile_strict_cached": "dbt v2 with parse cache, strict", + "v2_compile_dirty": "dbt v2 --dirty (changed models and descendants only)", + "v2_compile_strict_dirty": "dbt v2 --dirty, strict", + "v2_compile": "dbt v2 default", + "v2_compile_strict": "dbt v2 default, strict", + "v2_compile_strict_remote": "dbt v2 default, strict, source columns read from the warehouse", + "v2_parse": "dbt v2 parse", + "core_parse": "dbt Core parse", + "core_compile": "dbt Core compile", +} +# SQLBuild's default warm run replays the stored result of the previous compile. +REPLAY_VARIANTS = ("sqb_norules", "sqb_rules") +PROCESSED_RE = re.compile(r"Processed: (\d+) models?") +SKIPPED_ANALYSIS_RE = re.compile(r"Skipping analysis for '([^']+)'") +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") TIME_RE = re.compile(r"^(?P\w+)\.(?P\w+)\.r(?P\d+)\.time$") LOAD_RE = re.compile(r"^(\w+) (\w+) r\d+ load ([\d.]+) ([\d.]+)$") def path_taken(variant: str, out: str, err: str) -> str: - """What each tool reported doing: full work, a partial parse, a replay or a delta compile.""" + """What each tool reported doing. + + SQLBuild: a full compile, a replay of the stored result or a delta compile. + dbt Core: a full or partial parse. + dbt v2: how many models it processed, how many it skipped static analysis for, whether its + parse cache fell back to a full parse, or that `--dirty` found nothing to do. + """ if variant.startswith("sqb"): if "Inputs unchanged; reused the previous compile" in err: - return "reused" + return "replay" if "against the previous compile" in err: return "delta" return "full" @@ -37,6 +73,17 @@ def path_taken(variant: str, out: str, err: str) -> str: if "Unable to do partial parsing" in text or "Starting full parse" in text: return "full parse" return "partial parse" + if variant.startswith("v2"): + text = ANSI_RE.sub("", out + err) + processed = PROCESSED_RE.search(text) + if processed: + skipped = len(set(SKIPPED_ANALYSIS_RE.findall(text))) + note = f", analysis skipped for {skipped}" if skipped else "" + if "falling back to full parse" in text: + note += ", full parse" + return f"{processed[1]} models{note}" + if "NoNodesSelected" in text: + return "nothing to do" return "-" @@ -66,10 +113,10 @@ def main() -> None: loads[(m[1], m[2])] += [float(m[3]), float(m[4])] print( - "| variant | scenario | runs | wall s (median) | wall s (all) | user+sys s (median) " - "| peak RSS MiB (median) | exit codes | path | 1-min load before/after |" + "| tool | variant | scenario | runs | wall s (median) | wall s (all) | user+sys s (median) " + "| peak RSS MiB (median) | exit codes | what the tool did | 1-min load before/after |" ) - print("|---|---|---|---|---|---|---|---|---|---|") + print("|---|---|---|---|---|---|---|---|---|---|---|") found = {variant for variant, _ in runs} variants = [v for v in VARIANT_ORDER if v in found] + sorted(found - set(VARIANT_ORDER)) for variant in variants: @@ -78,9 +125,13 @@ def main() -> None: if not cell: continue walls = [r[0] for r in cell] + scenario_label = scenario + if scenario == "warm" and variant in REPLAY_VARIANTS: + scenario_label = "warm (replay)" load = loads.get((variant, scenario), [0.0]) print( - f"| {variant} | {scenario} | {len(cell)} | {statistics.median(walls):.2f} " + f"| {LABELS.get(variant, variant)} | {variant} | {scenario_label} | {len(cell)} " + f"| {statistics.median(walls):.2f} " f"| {', '.join(f'{w:.1f}' for w in walls)} " f"| {statistics.median(r[1] for r in cell):.1f} " f"| {statistics.median(r[2] for r in cell) / 1024:.0f} " diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py index aa0801bd76..6fa7cbb0d7 100644 --- a/benchmarks/dbt-compare/write_dbt_dense.py +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -8,8 +8,10 @@ Creates under DIR: sqlbuild/ SQLBuild dense project (the CI dense generator, adapter switched to duckdb) sqlbuild-norules/ the same project with `[rules] select = []` - dbt/ dbt project derived file by file from `sqlbuild/` - warehouse.duckdb DuckDB file holding every source table (empty) + dbt/ dbt project derived file by file from `sqlbuild/`; sources use their declared + columns (`schema_origin: local`), as SQLBuild does + dbt-remote/ the same dbt project with sources read from the warehouse (dbt's default) + warehouse.duckdb DuckDB file holding every source and model relation (empty), as after a build profiles/profiles.yml The SQLBuild project comes unchanged from the generator the compile-performance CI jobs use. The @@ -138,6 +140,8 @@ def write_dbt(sqb: Path, out: Path, n: int) -> Path: f" - name: {SOURCE_NAME}", f" database: {DATABASE}", f" schema: {SOURCE_SCHEMA}", + " config:", + " schema_origin: local", " tables:", ] for index in range(source_count): @@ -272,7 +276,39 @@ def write_dbt(sqb: Path, out: Path, n: int) -> Path: return dbt -def write_warehouse(out: Path, n: int) -> None: +MATERIALIZED_RE = re.compile(r"\bmaterialized (table|view)\b") + + +def model_relations(sqb: Path) -> list[tuple[str, str, list[tuple[str, str, bool, str, bool]]]]: + """Name, materialization and columns of every model, read from the SQLBuild headers.""" + relations = [] + for path in sorted((sqb / "models").rglob("*.sql")): + text = path.read_text() + materialized = MATERIALIZED_RE.search(text.split("\n", 1)[0]) + assert materialized, path + columns, _body = model_header_and_body(text) + relations.append((path.stem, materialized[1], columns)) + return relations + + +def write_dbt_remote(dbt: Path) -> Path: + """Copy the dbt project with source columns read from the warehouse, dbt's default.""" + target = dbt.parent / "dbt-remote" + shutil.copytree(dbt, target) + sources = target / "models" / "_sources.yml" + text = sources.read_text() + new = text.replace(" config:\n schema_origin: local\n", "", 1) + assert new != text + sources.write_text(new) + return target + + +def write_warehouse(out: Path, n: int, sqb: Path) -> None: + """Create every source and model relation empty, as the warehouse looks after `dbt build`. + + dbt v2 reads the columns of unselected upstream models from the warehouse in + `--static-analysis strict --dirty` runs, so the model relations must exist. + """ import duckdb path = (out / f"{DATABASE}.duckdb").resolve() @@ -287,6 +323,16 @@ def write_warehouse(out: Path, n: int) -> None: f"quantity_{c:03d} double" for c in range(width - 2) ] con.execute(f"create table {SOURCE_SCHEMA}.orders_{index:05d} ({', '.join(cols)})") + for name, materialized, columns in model_relations(sqb): + if materialized == "table": + cols = [ + f"{col} {typ.lower()}{' not null' if not_null else ''}" + for col, typ, not_null, _desc, _audit in columns + ] + con.execute(f"create table analytics.{name} ({', '.join(cols)})") + else: + cols = [f"cast(null as {typ.lower()}) as {col}" for col, typ, *_ in columns] + con.execute(f"create view analytics.{name} as select {', '.join(cols)} where false") con.close() profiles = out / "profiles" profiles.mkdir(exist_ok=True) @@ -304,7 +350,7 @@ def seed_error(out: Path, kind: str) -> None: """ name = dense_model_name(ERROR_MODEL_INDEX) upstream = dense_model_name(ERROR_MODEL_INDEX - 1) - for tool in ("sqlbuild", "sqlbuild-norules", "dbt"): + for tool in ("sqlbuild", "sqlbuild-norules", "dbt", "dbt-remote"): for path in (out / tool / "models").rglob(f"{name}.sql"): text = path.read_text() if kind == "column": @@ -331,9 +377,9 @@ def main() -> None: shutil.rmtree(args.out) args.out.mkdir(parents=True) sqb = write_sqlbuild(args.out, args.models) - write_dbt(sqb, args.out, args.models) + write_dbt_remote(write_dbt(sqb, args.out, args.models)) write_norules(sqb) - write_warehouse(args.out, args.models) + write_warehouse(args.out, args.models, sqb) if args.seed_error: seed_error(args.out, args.seed_error)