kvlonge · Oct 21, 2026 · Oct 21, 2026 · Oct 21, 2026 · Oct 21, 2026 · Oct 21, 2026
diff --git a/.github/workflows/benchmark-dbt.yml b/.github/workflows/benchmark-dbt.yml
@@ -0,0 +1,152 @@
+name: Benchmark dbt comparison
+
+# Manual only. Times SQLBuild, dbt v2 and dbt Core on the same generated dense DuckDB project with
+# released packages from PyPI, and publishes the tables as an artifact and in the job summary.
+# See benchmarks/dbt-compare/README.md.
+on:
+  workflow_dispatch:
+    inputs:
+      sqlbuild_version:
+        description: "sqlbuild version from PyPI (empty: latest release)"
+        required: false
+        default: ""
+        type: string
+      dbt_version:
+        description: "dbt (v2) version from PyPI"
+        required: false
+        default: "2.0.9"
+        type: string
+      dbt_core_version:
+        description: "dbt-core version from PyPI"
+        required: false
+        default: "1.12.6"
+        type: string
+      dbt_duckdb_version:
+        description: "dbt-duckdb version from PyPI"
+        required: false
+        default: "1.11.0"
+        type: string
+      models:
+        description: "Models in the generated project"
+        required: false
+        default: "10000"
+        type: string
+      rounds:
+        description: "Rounds per variant (the summary reports medians)"
+        required: false
+        default: "3"
+        type: string
+      variants:
+        description: "Space-separated bench.sh variants"
+        required: false
+        default: "sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse core_compile"
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: benchmark-dbt
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    name: SQLBuild vs dbt (${{ inputs.models }} models)
+    # The same runner type as the dense compile performance jobs in verify.yml.
+    runs-on: ubuntu-24.04
+    timeout-minutes: 300
+    env:
+      SQLBUILD_VERSION: ${{ inputs.sqlbuild_version }}
+      DBT_VERSION: ${{ inputs.dbt_version }}
+      DBT_CORE_VERSION: ${{ inputs.dbt_core_version }}
+      DBT_DUCKDB_VERSION: ${{ inputs.dbt_duckdb_version }}
+      MODELS: ${{ inputs.models }}
+      ROUNDS: ${{ inputs.rounds }}
+      VARIANTS: ${{ inputs.variants }}
+      BENCH: ${{ github.workspace }}/benchmarks/dbt-compare
+      DBT_SEND_ANONYMOUS_USAGE_STATS: "false"
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+      - name: Validate inputs
+        run: |
+          [[ $MODELS =~ ^[0-9]+$ && $MODELS -ge 100 ]] || { echo "models must be an integer >= 100"; exit 1; }
+          [[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || { echo "rounds must be a positive integer"; exit 1; }
+          for spec in "$SQLBUILD_VERSION" "$DBT_VERSION" "$DBT_CORE_VERSION" "$DBT_DUCKDB_VERSION"; do
+            [[ -z $spec || $spec =~ ^[0-9A-Za-z.+-]+$ ]] || { echo "invalid version: $spec"; exit 1; }
+          done
+      - name: Set paths
+        run: |
+          {
+            echo "WORK=$RUNNER_TEMP/dbt-compare"
+            echo "RESULTS=$RUNNER_TEMP/dbt-compare-results"
+            echo "SQB=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/sqb"
+            echo "DBT_V2=$RUNNER_TEMP/dbt-compare/venv-dbt2/bin/dbt"
+            echo "DBT_CORE=$RUNNER_TEMP/dbt-compare/venv-dbt/bin/dbt"
+            echo "PYTHON=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/python"
+          } >>"$GITHUB_ENV"
+      - name: Install released packages
+        run: |
+          uv python install 3.12
+          mkdir -p "$WORK" "$RESULTS"
+          sqlbuild_spec=sqlbuild${SQLBUILD_VERSION:+==$SQLBUILD_VERSION}
+          uv venv -q -p 3.12 "$WORK/venv-sqb"
+          uv pip install -q -p "$WORK/venv-sqb/bin/python" "$sqlbuild_spec" pytest duckdb
+          uv venv -q -p 3.12 "$WORK/venv-dbt2"
+          uv pip install -q -p "$WORK/venv-dbt2/bin/python" "dbt==$DBT_VERSION"
+          uv venv -q -p 3.12 "$WORK/venv-dbt"
+          uv pip install -q -p "$WORK/venv-dbt/bin/python" \
+            "dbt-core==$DBT_CORE_VERSION" "dbt-duckdb==$DBT_DUCKDB_VERSION"
+      - name: Record versions and runner
+        run: |
+          {
+            echo "## Environment"
+            echo
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo "- Commit: ${GITHUB_SHA}"
+            echo "- Runner: ${RUNNER_OS} ${ImageOS:-} ${ImageVersion:-}, $(nproc) CPUs ($(lscpu | sed -n 's/^Model name: *//p')), $(free -g | awk '/^Mem:/ {print $2}') GiB RAM"
+            echo "- Python: $("$WORK/venv-sqb/bin/python" -c 'import platform; print(platform.python_version())')"
+            echo "- sqlbuild: $("$WORK/venv-sqb/bin/python" -c 'import importlib.metadata as m; print(m.version("sqlbuild"))')"
+            echo "- dbt (v2): $("$WORK/venv-dbt2/bin/dbt" --version 2>&1 | head -1)"
+            echo "- dbt-core: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-core"))')"
+            echo "- dbt-duckdb: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-duckdb"))')"
+            echo "- Models: ${MODELS}, rounds: ${ROUNDS}, variants: ${VARIANTS}"
+            echo
+          } | tee "$RESULTS/environment.md"
+      - name: Generate the project
+        run: |
+          "$PYTHON" "$BENCH/write_dbt_dense.py" --models "$MODELS" --out "$WORK/project"
+      # Runs before the timed benchmark so dbt v2's one-time driver download is not timed.
+      - name: Correctness matrix
+        run: |
+          "$BENCH/correctness.sh" "$WORK/correctness" | tee "$RESULTS/correctness.md"
+          cp -r "$WORK/correctness/logs" "$RESULTS/correctness-logs"
+      - name: Benchmark
+        run: |
+          read -ra variants <<<"$VARIANTS"
+          "$BENCH/bench.sh" "$WORK/project" "$RESULTS/runs" "$ROUNDS" "${variants[@]}"
+      - name: Summarize
+        if: ${{ !cancelled() }}
+        run: |
+          {
+            cat "$RESULTS/environment.md"
+            echo "## Correctness (50 models)"
+            echo
+            cat "$RESULTS/correctness.md" 2>/dev/null || echo "Not run."
+            echo
+            echo "## Benchmark (${MODELS} models, medians of ${ROUNDS} rounds)"
+            echo
+            python3 "$BENCH/summarize.py" "$RESULTS/runs" 2>&1 || echo "No benchmark results."
+          } >"$RESULTS/summary.md"
+          cat "$RESULTS/summary.md" >>"$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: dbt-compare-${{ github.run_id }}
+          path: ${{ runner.temp }}/dbt-compare-results
+          retention-days: 90
diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md
@@ -0,0 +1,128 @@
+# SQLBuild vs dbt: dense project compile benchmark
+
+This harness generates one dense synthetic project in both SQLBuild and dbt form, then times the
+same edit loop in each tool. It compares SQLBuild with dbt v2 (the `dbt` package on PyPI), with
+dbt Core as a reference. Published results are on the
+[Benchmarks](https://sqlbuild.com/docs/benchmarks) page.
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `write_dbt_dense.py` | Writes `sqlbuild/`, `sqlbuild-norules/`, `dbt/`, `warehouse.duckdb` and `profiles/` for N models |
+| `edit_models.py` | Changes one constant in the SQL body of N evenly spaced models, in either project |
+| `bench.sh` | Times cold, warm, one-model edit and 100-model edit runs per variant, in alternating order |
+| `summarize.py` | Prints median wall time, CPU time and peak RSS per variant and scenario |
+| `correctness.sh` | Runs every tool on 50-model projects: clean, a missing `ref`, and a missing column |
+
+## The project
+
+`write_dbt_dense.py` imports the dense generator used by the compile-performance CI jobs
+(`scripts/cold_compile_performance/_helpers/dense_project.py`) and switches its adapter to DuckDB.
+Nothing else in the SQLBuild project changes. The dbt project is derived from it file by file, so
+model names, folders, SQL bodies and the DAG are the same.
+
+At 10,000 models the project has 2,400 sources, 500 seeds, 250 SQL functions, 120 macros,
+17,000 `not_null` checks and 10,000 SQL tests. Models are wide: 8 to 472 columns, about 31 on
+average.
+
+| SQLBuild | dbt |
+|---|---|
+| `MODEL (...)` header with typed columns | `_models.yml` per folder, `data_type`, `contract: {enforced: true}` |
+| `__ref`, `__source`, `__udf` | `ref`, `source`, `function` |
+| Python macros | Jinja macros producing the same SQL |
+| `audits [not_null]` | `data_tests: [not_null]` |
+| SQL tests with fixture CTEs | `unit_tests` with the same input and expected rows |
+| `[rules] select = ["SQBR", "XSQBR"]` (built-in Rules plus one custom Rule) | none |
+
+`sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules;
+column binding, type inference and contract checks still run.
+
+## Running it on GitHub Actions
+
+The `Benchmark dbt comparison` workflow (`.github/workflows/benchmark-dbt.yml`) runs only when
+started by hand. It installs the requested released versions from PyPI, runs `correctness.sh`, then
+`bench.sh`, and uploads the results directory with a `summary.md` (environment, correctness table,
+benchmark table). The same summary appears on the run page.
+
+```bash
+gh workflow run benchmark-dbt.yml -f sqlbuild_version=0.164.2 -f rounds=3
+```
+
+| Input | Default |
+|---|---|
+| `sqlbuild_version` | empty: the latest release |
+| `dbt_version` | `2.0.9` |
+| `dbt_core_version` | `1.12.6` |
+| `dbt_duckdb_version` | `1.11.0` |
+| `models` | `10000` |
+| `rounds` | `3` |
+| `variants` | all eight variants below |
+
+## Running it locally
+
+Use the released packages in fresh virtual environments. From the repository root:
+
+```bash
+uv venv -p 3.12 /path/to/venv-sqb && uv pip install -p /path/to/venv-sqb/bin/python sqlbuild pytest duckdb
+uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt
+uv venv -p 3.12 /path/to/venv-dbt && uv pip install -p /path/to/venv-dbt/bin/python dbt-core dbt-duckdb
+
+export SQB=/path/to/venv-sqb/bin/sqb DBT_V2=/path/to/venv-dbt2/bin/dbt \
+  DBT_CORE=/path/to/venv-dbt/bin/dbt PYTHON=/path/to/venv-sqb/bin/python
+
+"$PYTHON" benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k
+benchmarks/dbt-compare/correctness.sh /path/to/correctness
+benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3
+python3 benchmarks/dbt-compare/summarize.py /path/to/results
+```
+
+The generator imports the CI helpers from the checkout, so it needs a Python with `sqlbuild`,
+`duckdb` and `pytest` installed, and works best with a sqlbuild release close to the checkout.
+Nothing is built from source. Run on an otherwise idle machine: other load changes wall times far
+more than CPU times.
+
+## What each run does
+
+For every variant and round, `bench.sh` copies the pristine project into `work/VARIANT`, waits
+five seconds, then runs:
+
+1. **cold**: the command with no `target/` and no caches.
+2. **warm**: the same command again with nothing changed.
+3. **edit1**: one model's SQL body edited, then the command.
+4. **edit100**: 100 models' SQL bodies edited (1% of 10,000), then the command.
+
+Each edit changes one constant in a projection, so the query changes but its columns, types and
+references don't. Rounds alternate the variant order. `SCENARIOS` limits which of warm, edit1 and
+edit100 run.
+
+Commands, run from the project copy:
+
+| Variant | Command |
+|---|---|
+| `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` |
+| `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` |
+| `sqb_rules_full` | `sqb_rules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile that uses the per-model caches |
+| `v2_parse` / `v2_compile` | dbt v2 `parse` / `compile` with `--profiles-dir ../profiles --no-version-check` |
+| `v2_compile_strict` | dbt v2 `compile --static-analysis strict`, which checks columns like SQLBuild does |
+| `core_parse` / `core_compile` | the same `parse` / `compile` commands with dbt Core |
+
+Both tools use their defaults otherwise: SQLBuild's compile cache, reuse and delta compile, and
+dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets
+`threads: 4`.
+
+`/usr/bin/time -f '%e %U %S %M'` records wall time, user and system CPU, and the peak RSS of the
+largest process. `summarize.py` also reports what each tool said it did: SQLBuild prints whether it
+reused the previous compile or compiled only the changed files, and dbt Core logs when it falls back
+to a full parse. The 1-minute load average before and after each run is recorded too.
+
+## What isn't equivalent
+
+- SQLBuild compile binds every column, infers types and checks contracts. dbt v2 does this with
+  `--static-analysis strict`; its default compile doesn't report unknown columns. dbt Core compile
+  renders Jinja and resolves references without analysing the SQL.
+- SQLBuild compile also runs configurable Rules (`sqb_rules`). dbt has no built-in counterpart.
+- SQLBuild compile prepares its 10,000 SQL tests. dbt `compile` parses the unit tests but doesn't
+  render them; that happens in `dbt test` or `dbt build`.
+- dbt renders every model through Jinja; SQLBuild expands references and macros in its own compiler.
+  That cost is part of each tool and is included.
diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Dense-project compile benchmark: SQLBuild vs dbt v2 and dbt Core.
+#
+# Usage: bench.sh PROJECT_ROOT OUTDIR ROUNDS [VARIANT...]
+#
+# PROJECT_ROOT is a directory written by write_dbt_dense.py. Each round runs every variant once, in
+# alternating order (odd rounds forward, even rounds reversed). For each variant and round:
+#   1. copy the pristine project into PROJECT_ROOT/work/VARIANT (no target/, no caches)
+#   2. cold:    first compile or parse
+#   3. warm:    the same command again, nothing changed
+#   4. edit1:   edit one model's SQL body, run again
+#   5. edit100: edit 100 models' SQL bodies (1% of 10,000), run again
+#
+# Environment:
+#   SQB       sqb executable (default: sqb)
+#   DBT_CORE  dbt Core executable (default: dbt)
+#   DBT_V2    dbt v2 executable (default: dbt2)
+#   PYTHON    Python used for the edit helper (default: python3)
+#   SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three)
+#
+# Variants: sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse
+# core_compile. With no VARIANT arguments, all of them run.
+set -u
+ROOT=$(realpath "$1")
+OUT=$(realpath -m "$2")
+ROUNDS=$3
+shift 3
+HERE=$(cd "$(dirname "$0")" && pwd)
+WORK=$ROOT/work
+SQB=${SQB:-sqb}
+DBT_CORE=${DBT_CORE:-dbt}
+DBT_V2=${DBT_V2:-dbt2}
+PYTHON=${PYTHON:-python3}
+SCENARIOS=" ${SCENARIOS:-warm edit1 edit100} "
+MODELS=$(find "$ROOT/sqlbuild/models" -name '*.sql' | wc -l)
+export DBT_SEND_ANONYMOUS_USAGE_STATS=false
+DBTFLAGS="--profiles-dir $ROOT/profiles --no-version-check"
+declare -A SRC CMD
+SRC[sqb_rules]=sqlbuild
+CMD[sqb_rules]="$SQB --no-color compile"
+SRC[sqb_norules]=sqlbuild-norules
+CMD[sqb_norules]="$SQB --no-color compile"
+SRC[sqb_rules_full]=sqlbuild
+CMD[sqb_rules_full]="env SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1 $SQB --no-color compile"
+SRC[core_parse]=dbt
+CMD[core_parse]="$DBT_CORE parse $DBTFLAGS"
+SRC[core_compile]=dbt
+CMD[core_compile]="$DBT_CORE compile $DBTFLAGS"
+SRC[v2_parse]=dbt
+CMD[v2_parse]="$DBT_V2 parse $DBTFLAGS"
+SRC[v2_compile]=dbt
+CMD[v2_compile]="$DBT_V2 compile $DBTFLAGS"
+SRC[v2_compile_strict]=dbt
+CMD[v2_compile_strict]="$DBT_V2 compile $DBTFLAGS --static-analysis strict"
+
+VARIANTS=("$@")
+if [ ${#VARIANTS[@]} -eq 0 ]; then
+  VARIANTS=(sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict
+    core_parse core_compile)
+fi
+for v in "${VARIANTS[@]}"; do
+  if [ -z "${CMD[$v]+set}" ]; then
+    echo "unknown variant: $v" >&2
+    exit 2
+  fi
+done
+mkdir -p "$OUT" "$WORK"
+
+load1() { cut -d' ' -f1 /proc/loadavg; }
+
+measure() {
+  local v=$1 scenario=$2 round=$3 base
+  base="$OUT/$v.$scenario.r$round"
+  local before after
+  before=$(load1)
+  /usr/bin/time -f '%e %U %S %M' -o "$base.time" \
+    bash -c "cd '$WORK/$v' && exec ${CMD[$v]}" >"$base.out" 2>"$base.err"
+  echo $? >"$base.rc"
+  after=$(load1)
+  echo "$v $scenario r$round load $before $after" >>"$OUT/uptime.log"
+  echo "$v $scenario r$round rc=$(cat "$base.rc") time=$(tail -1 "$base.time") load=$before/$after" |
+    tee -a "$OUT/summary.log"
+}
+
+edit() {
+  "$PYTHON" "$HERE/edit_models.py" "$WORK/$1" --models "$MODELS" --count "$2" --value "$3"
+}
+
+for round in $(seq 1 "$ROUNDS"); do
+  n=${#VARIANTS[@]}
+  for i in $(seq 0 $((n - 1))); do
+    if [ $((round % 2)) -eq 0 ]; then v=${VARIANTS[$((n - 1 - i))]}; else v=${VARIANTS[$i]}; fi
+    rm -rf "${WORK:?}/$v"
+    cp -a "$ROOT/${SRC[$v]}" "$WORK/$v"
+    sync
+    # Let the copy age past file-change heuristics that treat just-written files specially.
+    sleep 5
+    measure "$v" cold "$round"
+    [[ $SCENARIOS == *" warm "* ]] && measure "$v" warm "$round"
+    if [[ $SCENARIOS == *" edit1 "* ]]; then
+      edit "$v" 1 $((round * 10 + 1))
+      measure "$v" edit1 "$round"
+    fi
+    if [[ $SCENARIOS == *" edit100 "* ]]; then
+      edit "$v" 100 $((round * 10 + 2))
+      measure "$v" edit100 "$round"
+    fi
+  done
+done
+rm -rf "${WORK:?}"