From abd8df40fa3dbcf6a0cc969674ffb293dbf883b8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:20:11 +0100 Subject: [PATCH 1/4] chore(bench): add dbt comparison harness for the dense project --- benchmarks/dbt-compare/README.md | 111 +++++++ benchmarks/dbt-compare/bench.sh | 116 ++++++++ benchmarks/dbt-compare/correctness.sh | 54 ++++ benchmarks/dbt-compare/edit_models.py | 50 ++++ benchmarks/dbt-compare/summarize.py | 82 ++++++ benchmarks/dbt-compare/write_dbt_dense.py | 342 ++++++++++++++++++++++ 6 files changed, 755 insertions(+) create mode 100644 benchmarks/dbt-compare/README.md create mode 100755 benchmarks/dbt-compare/bench.sh create mode 100755 benchmarks/dbt-compare/correctness.sh create mode 100644 benchmarks/dbt-compare/edit_models.py create mode 100644 benchmarks/dbt-compare/summarize.py create mode 100644 benchmarks/dbt-compare/write_dbt_dense.py diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md new file mode 100644 index 0000000000..0a0e06f8e3 --- /dev/null +++ b/benchmarks/dbt-compare/README.md @@ -0,0 +1,111 @@ +# SQLBuild vs dbt: dense project compile benchmark + +This harness generates one dense synthetic project in both SQLBuild and dbt form, then times the +same edit loop in each tool. The published results are on the +[Benchmarks](https://sqlbuild.com/docs/benchmarks) page. + +## Files + +| File | Purpose | +|---|---| +| `write_dbt_dense.py` | Writes `sqlbuild/`, `sqlbuild-norules/`, `dbt/`, `warehouse.duckdb` and `profiles/` for N models | +| `edit_models.py` | Changes one constant in the SQL body of N evenly spaced models, in either project | +| `bench.sh` | Times cold, warm, one-model edit and 100-model edit runs per variant, in alternating order | +| `summarize.py` | Prints median wall time, CPU time and peak RSS per variant and scenario | +| `correctness.sh` | Runs every tool on 50-model projects: clean, a missing `ref`, and a missing column | + +## The project + +`write_dbt_dense.py` imports the dense generator used by the compile-performance CI jobs +(`scripts/cold_compile_performance/_helpers/dense_project.py`) and switches its adapter to DuckDB. +Nothing else in the SQLBuild project changes. The dbt project is derived from it file by file, so +model names, folders, SQL bodies and the DAG are the same. + +At 10,000 models the project has 2,400 sources, 500 seeds, 250 SQL functions, 120 macros, +17,000 `not_null` checks and 10,000 SQL tests. Models are wide: 8 to 472 columns, about 31 on +average. + +| SQLBuild | dbt | +|---|---| +| `MODEL (...)` header with typed columns | `_models.yml` per folder, `data_type`, `contract: {enforced: true}` | +| `__ref`, `__source`, `__udf` | `ref`, `source`, `function` | +| Python macros | Jinja macros producing the same SQL | +| `audits [not_null]` | `data_tests: [not_null]` | +| SQL tests with fixture CTEs | `unit_tests` with the same input and expected rows | +| `[rules] select = ["SQBR", "XSQBR"]` (built-in Rules plus one custom Rule) | none | + +`sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules; +column binding, type inference and contract checks still run. + +## Running it + +Use the released packages in fresh virtual environments. From the repository root: + +```bash +uv venv -p 3.12 /path/to/venv-sqb && uv pip install -p /path/to/venv-sqb/bin/python sqlbuild pytest duckdb +uv venv -p 3.12 /path/to/venv-dbt && uv pip install -p /path/to/venv-dbt/bin/python dbt-core dbt-duckdb +uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt # optional, dbt v2 + +/path/to/venv-sqb/bin/python benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k + +SQB=/path/to/venv-sqb/bin/sqb \ +DBT_CORE=/path/to/venv-dbt/bin/dbt \ +DBT_V2=/path/to/venv-dbt2/bin/dbt \ +PYTHON=/path/to/venv-sqb/bin/python \ + benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3 \ + sqb_rules sqb_norules core_parse core_compile v2_parse v2_compile v2_compile_strict + +python3 benchmarks/dbt-compare/summarize.py /path/to/results + +SQB=... DBT_CORE=... DBT_V2=... PYTHON=... benchmarks/dbt-compare/correctness.sh /path/to/correctness +``` + +The generator needs the repository checkout, because it imports the CI helpers, and a Python with +`sqlbuild`, `duckdb` and `pytest` installed. Nothing is built from source. + +Point `HOME` and `XDG_*` at a scratch directory if you don't want either tool to write user-level +caches. + +## What each run does + +For every variant and round, `bench.sh` copies the pristine project into `work/VARIANT`, waits +five seconds, then runs: + +1. **cold**: the command with no `target/` and no caches. +2. **warm**: the same command again with nothing changed. +3. **edit1**: one model's SQL body edited, then the command. +4. **edit100**: 100 models' SQL bodies edited (1% of 10,000), then the command. + +Each edit changes one constant in a projection, so the query changes but its columns, types and +references don't. Rounds alternate the variant order. Before each timed run the script waits until +the 1-minute load average is below `LOAD_MAX` (default 2), for at most `LOAD_WAIT_MAX` seconds. + +Commands, run from the project copy: + +| Variant | Command | +|---|---| +| `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` | +| `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` | +| `core_parse` / `core_compile` | `dbt parse` / `dbt compile` with `--profiles-dir ../profiles --no-version-check` | +| `v2_parse` / `v2_compile` | the same commands with dbt v2 | +| `v2_compile_strict` | dbt v2 `compile --static-analysis strict` | + +Both tools use their defaults otherwise: SQLBuild's compile cache, reuse and delta compile, and +dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets +`threads: 4`. + +`/usr/bin/time -f '%e %U %S %M'` records wall time, user and system CPU, and the peak RSS of the +largest process. `summarize.py` also reports what each tool said it did: SQLBuild prints whether it +reused the previous compile or compiled only the changed files, and dbt Core logs when it falls back +to a full parse. + +## What isn't equivalent + +- SQLBuild compile binds every column, infers types and checks contracts. dbt Core compile renders + Jinja and resolves references; it doesn't analyse the SQL. dbt v2 does with + `--static-analysis strict`. +- SQLBuild compile also runs configurable Rules (`sqb_rules`). dbt has no built-in counterpart. +- SQLBuild compile prepares its 10,000 SQL tests. dbt `compile` parses the unit tests but doesn't + render them; that happens in `dbt test` or `dbt build`. +- dbt renders every model through Jinja; SQLBuild expands references and macros in its own compiler. + That cost is part of each tool and is included. diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh new file mode 100755 index 0000000000..bf822bb451 --- /dev/null +++ b/benchmarks/dbt-compare/bench.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Dense-project compile benchmark: SQLBuild vs dbt Core (and optionally dbt v2). +# +# Usage: bench.sh PROJECT_ROOT OUTDIR ROUNDS [VARIANT...] +# +# PROJECT_ROOT is a directory written by write_dbt_dense.py. Each round runs every variant once, in +# alternating order (odd rounds forward, even rounds reversed). For each variant and round: +# 1. copy the pristine project into PROJECT_ROOT/work/VARIANT (no target/, no caches) +# 2. cold: first compile or parse +# 3. warm: the same command again, nothing changed +# 4. edit1: edit one model's SQL body, run again +# 5. edit100: edit 100 models' SQL bodies (1% of 10,000), run again +# +# Environment: +# SQB sqb executable (default: sqb) +# DBT_CORE dbt Core executable (default: dbt) +# DBT_V2 dbt v2 executable; the v2_* variants are skipped when unset +# PYTHON Python used for the edit helper (default: python3) +# LOAD_MAX wait until the 1-minute load average is below this before each timed run (default 2) +# LOAD_WAIT_MAX give up waiting after this many seconds and run anyway (default 1800) +# SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three) +set -u +ROOT=$(realpath "$1") +OUT=$(realpath -m "$2") +ROUNDS=$3 +shift 3 +HERE=$(cd "$(dirname "$0")" && pwd) +WORK=$ROOT/work +SQB=${SQB:-sqb} +DBT_CORE=${DBT_CORE:-dbt} +DBT_V2=${DBT_V2:-} +PYTHON=${PYTHON:-python3} +LOAD_MAX=${LOAD_MAX:-2} +LOAD_WAIT_MAX=${LOAD_WAIT_MAX:-1800} +SCENARIOS=" ${SCENARIOS:-warm edit1 edit100} " +MODELS=$(find "$ROOT/sqlbuild/models" -name '*.sql' | wc -l) +export DBT_SEND_ANONYMOUS_USAGE_STATS=false +DBTFLAGS="--profiles-dir $ROOT/profiles --no-version-check" +mkdir -p "$OUT" "$WORK" + +declare -A SRC CMD +SRC[sqb_rules]=sqlbuild +CMD[sqb_rules]="$SQB --no-color compile" +SRC[sqb_norules]=sqlbuild-norules +CMD[sqb_norules]="$SQB --no-color compile" +SRC[core_parse]=dbt +CMD[core_parse]="$DBT_CORE parse $DBTFLAGS" +SRC[core_compile]=dbt +CMD[core_compile]="$DBT_CORE compile $DBTFLAGS" +SRC[v2_parse]=dbt +CMD[v2_parse]="$DBT_V2 parse $DBTFLAGS" +SRC[v2_compile]=dbt +CMD[v2_compile]="$DBT_V2 compile $DBTFLAGS" +SRC[v2_compile_strict]=dbt +CMD[v2_compile_strict]="$DBT_V2 compile $DBTFLAGS --static-analysis strict" + +VARIANTS=("$@") +if [ ${#VARIANTS[@]} -eq 0 ]; then + VARIANTS=(sqb_rules sqb_norules core_parse core_compile) + [ -n "$DBT_V2" ] && VARIANTS+=(v2_parse v2_compile) +fi + +load1() { cut -d' ' -f1 /proc/loadavg; } + +wait_quiet() { + local waited=0 + while awk -v l="$(load1)" -v m="$LOAD_MAX" 'BEGIN { exit !(l >= m) }'; do + if [ "$waited" -ge "$LOAD_WAIT_MAX" ]; then + echo "load gate timed out at $(load1) after ${waited}s" >>"$OUT/uptime.log" + return + fi + sleep 15 + waited=$((waited + 15)) + done +} + +measure() { + local v=$1 scenario=$2 round=$3 base + base="$OUT/$v.$scenario.r$round" + wait_quiet + local before after + before=$(load1) + /usr/bin/time -f '%e %U %S %M' -o "$base.time" \ + bash -c "cd '$WORK/$v' && exec ${CMD[$v]}" >"$base.out" 2>"$base.err" + echo $? >"$base.rc" + after=$(load1) + echo "$v $scenario r$round load $before $after" >>"$OUT/uptime.log" + echo "$v $scenario r$round rc=$(cat "$base.rc") time=$(tail -1 "$base.time") load=$before/$after" | + tee -a "$OUT/summary.log" +} + +edit() { + "$PYTHON" "$HERE/edit_models.py" "$WORK/$1" --models "$MODELS" --count "$2" --value "$3" +} + +for round in $(seq 1 "$ROUNDS"); do + n=${#VARIANTS[@]} + for i in $(seq 0 $((n - 1))); do + if [ $((round % 2)) -eq 0 ]; then v=${VARIANTS[$((n - 1 - i))]}; else v=${VARIANTS[$i]}; fi + rm -rf "${WORK:?}/$v" + cp -a "$ROOT/${SRC[$v]}" "$WORK/$v" + sync + # Let the copy age past file-change heuristics that treat just-written files specially. + sleep 5 + measure "$v" cold "$round" + [[ $SCENARIOS == *" warm "* ]] && measure "$v" warm "$round" + if [[ $SCENARIOS == *" edit1 "* ]]; then + edit "$v" 1 $((round * 10 + 1)) + measure "$v" edit1 "$round" + fi + if [[ $SCENARIOS == *" edit100 "* ]]; then + edit "$v" 100 $((round * 10 + 2)) + measure "$v" edit100 "$round" + fi + done +done diff --git a/benchmarks/dbt-compare/correctness.sh b/benchmarks/dbt-compare/correctness.sh new file mode 100755 index 0000000000..d944dfde8a --- /dev/null +++ b/benchmarks/dbt-compare/correctness.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Correctness matrix on 50-model projects: clean, a reference to a missing model, and a column the +# upstream model does not have. Run from the repository root. +# +# Usage: correctness.sh OUTROOT +# Environment: SQB, DBT_CORE, DBT_V2 (optional) and PYTHON, as for bench.sh. PYTHON must be able to +# import duckdb, pytest and the installed sqlbuild package (the generator uses the CI helpers). +set -u +OUT=$(realpath -m "$1") +HERE=$(cd "$(dirname "$0")" && pwd) +SQB=${SQB:-sqb} +DBT_CORE=${DBT_CORE:-dbt} +DBT_V2=${DBT_V2:-} +PYTHON=${PYTHON:-python3} +export DBT_SEND_ANONYMOUS_USAGE_STATS=false +mkdir -p "$OUT/logs" + +for kind in clean ref column; do + args=() + [ "$kind" != clean ] && args=(--seed-error "$kind") + "$PYTHON" "$HERE/write_dbt_dense.py" --models 50 --out "$OUT/p50-$kind" "${args[@]}" || exit 1 +done + +first_problem() { + sed 's/\x1b\[[0-9;]*m//g' "$1" | grep -m1 -E 'error\[|\[error\]|depends on a node' | + sed -E 's/^ +//' | cut -c1-140 +} + +run() { + local kind=$1 label=$2 dir=$3 + shift 3 + local log="$OUT/logs/$kind.$label.log" + rm -rf "$dir/target" "$dir/logs" + (cd "$dir" && "$@" >"$log" 2>&1) + local rc=$? + printf '| %s | %s | %s | %s |\n' "$kind" "$label" "$rc" "$(first_problem "$log")" +} + +echo "| project | tool and command | exit code | first problem reported |" +echo "|---|---|---|---|" +for kind in clean ref column; do + P=$OUT/p50-$kind + DBTFLAGS=(--profiles-dir "$P/profiles" --no-version-check) + run "$kind" "sqb compile (rules on)" "$P/sqlbuild" "$SQB" --no-color compile + run "$kind" "sqb compile (rules off)" "$P/sqlbuild-norules" "$SQB" --no-color compile + run "$kind" "dbt Core parse" "$P/dbt" "$DBT_CORE" parse "${DBTFLAGS[@]}" + run "$kind" "dbt Core compile" "$P/dbt" "$DBT_CORE" compile "${DBTFLAGS[@]}" + if [ -n "$DBT_V2" ]; then + run "$kind" "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ + "${DBTFLAGS[@]}" --static-analysis strict + fi +done diff --git a/benchmarks/dbt-compare/edit_models.py b/benchmarks/dbt-compare/edit_models.py new file mode 100644 index 0000000000..d77b50a0de --- /dev/null +++ b/benchmarks/dbt-compare/edit_models.py @@ -0,0 +1,50 @@ +"""Edit the SQL body of evenly spaced models in a generated dense project. + +Usage: python edit_models.py PROJECT_DIR --models TOTAL --count N --value V + +Works on both the SQLBuild and the dbt project. Each selected model's last plain +`COALESCE(b.amount, )` projection gets the constant V, so the edit changes the query but not the +model's columns, types, references or header. +""" + +import argparse +import re +from pathlib import Path + +PATTERN = re.compile(r"CAST\(COALESCE\(b\.amount, \d+\)") + + +def model_name(index: int) -> str: + return f"sales{index // 1000:03d}__int_clean__orders_{index:05d}" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("project", type=Path) + parser.add_argument("--models", type=int, required=True) + parser.add_argument("--count", type=int, required=True) + parser.add_argument("--value", type=int, required=True) + args = parser.parse_args() + step = args.models // args.count + for k in range(args.count): + index = k * step + step // 2 + path = ( + args.project + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{model_name(index)}.sql" + ) + text = path.read_text() + matches = list(PATTERN.finditer(text)) + assert matches, path + last = matches[-1] + replacement = f"CAST(COALESCE(b.amount, {args.value})" + assert last.group(0) != replacement, path + path.write_text(text[: last.start()] + replacement + text[last.end() :]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/dbt-compare/summarize.py b/benchmarks/dbt-compare/summarize.py new file mode 100644 index 0000000000..2ce83c8013 --- /dev/null +++ b/benchmarks/dbt-compare/summarize.py @@ -0,0 +1,82 @@ +"""Summarize bench.sh output: median wall, user+sys CPU and peak RSS per variant and scenario. + +Usage: python summarize.py OUTDIR +""" + +import re +import statistics +import sys +from collections import defaultdict +from pathlib import Path + +SCENARIOS = ("cold", "warm", "edit1", "edit100") +TIME_RE = re.compile(r"^(?P\w+)\.(?P\w+)\.r(?P\d+)\.time$") +LOAD_RE = re.compile(r"^(\w+) (\w+) r\d+ load ([\d.]+) ([\d.]+)$") + + +def path_taken(variant: str, out: str, err: str) -> str: + """What each tool reported doing: full work, a partial parse, a replay or a delta compile.""" + if variant.startswith("sqb"): + if "Inputs unchanged; reused the previous compile" in err: + return "reused" + if "against the previous compile" in err: + return "delta" + return "full" + if variant.startswith("core"): + text = out + err + if "Unable to do partial parsing" in text or "Starting full parse" in text: + return "full parse" + return "partial parse" + return "-" + + +def main() -> None: + out = Path(sys.argv[1]) + runs: dict[tuple[str, str], list[tuple[float, float, int, int, str]]] = defaultdict(list) + for path in sorted(out.glob("*.time")): + m = TIME_RE.match(path.name) + if not m: + continue + stem = str(path)[: -len(".time")] + wall, user, sys_, rss = path.read_text().strip().splitlines()[-1].split() + rc = int(Path(stem + ".rc").read_text()) + taken = path_taken( + m["variant"], + Path(stem + ".out").read_text(errors="replace"), + Path(stem + ".err").read_text(errors="replace"), + ) + runs[(m["variant"], m["scenario"])].append( + (float(wall), float(user) + float(sys_), int(rss), rc, taken) + ) + loads: dict[tuple[str, str], list[float]] = defaultdict(list) + for line in (out / "uptime.log").read_text().splitlines(): + m = LOAD_RE.match(line) + if m: + loads[(m[1], m[2])] += [float(m[3]), float(m[4])] + + print( + "| variant | scenario | runs | wall s (median) | wall s (all) | user+sys s (median) " + "| peak RSS MiB (median) | exit codes | path | 1-min load before/after |" + ) + print("|---|---|---|---|---|---|---|---|---|---|") + variants = sorted({variant for variant, _ in runs}) + for variant in variants: + for scenario in SCENARIOS: + cell = runs.get((variant, scenario)) + if not cell: + continue + walls = [r[0] for r in cell] + load = loads.get((variant, scenario), [0.0]) + print( + f"| {variant} | {scenario} | {len(cell)} | {statistics.median(walls):.2f} " + f"| {', '.join(f'{w:.1f}' for w in walls)} " + f"| {statistics.median(r[1] for r in cell):.1f} " + f"| {statistics.median(r[2] for r in cell) / 1024:.0f} " + f"| {','.join(sorted({str(r[3]) for r in cell}))} " + f"| {','.join(sorted({r[4] for r in cell}))} " + f"| {min(load):.2f}-{max(load):.2f} |" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py new file mode 100644 index 0000000000..aa0801bd76 --- /dev/null +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -0,0 +1,342 @@ +# ruff: noqa: E501, PLR0915 +"""Write the same dense project for SQLBuild (DuckDB) and dbt (DuckDB). + +Usage, from the repository root: + + python benchmarks/dbt-compare/write_dbt_dense.py --models N --out DIR [--seed-error ref|column] + +Creates under DIR: + sqlbuild/ SQLBuild dense project (the CI dense generator, adapter switched to duckdb) + sqlbuild-norules/ the same project with `[rules] select = []` + dbt/ dbt project derived file by file from `sqlbuild/` + warehouse.duckdb DuckDB file holding every source table (empty) + profiles/profiles.yml + +The SQLBuild project comes unchanged from the generator the compile-performance CI jobs use. The +dbt project is derived from the generated SQLBuild files, so SQL, DAG, names and folders match. +""" + +import argparse +import re +import shutil +import sys +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPOSITORY_ROOT)) + +from scripts.cold_compile_performance._helpers.dense_project import ( # noqa: E402 + dense_column_count, + dense_model_name, + write_dense_compile_project, +) + +SOURCE_NAME = "raw" +SOURCE_SCHEMA = "raw" +DATABASE = "warehouse" +ERROR_MODEL_INDEX = 2 +ERROR_COLUMN = "missing_col" +ERROR_REF = "sales000__int_clean__orders_missing" + +COLUMN_RE = re.compile( + r'^\s+(\w+) \(type (\w+), nullable (true|false), description "([^"]*)"(, audits \[not_null\])?\)' +) +MACRO_CALL_RE = re.compile(r'@(macro_\d{3}_\d{5})\("([^"]*)"\)') +CAST_RE = re.compile(r"CAST\(([^ ]+) AS (\w+)\) AS (\w+)") + + +def q(text: str) -> str: + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def write_sqlbuild(out: Path, n: int) -> Path: + project = out / "sqlbuild" + write_dense_compile_project(project_dir=project, model_count=n) + toml = project / "sqlbuild_project.toml" + text = toml.read_text() + assert 'adapter = "snowflake"' in text + toml.write_text(text.replace('adapter = "snowflake"', 'adapter = "duckdb"')) + return project + + +def write_norules(project: Path) -> Path: + target = project.parent / "sqlbuild-norules" + shutil.copytree(project, target) + toml = target / "sqlbuild_project.toml" + text = toml.read_text() + new = text.replace('select = ["SQBR", "XSQBR"]', "select = []") + assert new != text + toml.write_text(new) + return target + + +def model_header_and_body(text: str) -> tuple[list[tuple[str, str, bool, str, bool]], str]: + header, body = text.split("\n));\n", 1) + columns = [] + for line in header.splitlines()[1:]: + m = COLUMN_RE.match(line) + assert m, line + columns.append((m[1], m[2], m[3] == "false", m[4], bool(m[5]))) + return columns, body + + +def to_jinja_sql(body: str) -> str: + body = re.sub(r'__ref\("([^"]+)"\)', r"{{ ref('\1') }}", body) + body = re.sub(r'__source\("([^"]+)"\)', rf"{{{{ source('{SOURCE_NAME}', '\1') }}}}", body) + body = re.sub(r'__udf\("([^"]+)"\)\(', r"{{ function('\1') }}(", body) + body = MACRO_CALL_RE.sub(lambda m: "{{ " + m[1] + "(" + q(m[2]) + ") }}", body) + assert not re.search(r"__(ref|source|udf)\(", body), body[:400] + assert "@" not in body + return body.lstrip("\n") + + +def parse_cast_row(select_list: str) -> dict[str, str]: + return {col: value for value, _type, col in CAST_RE.findall(select_list)} + + +def parse_test(text: str) -> tuple[dict[str, dict[str, str]], dict[str, str]]: + inputs: dict[str, dict[str, str]] = {} + expected: dict[str, str] = {} + for line in text.splitlines(): + line = line.strip().removeprefix("WITH ").rstrip(",") + m = re.match(r"(__(ref|source|expected)__(\w+)) AS \(SELECT (.*)\)$", line) + if not m: + continue + row = parse_cast_row(m[4]) + if m[2] == "expected": + expected = row + elif m[2] == "ref": + inputs[f"ref('{m[3]}')"] = row + else: + inputs[f"source('{SOURCE_NAME}', '{m[3]}')"] = row + assert expected and inputs, text[:300] + return inputs, expected + + +def flow_row(row: dict[str, str]) -> str: + return "{" + ", ".join(f"{k}: {v}" for k, v in row.items()) + "}" + + +def write_dbt(sqb: Path, out: Path, n: int) -> Path: + dbt = out / "dbt" + dbt.mkdir() + (dbt / "dbt_project.yml").write_text( + "name: dense_orders\n" + "version: '1.0.0'\n" + "profile: dense_orders\n" + "model-paths: [models]\n" + "macro-paths: [macros]\n" + "seed-paths: [seeds]\n" + "function-paths: [functions]\n" + "test-paths: [tests]\n" + "models:\n dense_orders:\n +materialized: table\n" + ) + # Sources + source_count = max(1, n * 24 // 100) + lines = [ + "sources:", + f" - name: {SOURCE_NAME}", + f" database: {DATABASE}", + f" schema: {SOURCE_SCHEMA}", + " tables:", + ] + for index in range(source_count): + width = dense_column_count(index % 240) + lines += [ + f" - name: orders_{index:05d}", + f" description: Generated source orders_{index:05d}.", + " columns:", + " - name: id", + " data_type: integer", + " constraints: [{type: not_null}]", + " - name: amount", + " data_type: double", + ] + for column in range(width - 2): + lines += [f" - name: quantity_{column:03d}", " data_type: double"] + (dbt / "models").mkdir() + (dbt / "models" / "_sources.yml").write_text("\n".join(lines) + "\n") + # Seeds + (dbt / "seeds").mkdir() + seed_yml = ["seeds:"] + for csv in sorted((sqb / "seeds").glob("*.csv")): + shutil.copy(csv, dbt / "seeds" / csv.name) + seed_yml += [ + f" - name: {csv.stem}", + f" description: Test seed {csv.stem}.", + " config:", + " column_types: {id: integer, label: varchar}", + " columns:", + " - name: id", + " - name: label", + ] + (dbt / "seeds" / "_seeds.yml").write_text("\n".join(seed_yml) + "\n") + # Functions (dbt UDF resources) + (dbt / "functions").mkdir() + fn_yml = ["functions:"] + for fn in sorted((sqb / "functions" / "sql").glob("*.sql")): + body = fn.read_text().split(");\n", 1)[1].strip() + (dbt / "functions" / fn.name).write_text(body + "\n") + fn_yml += [ + f" - name: {fn.stem}", + " description: Test function.", + " arguments:", + " - name: input_value", + " data_type: double", + " returns:", + " data_type: double", + ] + (dbt / "functions" / "_functions.yml").write_text("\n".join(fn_yml) + "\n") + # Macros: SQLBuild Python macros -> Jinja macros, same per-cohort bucket layout + macro_def_re = re.compile( + r"def (macro_\d{3}_\d{5})\(expression: str\) -> str:\n return (.*)\n" + ) + offset_re = re.compile( + r'def _offset\(expression: str\) -> str:\n return f"\(\{expression\} \+ (\d+)\)"' + ) + base_re = re.compile( + r'def (base_offset_\d{3})\(expression: str\) -> str:\n return f"\(\{expression\} \+ 1\)"' + ) + for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/*/*.py")): + text = macro.read_text() + name, ret = macro_def_re.search(text).groups() + offset = offset_re.search(text)[1] + parts = [] + base = base_re.search(text) + if base: + parts.append( + f"{{% macro {base[1]}(expression) -%}}({{{{ expression }}}} + 1){{%- endmacro %}}" + ) + assert ret == f"{base[1]}(_offset(expression))" + parts.append( + f"{{% macro {name}(expression) -%}}" + f"{{{{ {base[1]}('(' ~ expression ~ ' + {offset})') }}}}{{%- endmacro %}}" + ) + else: + assert ret == "_offset(expression)" + parts.append( + f"{{% macro {name}(expression) -%}}({{{{ expression }}}} + {offset}){{%- endmacro %}}" + ) + domain = macro.relative_to(sqb / "models").parts[0] + target = dbt / "macros" / domain / macro.parent.name / (macro.stem + ".sql") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("\n".join(parts) + "\n") + # Models + properties + unit tests, one YAML per batch folder + for index in range(n): + name = dense_model_name(index) + domain = f"sales{index // 1000:03d}" + batch = f"batch{index // 5:05d}" + rel = Path(domain) / "intermediate" / "clean" / batch + src = sqb / "models" / rel / f"{name}.sql" + columns, body = model_header_and_body(src.read_text()) + out_dir = dbt / "models" / rel + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / f"{name}.sql").write_text(to_jinja_sql(body)) + yml_path = out_dir / "_models.yml" + if not yml_path.exists(): + yml_path.write_text("models:\n") + entry = [ + f" - name: {name}", + " description: Generated dense model.", + " config:", + " materialized: table", + " contract: {enforced: true}", + " columns:", + ] + for col, typ, not_null, desc, audit in columns: + entry += [ + f" - name: {col}", + f" data_type: {typ.lower()}", + f" description: {desc}", + ] + if not_null: + entry.append(" constraints: [{type: not_null}]") + if audit: + entry.append(" data_tests: [not_null]") + with yml_path.open("a") as fh: + fh.write("\n".join(entry) + "\n") + test_src = ( + sqb / "tests" / "unit" / rel / f"test_{name.replace('__', '_')}__preserves_order_id.sql" + ) + inputs, expected = parse_test(test_src.read_text()) + ut_path = out_dir / "_unit_tests.yml" + if not ut_path.exists(): + ut_path.write_text("unit_tests:\n") + ut = [f" - name: {name}__preserves_order_id", f" model: {name}", " given:"] + for inp, row in inputs.items(): + ut += [f" - input: {inp}", " rows:", f" - {flow_row(row)}"] + ut += [" expect:", " rows:", f" - {flow_row(expected)}"] + with ut_path.open("a") as fh: + fh.write("\n".join(ut) + "\n") + (dbt / "tests").mkdir() + return dbt + + +def write_warehouse(out: Path, n: int) -> None: + import duckdb + + path = (out / f"{DATABASE}.duckdb").resolve() + if path.exists(): + path.unlink() + con = duckdb.connect(str(path)) + con.execute(f"create schema {SOURCE_SCHEMA}") + con.execute("create schema analytics") + for index in range(max(1, n * 24 // 100)): + width = dense_column_count(index % 240) + cols = ["id integer not null", "amount double"] + [ + f"quantity_{c:03d} double" for c in range(width - 2) + ] + con.execute(f"create table {SOURCE_SCHEMA}.orders_{index:05d} ({', '.join(cols)})") + con.close() + profiles = out / "profiles" + profiles.mkdir(exist_ok=True) + (profiles / "profiles.yml").write_text( + "dense_orders:\n target: dev\n outputs:\n dev:\n type: duckdb\n" + f" path: {path}\n schema: analytics\n threads: 4\n" + ) + + +def seed_error(out: Path, kind: str) -> None: + """Break model ERROR_MODEL_INDEX the same way in every project. + + `column` selects a column the upstream model does not have; `ref` references a model that does + not exist. + """ + name = dense_model_name(ERROR_MODEL_INDEX) + upstream = dense_model_name(ERROR_MODEL_INDEX - 1) + for tool in ("sqlbuild", "sqlbuild-norules", "dbt"): + for path in (out / tool / "models").rglob(f"{name}.sql"): + text = path.read_text() + if kind == "column": + new = text.replace( + "COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + f"COALESCE(b.{ERROR_COLUMN}, 0) + CAST(b.id AS DOUBLE)", + 1, + ) + else: + new = text.replace(f'__ref("{upstream}")', f'__ref("{ERROR_REF}")', 1).replace( + f"{{{{ ref('{upstream}') }}}}", f"{{{{ ref('{ERROR_REF}') }}}}", 1 + ) + assert new != text + path.write_text(new) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--models", type=int, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--seed-error", choices=("ref", "column")) + args = parser.parse_args() + if args.out.exists(): + shutil.rmtree(args.out) + args.out.mkdir(parents=True) + sqb = write_sqlbuild(args.out, args.models) + write_dbt(sqb, args.out, args.models) + write_norules(sqb) + write_warehouse(args.out, args.models) + if args.seed_error: + seed_error(args.out, args.seed_error) + + +if __name__ == "__main__": + main() From 28da7f9cd59115280e4e3ce420ed3dddac268b0f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:21:06 +0100 Subject: [PATCH 2/4] chore(bench): add a full-compile SQLBuild variant to the dbt comparison --- benchmarks/dbt-compare/README.md | 1 + benchmarks/dbt-compare/bench.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md index 0a0e06f8e3..913cae2a9d 100644 --- a/benchmarks/dbt-compare/README.md +++ b/benchmarks/dbt-compare/README.md @@ -86,6 +86,7 @@ Commands, run from the project copy: |---|---| | `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` | | `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` | +| `sqb_rules_full` | `sqb_rules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile that uses the per-model caches | | `core_parse` / `core_compile` | `dbt parse` / `dbt compile` with `--profiles-dir ../profiles --no-version-check` | | `v2_parse` / `v2_compile` | the same commands with dbt v2 | | `v2_compile_strict` | dbt v2 `compile --static-analysis strict` | diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh index bf822bb451..7457e999dc 100755 --- a/benchmarks/dbt-compare/bench.sh +++ b/benchmarks/dbt-compare/bench.sh @@ -43,6 +43,8 @@ SRC[sqb_rules]=sqlbuild CMD[sqb_rules]="$SQB --no-color compile" SRC[sqb_norules]=sqlbuild-norules CMD[sqb_norules]="$SQB --no-color compile" +SRC[sqb_rules_full]=sqlbuild +CMD[sqb_rules_full]="env SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1 $SQB --no-color compile" SRC[core_parse]=dbt CMD[core_parse]="$DBT_CORE parse $DBTFLAGS" SRC[core_compile]=dbt From b203fb78922518ff44f114708baa2c20b7c8977d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:40:18 +0100 Subject: [PATCH 3/4] fix(bench): compare load averages in bash and log load-gate waits --- benchmarks/dbt-compare/bench.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh index 7457e999dc..9997bdf857 100755 --- a/benchmarks/dbt-compare/bench.sh +++ b/benchmarks/dbt-compare/bench.sh @@ -16,7 +16,8 @@ # DBT_CORE dbt Core executable (default: dbt) # DBT_V2 dbt v2 executable; the v2_* variants are skipped when unset # PYTHON Python used for the edit helper (default: python3) -# LOAD_MAX wait until the 1-minute load average is below this before each timed run (default 2) +# LOAD_MAX wait until the 1-minute load average is below this integer before each timed run +# (default 2) # LOAD_WAIT_MAX give up waiting after this many seconds and run anyway (default 1800) # SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three) set -u @@ -64,9 +65,15 @@ fi load1() { cut -d' ' -f1 /proc/loadavg; } +above_limit() { + local load + read -r load _ = m) }'; do + while above_limit; do if [ "$waited" -ge "$LOAD_WAIT_MAX" ]; then echo "load gate timed out at $(load1) after ${waited}s" >>"$OUT/uptime.log" return @@ -74,6 +81,9 @@ wait_quiet() { sleep 15 waited=$((waited + 15)) done + if [ "$waited" -gt 0 ]; then + echo "waited ${waited}s for load below $LOAD_MAX" >>"$OUT/uptime.log" + fi } measure() { From cb9bd82ff1aea8220fed49cbe2731e8c9a3be30a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:54:15 +0100 Subject: [PATCH 4/4] ci(bench): run the dbt comparison on a manual GitHub Actions workflow --- .github/workflows/benchmark-dbt.yml | 152 ++++++++++++++++++++++++++ benchmarks/dbt-compare/README.md | 70 +++++++----- benchmarks/dbt-compare/bench.sh | 50 +++------ benchmarks/dbt-compare/correctness.sh | 14 +-- benchmarks/dbt-compare/summarize.py | 16 ++- 5 files changed, 231 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/benchmark-dbt.yml diff --git a/.github/workflows/benchmark-dbt.yml b/.github/workflows/benchmark-dbt.yml new file mode 100644 index 0000000000..01203854ee --- /dev/null +++ b/.github/workflows/benchmark-dbt.yml @@ -0,0 +1,152 @@ +name: Benchmark dbt comparison + +# Manual only. Times SQLBuild, dbt v2 and dbt Core on the same generated dense DuckDB project with +# released packages from PyPI, and publishes the tables as an artifact and in the job summary. +# See benchmarks/dbt-compare/README.md. +on: + workflow_dispatch: + inputs: + sqlbuild_version: + description: "sqlbuild version from PyPI (empty: latest release)" + required: false + default: "" + type: string + dbt_version: + description: "dbt (v2) version from PyPI" + required: false + default: "2.0.9" + type: string + dbt_core_version: + description: "dbt-core version from PyPI" + required: false + default: "1.12.6" + type: string + dbt_duckdb_version: + description: "dbt-duckdb version from PyPI" + required: false + default: "1.11.0" + type: string + models: + description: "Models in the generated project" + required: false + default: "10000" + type: string + rounds: + description: "Rounds per variant (the summary reports medians)" + required: false + default: "3" + type: string + variants: + description: "Space-separated bench.sh variants" + required: false + default: "sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse core_compile" + type: string + +permissions: + contents: read + +concurrency: + group: benchmark-dbt + cancel-in-progress: false + +jobs: + benchmark: + name: SQLBuild vs dbt (${{ inputs.models }} models) + # The same runner type as the dense compile performance jobs in verify.yml. + runs-on: ubuntu-24.04 + timeout-minutes: 300 + env: + SQLBUILD_VERSION: ${{ inputs.sqlbuild_version }} + DBT_VERSION: ${{ inputs.dbt_version }} + DBT_CORE_VERSION: ${{ inputs.dbt_core_version }} + DBT_DUCKDB_VERSION: ${{ inputs.dbt_duckdb_version }} + MODELS: ${{ inputs.models }} + ROUNDS: ${{ inputs.rounds }} + VARIANTS: ${{ inputs.variants }} + BENCH: ${{ github.workspace }}/benchmarks/dbt-compare + DBT_SEND_ANONYMOUS_USAGE_STATS: "false" + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + - name: Validate inputs + run: | + [[ $MODELS =~ ^[0-9]+$ && $MODELS -ge 100 ]] || { echo "models must be an integer >= 100"; exit 1; } + [[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || { echo "rounds must be a positive integer"; exit 1; } + for spec in "$SQLBUILD_VERSION" "$DBT_VERSION" "$DBT_CORE_VERSION" "$DBT_DUCKDB_VERSION"; do + [[ -z $spec || $spec =~ ^[0-9A-Za-z.+-]+$ ]] || { echo "invalid version: $spec"; exit 1; } + done + - name: Set paths + run: | + { + echo "WORK=$RUNNER_TEMP/dbt-compare" + echo "RESULTS=$RUNNER_TEMP/dbt-compare-results" + echo "SQB=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/sqb" + echo "DBT_V2=$RUNNER_TEMP/dbt-compare/venv-dbt2/bin/dbt" + echo "DBT_CORE=$RUNNER_TEMP/dbt-compare/venv-dbt/bin/dbt" + echo "PYTHON=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/python" + } >>"$GITHUB_ENV" + - name: Install released packages + run: | + uv python install 3.12 + mkdir -p "$WORK" "$RESULTS" + sqlbuild_spec=sqlbuild${SQLBUILD_VERSION:+==$SQLBUILD_VERSION} + uv venv -q -p 3.12 "$WORK/venv-sqb" + uv pip install -q -p "$WORK/venv-sqb/bin/python" "$sqlbuild_spec" pytest duckdb + uv venv -q -p 3.12 "$WORK/venv-dbt2" + uv pip install -q -p "$WORK/venv-dbt2/bin/python" "dbt==$DBT_VERSION" + uv venv -q -p 3.12 "$WORK/venv-dbt" + uv pip install -q -p "$WORK/venv-dbt/bin/python" \ + "dbt-core==$DBT_CORE_VERSION" "dbt-duckdb==$DBT_DUCKDB_VERSION" + - name: Record versions and runner + run: | + { + echo "## Environment" + echo + echo "- Date: $(date -u +%Y-%m-%d)" + echo "- Commit: ${GITHUB_SHA}" + echo "- Runner: ${RUNNER_OS} ${ImageOS:-} ${ImageVersion:-}, $(nproc) CPUs ($(lscpu | sed -n 's/^Model name: *//p')), $(free -g | awk '/^Mem:/ {print $2}') GiB RAM" + echo "- Python: $("$WORK/venv-sqb/bin/python" -c 'import platform; print(platform.python_version())')" + echo "- sqlbuild: $("$WORK/venv-sqb/bin/python" -c 'import importlib.metadata as m; print(m.version("sqlbuild"))')" + echo "- dbt (v2): $("$WORK/venv-dbt2/bin/dbt" --version 2>&1 | head -1)" + echo "- dbt-core: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-core"))')" + echo "- dbt-duckdb: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-duckdb"))')" + echo "- Models: ${MODELS}, rounds: ${ROUNDS}, variants: ${VARIANTS}" + echo + } | tee "$RESULTS/environment.md" + - name: Generate the project + run: | + "$PYTHON" "$BENCH/write_dbt_dense.py" --models "$MODELS" --out "$WORK/project" + # Runs before the timed benchmark so dbt v2's one-time driver download is not timed. + - name: Correctness matrix + run: | + "$BENCH/correctness.sh" "$WORK/correctness" | tee "$RESULTS/correctness.md" + cp -r "$WORK/correctness/logs" "$RESULTS/correctness-logs" + - name: Benchmark + run: | + read -ra variants <<<"$VARIANTS" + "$BENCH/bench.sh" "$WORK/project" "$RESULTS/runs" "$ROUNDS" "${variants[@]}" + - name: Summarize + if: ${{ !cancelled() }} + run: | + { + cat "$RESULTS/environment.md" + echo "## Correctness (50 models)" + echo + cat "$RESULTS/correctness.md" 2>/dev/null || echo "Not run." + echo + echo "## Benchmark (${MODELS} models, medians of ${ROUNDS} rounds)" + echo + python3 "$BENCH/summarize.py" "$RESULTS/runs" 2>&1 || echo "No benchmark results." + } >"$RESULTS/summary.md" + cat "$RESULTS/summary.md" >>"$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: dbt-compare-${{ github.run_id }} + path: ${{ runner.temp }}/dbt-compare-results + retention-days: 90 diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md index 913cae2a9d..683e3f9be2 100644 --- a/benchmarks/dbt-compare/README.md +++ b/benchmarks/dbt-compare/README.md @@ -1,7 +1,8 @@ # SQLBuild vs dbt: dense project compile benchmark This harness generates one dense synthetic project in both SQLBuild and dbt form, then times the -same edit loop in each tool. The published results are on the +same edit loop in each tool. It compares SQLBuild with dbt v2 (the `dbt` package on PyPI), with +dbt Core as a reference. Published results are on the [Benchmarks](https://sqlbuild.com/docs/benchmarks) page. ## Files @@ -37,34 +38,49 @@ average. `sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules; column binding, type inference and contract checks still run. -## Running it +## Running it on GitHub Actions + +The `Benchmark dbt comparison` workflow (`.github/workflows/benchmark-dbt.yml`) runs only when +started by hand. It installs the requested released versions from PyPI, runs `correctness.sh`, then +`bench.sh`, and uploads the results directory with a `summary.md` (environment, correctness table, +benchmark table). The same summary appears on the run page. + +```bash +gh workflow run benchmark-dbt.yml -f sqlbuild_version=0.164.2 -f rounds=3 +``` + +| Input | Default | +|---|---| +| `sqlbuild_version` | empty: the latest release | +| `dbt_version` | `2.0.9` | +| `dbt_core_version` | `1.12.6` | +| `dbt_duckdb_version` | `1.11.0` | +| `models` | `10000` | +| `rounds` | `3` | +| `variants` | all eight variants below | + +## Running it locally Use the released packages in fresh virtual environments. From the repository root: ```bash uv venv -p 3.12 /path/to/venv-sqb && uv pip install -p /path/to/venv-sqb/bin/python sqlbuild pytest duckdb +uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt uv venv -p 3.12 /path/to/venv-dbt && uv pip install -p /path/to/venv-dbt/bin/python dbt-core dbt-duckdb -uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt # optional, dbt v2 - -/path/to/venv-sqb/bin/python benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k -SQB=/path/to/venv-sqb/bin/sqb \ -DBT_CORE=/path/to/venv-dbt/bin/dbt \ -DBT_V2=/path/to/venv-dbt2/bin/dbt \ -PYTHON=/path/to/venv-sqb/bin/python \ - benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3 \ - sqb_rules sqb_norules core_parse core_compile v2_parse v2_compile v2_compile_strict +export SQB=/path/to/venv-sqb/bin/sqb DBT_V2=/path/to/venv-dbt2/bin/dbt \ + DBT_CORE=/path/to/venv-dbt/bin/dbt PYTHON=/path/to/venv-sqb/bin/python +"$PYTHON" benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k +benchmarks/dbt-compare/correctness.sh /path/to/correctness +benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3 python3 benchmarks/dbt-compare/summarize.py /path/to/results - -SQB=... DBT_CORE=... DBT_V2=... PYTHON=... benchmarks/dbt-compare/correctness.sh /path/to/correctness ``` -The generator needs the repository checkout, because it imports the CI helpers, and a Python with -`sqlbuild`, `duckdb` and `pytest` installed. Nothing is built from source. - -Point `HOME` and `XDG_*` at a scratch directory if you don't want either tool to write user-level -caches. +The generator imports the CI helpers from the checkout, so it needs a Python with `sqlbuild`, +`duckdb` and `pytest` installed, and works best with a sqlbuild release close to the checkout. +Nothing is built from source. Run on an otherwise idle machine: other load changes wall times far +more than CPU times. ## What each run does @@ -77,8 +93,8 @@ five seconds, then runs: 4. **edit100**: 100 models' SQL bodies edited (1% of 10,000), then the command. Each edit changes one constant in a projection, so the query changes but its columns, types and -references don't. Rounds alternate the variant order. Before each timed run the script waits until -the 1-minute load average is below `LOAD_MAX` (default 2), for at most `LOAD_WAIT_MAX` seconds. +references don't. Rounds alternate the variant order. `SCENARIOS` limits which of warm, edit1 and +edit100 run. Commands, run from the project copy: @@ -87,9 +103,9 @@ Commands, run from the project copy: | `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` | | `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` | | `sqb_rules_full` | `sqb_rules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile that uses the per-model caches | -| `core_parse` / `core_compile` | `dbt parse` / `dbt compile` with `--profiles-dir ../profiles --no-version-check` | -| `v2_parse` / `v2_compile` | the same commands with dbt v2 | -| `v2_compile_strict` | dbt v2 `compile --static-analysis strict` | +| `v2_parse` / `v2_compile` | dbt v2 `parse` / `compile` with `--profiles-dir ../profiles --no-version-check` | +| `v2_compile_strict` | dbt v2 `compile --static-analysis strict`, which checks columns like SQLBuild does | +| `core_parse` / `core_compile` | the same `parse` / `compile` commands with dbt Core | Both tools use their defaults otherwise: SQLBuild's compile cache, reuse and delta compile, and dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets @@ -98,13 +114,13 @@ dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB `/usr/bin/time -f '%e %U %S %M'` records wall time, user and system CPU, and the peak RSS of the largest process. `summarize.py` also reports what each tool said it did: SQLBuild prints whether it reused the previous compile or compiled only the changed files, and dbt Core logs when it falls back -to a full parse. +to a full parse. The 1-minute load average before and after each run is recorded too. ## What isn't equivalent -- SQLBuild compile binds every column, infers types and checks contracts. dbt Core compile renders - Jinja and resolves references; it doesn't analyse the SQL. dbt v2 does with - `--static-analysis strict`. +- SQLBuild compile binds every column, infers types and checks contracts. dbt v2 does this with + `--static-analysis strict`; its default compile doesn't report unknown columns. dbt Core compile + renders Jinja and resolves references without analysing the SQL. - SQLBuild compile also runs configurable Rules (`sqb_rules`). dbt has no built-in counterpart. - SQLBuild compile prepares its 10,000 SQL tests. dbt `compile` parses the unit tests but doesn't render them; that happens in `dbt test` or `dbt build`. diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh index 9997bdf857..0a77ec6c3e 100755 --- a/benchmarks/dbt-compare/bench.sh +++ b/benchmarks/dbt-compare/bench.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Dense-project compile benchmark: SQLBuild vs dbt Core (and optionally dbt v2). +# Dense-project compile benchmark: SQLBuild vs dbt v2 and dbt Core. # # Usage: bench.sh PROJECT_ROOT OUTDIR ROUNDS [VARIANT...] # @@ -14,12 +14,12 @@ # Environment: # SQB sqb executable (default: sqb) # DBT_CORE dbt Core executable (default: dbt) -# DBT_V2 dbt v2 executable; the v2_* variants are skipped when unset +# DBT_V2 dbt v2 executable (default: dbt2) # PYTHON Python used for the edit helper (default: python3) -# LOAD_MAX wait until the 1-minute load average is below this integer before each timed run -# (default 2) -# LOAD_WAIT_MAX give up waiting after this many seconds and run anyway (default 1800) # SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three) +# +# Variants: sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse +# core_compile. With no VARIANT arguments, all of them run. set -u ROOT=$(realpath "$1") OUT=$(realpath -m "$2") @@ -29,16 +29,12 @@ HERE=$(cd "$(dirname "$0")" && pwd) WORK=$ROOT/work SQB=${SQB:-sqb} DBT_CORE=${DBT_CORE:-dbt} -DBT_V2=${DBT_V2:-} +DBT_V2=${DBT_V2:-dbt2} PYTHON=${PYTHON:-python3} -LOAD_MAX=${LOAD_MAX:-2} -LOAD_WAIT_MAX=${LOAD_WAIT_MAX:-1800} SCENARIOS=" ${SCENARIOS:-warm edit1 edit100} " MODELS=$(find "$ROOT/sqlbuild/models" -name '*.sql' | wc -l) export DBT_SEND_ANONYMOUS_USAGE_STATS=false DBTFLAGS="--profiles-dir $ROOT/profiles --no-version-check" -mkdir -p "$OUT" "$WORK" - declare -A SRC CMD SRC[sqb_rules]=sqlbuild CMD[sqb_rules]="$SQB --no-color compile" @@ -59,37 +55,22 @@ CMD[v2_compile_strict]="$DBT_V2 compile $DBTFLAGS --static-analysis strict" VARIANTS=("$@") if [ ${#VARIANTS[@]} -eq 0 ]; then - VARIANTS=(sqb_rules sqb_norules core_parse core_compile) - [ -n "$DBT_V2" ] && VARIANTS+=(v2_parse v2_compile) + VARIANTS=(sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict + core_parse core_compile) fi +for v in "${VARIANTS[@]}"; do + if [ -z "${CMD[$v]+set}" ]; then + echo "unknown variant: $v" >&2 + exit 2 + fi +done +mkdir -p "$OUT" "$WORK" load1() { cut -d' ' -f1 /proc/loadavg; } -above_limit() { - local load - read -r load _ >"$OUT/uptime.log" - return - fi - sleep 15 - waited=$((waited + 15)) - done - if [ "$waited" -gt 0 ]; then - echo "waited ${waited}s for load below $LOAD_MAX" >>"$OUT/uptime.log" - fi -} - measure() { local v=$1 scenario=$2 round=$3 base base="$OUT/$v.$scenario.r$round" - wait_quiet local before after before=$(load1) /usr/bin/time -f '%e %U %S %M' -o "$base.time" \ @@ -126,3 +107,4 @@ for round in $(seq 1 "$ROUNDS"); do fi done done +rm -rf "${WORK:?}" diff --git a/benchmarks/dbt-compare/correctness.sh b/benchmarks/dbt-compare/correctness.sh index d944dfde8a..235b8fde2a 100755 --- a/benchmarks/dbt-compare/correctness.sh +++ b/benchmarks/dbt-compare/correctness.sh @@ -3,14 +3,14 @@ # upstream model does not have. Run from the repository root. # # Usage: correctness.sh OUTROOT -# Environment: SQB, DBT_CORE, DBT_V2 (optional) and PYTHON, as for bench.sh. PYTHON must be able to +# Environment: SQB, DBT_V2, DBT_CORE and PYTHON, as for bench.sh. PYTHON must be able to # import duckdb, pytest and the installed sqlbuild package (the generator uses the CI helpers). set -u OUT=$(realpath -m "$1") HERE=$(cd "$(dirname "$0")" && pwd) SQB=${SQB:-sqb} DBT_CORE=${DBT_CORE:-dbt} -DBT_V2=${DBT_V2:-} +DBT_V2=${DBT_V2:-dbt2} PYTHON=${PYTHON:-python3} export DBT_SEND_ANONYMOUS_USAGE_STATS=false mkdir -p "$OUT/logs" @@ -43,12 +43,10 @@ for kind in clean ref column; do DBTFLAGS=(--profiles-dir "$P/profiles" --no-version-check) run "$kind" "sqb compile (rules on)" "$P/sqlbuild" "$SQB" --no-color compile run "$kind" "sqb compile (rules off)" "$P/sqlbuild-norules" "$SQB" --no-color compile + run "$kind" "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ + "${DBTFLAGS[@]}" --static-analysis strict run "$kind" "dbt Core parse" "$P/dbt" "$DBT_CORE" parse "${DBTFLAGS[@]}" run "$kind" "dbt Core compile" "$P/dbt" "$DBT_CORE" compile "${DBTFLAGS[@]}" - if [ -n "$DBT_V2" ]; then - run "$kind" "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" - run "$kind" "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" - run "$kind" "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ - "${DBTFLAGS[@]}" --static-analysis strict - fi done diff --git a/benchmarks/dbt-compare/summarize.py b/benchmarks/dbt-compare/summarize.py index 2ce83c8013..fb9fd89983 100644 --- a/benchmarks/dbt-compare/summarize.py +++ b/benchmarks/dbt-compare/summarize.py @@ -10,6 +10,16 @@ from pathlib import Path SCENARIOS = ("cold", "warm", "edit1", "edit100") +VARIANT_ORDER = ( + "sqb_rules", + "sqb_norules", + "sqb_rules_full", + "v2_parse", + "v2_compile", + "v2_compile_strict", + "core_parse", + "core_compile", +) TIME_RE = re.compile(r"^(?P\w+)\.(?P\w+)\.r(?P\d+)\.time$") LOAD_RE = re.compile(r"^(\w+) (\w+) r\d+ load ([\d.]+) ([\d.]+)$") @@ -49,7 +59,8 @@ def main() -> None: (float(wall), float(user) + float(sys_), int(rss), rc, taken) ) loads: dict[tuple[str, str], list[float]] = defaultdict(list) - for line in (out / "uptime.log").read_text().splitlines(): + uptime = out / "uptime.log" + for line in uptime.read_text().splitlines() if uptime.exists() else []: m = LOAD_RE.match(line) if m: loads[(m[1], m[2])] += [float(m[3]), float(m[4])] @@ -59,7 +70,8 @@ def main() -> None: "| peak RSS MiB (median) | exit codes | path | 1-min load before/after |" ) print("|---|---|---|---|---|---|---|---|---|---|") - variants = sorted({variant for variant, _ in runs}) + found = {variant for variant, _ in runs} + variants = [v for v in VARIANT_ORDER if v in found] + sorted(found - set(VARIANT_ORDER)) for variant in variants: for scenario in SCENARIOS: cell = runs.get((variant, scenario))