diff --git a/.github/workflows/benchmark-dbt.yml b/.github/workflows/benchmark-dbt.yml new file mode 100644 index 0000000000..01203854ee --- /dev/null +++ b/.github/workflows/benchmark-dbt.yml @@ -0,0 +1,152 @@ +name: Benchmark dbt comparison + +# Manual only. Times SQLBuild, dbt v2 and dbt Core on the same generated dense DuckDB project with +# released packages from PyPI, and publishes the tables as an artifact and in the job summary. +# See benchmarks/dbt-compare/README.md. +on: + workflow_dispatch: + inputs: + sqlbuild_version: + description: "sqlbuild version from PyPI (empty: latest release)" + required: false + default: "" + type: string + dbt_version: + description: "dbt (v2) version from PyPI" + required: false + default: "2.0.9" + type: string + dbt_core_version: + description: "dbt-core version from PyPI" + required: false + default: "1.12.6" + type: string + dbt_duckdb_version: + description: "dbt-duckdb version from PyPI" + required: false + default: "1.11.0" + type: string + models: + description: "Models in the generated project" + required: false + default: "10000" + type: string + rounds: + description: "Rounds per variant (the summary reports medians)" + required: false + default: "3" + type: string + variants: + description: "Space-separated bench.sh variants" + required: false + default: "sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse core_compile" + type: string + +permissions: + contents: read + +concurrency: + group: benchmark-dbt + cancel-in-progress: false + +jobs: + benchmark: + name: SQLBuild vs dbt (${{ inputs.models }} models) + # The same runner type as the dense compile performance jobs in verify.yml. + runs-on: ubuntu-24.04 + timeout-minutes: 300 + env: + SQLBUILD_VERSION: ${{ inputs.sqlbuild_version }} + DBT_VERSION: ${{ inputs.dbt_version }} + DBT_CORE_VERSION: ${{ inputs.dbt_core_version }} + DBT_DUCKDB_VERSION: ${{ inputs.dbt_duckdb_version }} + MODELS: ${{ inputs.models }} + ROUNDS: ${{ inputs.rounds }} + VARIANTS: ${{ inputs.variants }} + BENCH: ${{ github.workspace }}/benchmarks/dbt-compare + DBT_SEND_ANONYMOUS_USAGE_STATS: "false" + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + - name: Validate inputs + run: | + [[ $MODELS =~ ^[0-9]+$ && $MODELS -ge 100 ]] || { echo "models must be an integer >= 100"; exit 1; } + [[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || { echo "rounds must be a positive integer"; exit 1; } + for spec in "$SQLBUILD_VERSION" "$DBT_VERSION" "$DBT_CORE_VERSION" "$DBT_DUCKDB_VERSION"; do + [[ -z $spec || $spec =~ ^[0-9A-Za-z.+-]+$ ]] || { echo "invalid version: $spec"; exit 1; } + done + - name: Set paths + run: | + { + echo "WORK=$RUNNER_TEMP/dbt-compare" + echo "RESULTS=$RUNNER_TEMP/dbt-compare-results" + echo "SQB=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/sqb" + echo "DBT_V2=$RUNNER_TEMP/dbt-compare/venv-dbt2/bin/dbt" + echo "DBT_CORE=$RUNNER_TEMP/dbt-compare/venv-dbt/bin/dbt" + echo "PYTHON=$RUNNER_TEMP/dbt-compare/venv-sqb/bin/python" + } >>"$GITHUB_ENV" + - name: Install released packages + run: | + uv python install 3.12 + mkdir -p "$WORK" "$RESULTS" + sqlbuild_spec=sqlbuild${SQLBUILD_VERSION:+==$SQLBUILD_VERSION} + uv venv -q -p 3.12 "$WORK/venv-sqb" + uv pip install -q -p "$WORK/venv-sqb/bin/python" "$sqlbuild_spec" pytest duckdb + uv venv -q -p 3.12 "$WORK/venv-dbt2" + uv pip install -q -p "$WORK/venv-dbt2/bin/python" "dbt==$DBT_VERSION" + uv venv -q -p 3.12 "$WORK/venv-dbt" + uv pip install -q -p "$WORK/venv-dbt/bin/python" \ + "dbt-core==$DBT_CORE_VERSION" "dbt-duckdb==$DBT_DUCKDB_VERSION" + - name: Record versions and runner + run: | + { + echo "## Environment" + echo + echo "- Date: $(date -u +%Y-%m-%d)" + echo "- Commit: ${GITHUB_SHA}" + echo "- Runner: ${RUNNER_OS} ${ImageOS:-} ${ImageVersion:-}, $(nproc) CPUs ($(lscpu | sed -n 's/^Model name: *//p')), $(free -g | awk '/^Mem:/ {print $2}') GiB RAM" + echo "- Python: $("$WORK/venv-sqb/bin/python" -c 'import platform; print(platform.python_version())')" + echo "- sqlbuild: $("$WORK/venv-sqb/bin/python" -c 'import importlib.metadata as m; print(m.version("sqlbuild"))')" + echo "- dbt (v2): $("$WORK/venv-dbt2/bin/dbt" --version 2>&1 | head -1)" + echo "- dbt-core: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-core"))')" + echo "- dbt-duckdb: $("$WORK/venv-dbt/bin/python" -c 'import importlib.metadata as m; print(m.version("dbt-duckdb"))')" + echo "- Models: ${MODELS}, rounds: ${ROUNDS}, variants: ${VARIANTS}" + echo + } | tee "$RESULTS/environment.md" + - name: Generate the project + run: | + "$PYTHON" "$BENCH/write_dbt_dense.py" --models "$MODELS" --out "$WORK/project" + # Runs before the timed benchmark so dbt v2's one-time driver download is not timed. + - name: Correctness matrix + run: | + "$BENCH/correctness.sh" "$WORK/correctness" | tee "$RESULTS/correctness.md" + cp -r "$WORK/correctness/logs" "$RESULTS/correctness-logs" + - name: Benchmark + run: | + read -ra variants <<<"$VARIANTS" + "$BENCH/bench.sh" "$WORK/project" "$RESULTS/runs" "$ROUNDS" "${variants[@]}" + - name: Summarize + if: ${{ !cancelled() }} + run: | + { + cat "$RESULTS/environment.md" + echo "## Correctness (50 models)" + echo + cat "$RESULTS/correctness.md" 2>/dev/null || echo "Not run." + echo + echo "## Benchmark (${MODELS} models, medians of ${ROUNDS} rounds)" + echo + python3 "$BENCH/summarize.py" "$RESULTS/runs" 2>&1 || echo "No benchmark results." + } >"$RESULTS/summary.md" + cat "$RESULTS/summary.md" >>"$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: dbt-compare-${{ github.run_id }} + path: ${{ runner.temp }}/dbt-compare-results + retention-days: 90 diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md new file mode 100644 index 0000000000..683e3f9be2 --- /dev/null +++ b/benchmarks/dbt-compare/README.md @@ -0,0 +1,128 @@ +# SQLBuild vs dbt: dense project compile benchmark + +This harness generates one dense synthetic project in both SQLBuild and dbt form, then times the +same edit loop in each tool. It compares SQLBuild with dbt v2 (the `dbt` package on PyPI), with +dbt Core as a reference. Published results are on the +[Benchmarks](https://sqlbuild.com/docs/benchmarks) page. + +## Files + +| File | Purpose | +|---|---| +| `write_dbt_dense.py` | Writes `sqlbuild/`, `sqlbuild-norules/`, `dbt/`, `warehouse.duckdb` and `profiles/` for N models | +| `edit_models.py` | Changes one constant in the SQL body of N evenly spaced models, in either project | +| `bench.sh` | Times cold, warm, one-model edit and 100-model edit runs per variant, in alternating order | +| `summarize.py` | Prints median wall time, CPU time and peak RSS per variant and scenario | +| `correctness.sh` | Runs every tool on 50-model projects: clean, a missing `ref`, and a missing column | + +## The project + +`write_dbt_dense.py` imports the dense generator used by the compile-performance CI jobs +(`scripts/cold_compile_performance/_helpers/dense_project.py`) and switches its adapter to DuckDB. +Nothing else in the SQLBuild project changes. The dbt project is derived from it file by file, so +model names, folders, SQL bodies and the DAG are the same. + +At 10,000 models the project has 2,400 sources, 500 seeds, 250 SQL functions, 120 macros, +17,000 `not_null` checks and 10,000 SQL tests. Models are wide: 8 to 472 columns, about 31 on +average. + +| SQLBuild | dbt | +|---|---| +| `MODEL (...)` header with typed columns | `_models.yml` per folder, `data_type`, `contract: {enforced: true}` | +| `__ref`, `__source`, `__udf` | `ref`, `source`, `function` | +| Python macros | Jinja macros producing the same SQL | +| `audits [not_null]` | `data_tests: [not_null]` | +| SQL tests with fixture CTEs | `unit_tests` with the same input and expected rows | +| `[rules] select = ["SQBR", "XSQBR"]` (built-in Rules plus one custom Rule) | none | + +`sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules; +column binding, type inference and contract checks still run. + +## Running it on GitHub Actions + +The `Benchmark dbt comparison` workflow (`.github/workflows/benchmark-dbt.yml`) runs only when +started by hand. It installs the requested released versions from PyPI, runs `correctness.sh`, then +`bench.sh`, and uploads the results directory with a `summary.md` (environment, correctness table, +benchmark table). The same summary appears on the run page. + +```bash +gh workflow run benchmark-dbt.yml -f sqlbuild_version=0.164.2 -f rounds=3 +``` + +| Input | Default | +|---|---| +| `sqlbuild_version` | empty: the latest release | +| `dbt_version` | `2.0.9` | +| `dbt_core_version` | `1.12.6` | +| `dbt_duckdb_version` | `1.11.0` | +| `models` | `10000` | +| `rounds` | `3` | +| `variants` | all eight variants below | + +## Running it locally + +Use the released packages in fresh virtual environments. From the repository root: + +```bash +uv venv -p 3.12 /path/to/venv-sqb && uv pip install -p /path/to/venv-sqb/bin/python sqlbuild pytest duckdb +uv venv -p 3.12 /path/to/venv-dbt2 && uv pip install -p /path/to/venv-dbt2/bin/python dbt +uv venv -p 3.12 /path/to/venv-dbt && uv pip install -p /path/to/venv-dbt/bin/python dbt-core dbt-duckdb + +export SQB=/path/to/venv-sqb/bin/sqb DBT_V2=/path/to/venv-dbt2/bin/dbt \ + DBT_CORE=/path/to/venv-dbt/bin/dbt PYTHON=/path/to/venv-sqb/bin/python + +"$PYTHON" benchmarks/dbt-compare/write_dbt_dense.py --models 10000 --out /path/to/p10k +benchmarks/dbt-compare/correctness.sh /path/to/correctness +benchmarks/dbt-compare/bench.sh /path/to/p10k /path/to/results 3 +python3 benchmarks/dbt-compare/summarize.py /path/to/results +``` + +The generator imports the CI helpers from the checkout, so it needs a Python with `sqlbuild`, +`duckdb` and `pytest` installed, and works best with a sqlbuild release close to the checkout. +Nothing is built from source. Run on an otherwise idle machine: other load changes wall times far +more than CPU times. + +## What each run does + +For every variant and round, `bench.sh` copies the pristine project into `work/VARIANT`, waits +five seconds, then runs: + +1. **cold**: the command with no `target/` and no caches. +2. **warm**: the same command again with nothing changed. +3. **edit1**: one model's SQL body edited, then the command. +4. **edit100**: 100 models' SQL bodies edited (1% of 10,000), then the command. + +Each edit changes one constant in a projection, so the query changes but its columns, types and +references don't. Rounds alternate the variant order. `SCENARIOS` limits which of warm, edit1 and +edit100 run. + +Commands, run from the project copy: + +| Variant | Command | +|---|---| +| `sqb_rules` | `sqb --no-color compile` in `sqlbuild/` | +| `sqb_norules` | `sqb --no-color compile` in `sqlbuild-norules/` | +| `sqb_rules_full` | `sqb_rules` with `SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1`: every run is a full compile that uses the per-model caches | +| `v2_parse` / `v2_compile` | dbt v2 `parse` / `compile` with `--profiles-dir ../profiles --no-version-check` | +| `v2_compile_strict` | dbt v2 `compile --static-analysis strict`, which checks columns like SQLBuild does | +| `core_parse` / `core_compile` | the same `parse` / `compile` commands with dbt Core | + +Both tools use their defaults otherwise: SQLBuild's compile cache, reuse and delta compile, and +dbt's partial parsing. `DBT_SEND_ANONYMOUS_USAGE_STATS=false` is set. The DuckDB profile sets +`threads: 4`. + +`/usr/bin/time -f '%e %U %S %M'` records wall time, user and system CPU, and the peak RSS of the +largest process. `summarize.py` also reports what each tool said it did: SQLBuild prints whether it +reused the previous compile or compiled only the changed files, and dbt Core logs when it falls back +to a full parse. The 1-minute load average before and after each run is recorded too. + +## What isn't equivalent + +- SQLBuild compile binds every column, infers types and checks contracts. dbt v2 does this with + `--static-analysis strict`; its default compile doesn't report unknown columns. dbt Core compile + renders Jinja and resolves references without analysing the SQL. +- SQLBuild compile also runs configurable Rules (`sqb_rules`). dbt has no built-in counterpart. +- SQLBuild compile prepares its 10,000 SQL tests. dbt `compile` parses the unit tests but doesn't + render them; that happens in `dbt test` or `dbt build`. +- dbt renders every model through Jinja; SQLBuild expands references and macros in its own compiler. + That cost is part of each tool and is included. diff --git a/benchmarks/dbt-compare/bench.sh b/benchmarks/dbt-compare/bench.sh new file mode 100755 index 0000000000..0a77ec6c3e --- /dev/null +++ b/benchmarks/dbt-compare/bench.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Dense-project compile benchmark: SQLBuild vs dbt v2 and dbt Core. +# +# Usage: bench.sh PROJECT_ROOT OUTDIR ROUNDS [VARIANT...] +# +# PROJECT_ROOT is a directory written by write_dbt_dense.py. Each round runs every variant once, in +# alternating order (odd rounds forward, even rounds reversed). For each variant and round: +# 1. copy the pristine project into PROJECT_ROOT/work/VARIANT (no target/, no caches) +# 2. cold: first compile or parse +# 3. warm: the same command again, nothing changed +# 4. edit1: edit one model's SQL body, run again +# 5. edit100: edit 100 models' SQL bodies (1% of 10,000), run again +# +# Environment: +# SQB sqb executable (default: sqb) +# DBT_CORE dbt Core executable (default: dbt) +# DBT_V2 dbt v2 executable (default: dbt2) +# PYTHON Python used for the edit helper (default: python3) +# SCENARIOS which of warm, edit1 and edit100 to run after cold (default: all three) +# +# Variants: sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict core_parse +# core_compile. With no VARIANT arguments, all of them run. +set -u +ROOT=$(realpath "$1") +OUT=$(realpath -m "$2") +ROUNDS=$3 +shift 3 +HERE=$(cd "$(dirname "$0")" && pwd) +WORK=$ROOT/work +SQB=${SQB:-sqb} +DBT_CORE=${DBT_CORE:-dbt} +DBT_V2=${DBT_V2:-dbt2} +PYTHON=${PYTHON:-python3} +SCENARIOS=" ${SCENARIOS:-warm edit1 edit100} " +MODELS=$(find "$ROOT/sqlbuild/models" -name '*.sql' | wc -l) +export DBT_SEND_ANONYMOUS_USAGE_STATS=false +DBTFLAGS="--profiles-dir $ROOT/profiles --no-version-check" +declare -A SRC CMD +SRC[sqb_rules]=sqlbuild +CMD[sqb_rules]="$SQB --no-color compile" +SRC[sqb_norules]=sqlbuild-norules +CMD[sqb_norules]="$SQB --no-color compile" +SRC[sqb_rules_full]=sqlbuild +CMD[sqb_rules_full]="env SQLBUILD_DISABLE_COMPILE_REUSE=1 SQLBUILD_DISABLE_DELTA_COMPILE=1 $SQB --no-color compile" +SRC[core_parse]=dbt +CMD[core_parse]="$DBT_CORE parse $DBTFLAGS" +SRC[core_compile]=dbt +CMD[core_compile]="$DBT_CORE compile $DBTFLAGS" +SRC[v2_parse]=dbt +CMD[v2_parse]="$DBT_V2 parse $DBTFLAGS" +SRC[v2_compile]=dbt +CMD[v2_compile]="$DBT_V2 compile $DBTFLAGS" +SRC[v2_compile_strict]=dbt +CMD[v2_compile_strict]="$DBT_V2 compile $DBTFLAGS --static-analysis strict" + +VARIANTS=("$@") +if [ ${#VARIANTS[@]} -eq 0 ]; then + VARIANTS=(sqb_rules sqb_norules sqb_rules_full v2_parse v2_compile v2_compile_strict + core_parse core_compile) +fi +for v in "${VARIANTS[@]}"; do + if [ -z "${CMD[$v]+set}" ]; then + echo "unknown variant: $v" >&2 + exit 2 + fi +done +mkdir -p "$OUT" "$WORK" + +load1() { cut -d' ' -f1 /proc/loadavg; } + +measure() { + local v=$1 scenario=$2 round=$3 base + base="$OUT/$v.$scenario.r$round" + local before after + before=$(load1) + /usr/bin/time -f '%e %U %S %M' -o "$base.time" \ + bash -c "cd '$WORK/$v' && exec ${CMD[$v]}" >"$base.out" 2>"$base.err" + echo $? >"$base.rc" + after=$(load1) + echo "$v $scenario r$round load $before $after" >>"$OUT/uptime.log" + echo "$v $scenario r$round rc=$(cat "$base.rc") time=$(tail -1 "$base.time") load=$before/$after" | + tee -a "$OUT/summary.log" +} + +edit() { + "$PYTHON" "$HERE/edit_models.py" "$WORK/$1" --models "$MODELS" --count "$2" --value "$3" +} + +for round in $(seq 1 "$ROUNDS"); do + n=${#VARIANTS[@]} + for i in $(seq 0 $((n - 1))); do + if [ $((round % 2)) -eq 0 ]; then v=${VARIANTS[$((n - 1 - i))]}; else v=${VARIANTS[$i]}; fi + rm -rf "${WORK:?}/$v" + cp -a "$ROOT/${SRC[$v]}" "$WORK/$v" + sync + # Let the copy age past file-change heuristics that treat just-written files specially. + sleep 5 + measure "$v" cold "$round" + [[ $SCENARIOS == *" warm "* ]] && measure "$v" warm "$round" + if [[ $SCENARIOS == *" edit1 "* ]]; then + edit "$v" 1 $((round * 10 + 1)) + measure "$v" edit1 "$round" + fi + if [[ $SCENARIOS == *" edit100 "* ]]; then + edit "$v" 100 $((round * 10 + 2)) + measure "$v" edit100 "$round" + fi + done +done +rm -rf "${WORK:?}" diff --git a/benchmarks/dbt-compare/correctness.sh b/benchmarks/dbt-compare/correctness.sh new file mode 100755 index 0000000000..235b8fde2a --- /dev/null +++ b/benchmarks/dbt-compare/correctness.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Correctness matrix on 50-model projects: clean, a reference to a missing model, and a column the +# upstream model does not have. Run from the repository root. +# +# Usage: correctness.sh OUTROOT +# Environment: SQB, DBT_V2, DBT_CORE and PYTHON, as for bench.sh. PYTHON must be able to +# import duckdb, pytest and the installed sqlbuild package (the generator uses the CI helpers). +set -u +OUT=$(realpath -m "$1") +HERE=$(cd "$(dirname "$0")" && pwd) +SQB=${SQB:-sqb} +DBT_CORE=${DBT_CORE:-dbt} +DBT_V2=${DBT_V2:-dbt2} +PYTHON=${PYTHON:-python3} +export DBT_SEND_ANONYMOUS_USAGE_STATS=false +mkdir -p "$OUT/logs" + +for kind in clean ref column; do + args=() + [ "$kind" != clean ] && args=(--seed-error "$kind") + "$PYTHON" "$HERE/write_dbt_dense.py" --models 50 --out "$OUT/p50-$kind" "${args[@]}" || exit 1 +done + +first_problem() { + sed 's/\x1b\[[0-9;]*m//g' "$1" | grep -m1 -E 'error\[|\[error\]|depends on a node' | + sed -E 's/^ +//' | cut -c1-140 +} + +run() { + local kind=$1 label=$2 dir=$3 + shift 3 + local log="$OUT/logs/$kind.$label.log" + rm -rf "$dir/target" "$dir/logs" + (cd "$dir" && "$@" >"$log" 2>&1) + local rc=$? + printf '| %s | %s | %s | %s |\n' "$kind" "$label" "$rc" "$(first_problem "$log")" +} + +echo "| project | tool and command | exit code | first problem reported |" +echo "|---|---|---|---|" +for kind in clean ref column; do + P=$OUT/p50-$kind + DBTFLAGS=(--profiles-dir "$P/profiles" --no-version-check) + run "$kind" "sqb compile (rules on)" "$P/sqlbuild" "$SQB" --no-color compile + run "$kind" "sqb compile (rules off)" "$P/sqlbuild-norules" "$SQB" --no-color compile + run "$kind" "dbt v2 parse" "$P/dbt" "$DBT_V2" parse "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile" "$P/dbt" "$DBT_V2" compile "${DBTFLAGS[@]}" + run "$kind" "dbt v2 compile --static-analysis strict" "$P/dbt" "$DBT_V2" compile \ + "${DBTFLAGS[@]}" --static-analysis strict + run "$kind" "dbt Core parse" "$P/dbt" "$DBT_CORE" parse "${DBTFLAGS[@]}" + run "$kind" "dbt Core compile" "$P/dbt" "$DBT_CORE" compile "${DBTFLAGS[@]}" +done diff --git a/benchmarks/dbt-compare/edit_models.py b/benchmarks/dbt-compare/edit_models.py new file mode 100644 index 0000000000..d77b50a0de --- /dev/null +++ b/benchmarks/dbt-compare/edit_models.py @@ -0,0 +1,50 @@ +"""Edit the SQL body of evenly spaced models in a generated dense project. + +Usage: python edit_models.py PROJECT_DIR --models TOTAL --count N --value V + +Works on both the SQLBuild and the dbt project. Each selected model's last plain +`COALESCE(b.amount, )` projection gets the constant V, so the edit changes the query but not the +model's columns, types, references or header. +""" + +import argparse +import re +from pathlib import Path + +PATTERN = re.compile(r"CAST\(COALESCE\(b\.amount, \d+\)") + + +def model_name(index: int) -> str: + return f"sales{index // 1000:03d}__int_clean__orders_{index:05d}" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("project", type=Path) + parser.add_argument("--models", type=int, required=True) + parser.add_argument("--count", type=int, required=True) + parser.add_argument("--value", type=int, required=True) + args = parser.parse_args() + step = args.models // args.count + for k in range(args.count): + index = k * step + step // 2 + path = ( + args.project + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{model_name(index)}.sql" + ) + text = path.read_text() + matches = list(PATTERN.finditer(text)) + assert matches, path + last = matches[-1] + replacement = f"CAST(COALESCE(b.amount, {args.value})" + assert last.group(0) != replacement, path + path.write_text(text[: last.start()] + replacement + text[last.end() :]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/dbt-compare/summarize.py b/benchmarks/dbt-compare/summarize.py new file mode 100644 index 0000000000..fb9fd89983 --- /dev/null +++ b/benchmarks/dbt-compare/summarize.py @@ -0,0 +1,94 @@ +"""Summarize bench.sh output: median wall, user+sys CPU and peak RSS per variant and scenario. + +Usage: python summarize.py OUTDIR +""" + +import re +import statistics +import sys +from collections import defaultdict +from pathlib import Path + +SCENARIOS = ("cold", "warm", "edit1", "edit100") +VARIANT_ORDER = ( + "sqb_rules", + "sqb_norules", + "sqb_rules_full", + "v2_parse", + "v2_compile", + "v2_compile_strict", + "core_parse", + "core_compile", +) +TIME_RE = re.compile(r"^(?P\w+)\.(?P\w+)\.r(?P\d+)\.time$") +LOAD_RE = re.compile(r"^(\w+) (\w+) r\d+ load ([\d.]+) ([\d.]+)$") + + +def path_taken(variant: str, out: str, err: str) -> str: + """What each tool reported doing: full work, a partial parse, a replay or a delta compile.""" + if variant.startswith("sqb"): + if "Inputs unchanged; reused the previous compile" in err: + return "reused" + if "against the previous compile" in err: + return "delta" + return "full" + if variant.startswith("core"): + text = out + err + if "Unable to do partial parsing" in text or "Starting full parse" in text: + return "full parse" + return "partial parse" + return "-" + + +def main() -> None: + out = Path(sys.argv[1]) + runs: dict[tuple[str, str], list[tuple[float, float, int, int, str]]] = defaultdict(list) + for path in sorted(out.glob("*.time")): + m = TIME_RE.match(path.name) + if not m: + continue + stem = str(path)[: -len(".time")] + wall, user, sys_, rss = path.read_text().strip().splitlines()[-1].split() + rc = int(Path(stem + ".rc").read_text()) + taken = path_taken( + m["variant"], + Path(stem + ".out").read_text(errors="replace"), + Path(stem + ".err").read_text(errors="replace"), + ) + runs[(m["variant"], m["scenario"])].append( + (float(wall), float(user) + float(sys_), int(rss), rc, taken) + ) + loads: dict[tuple[str, str], list[float]] = defaultdict(list) + uptime = out / "uptime.log" + for line in uptime.read_text().splitlines() if uptime.exists() else []: + m = LOAD_RE.match(line) + if m: + loads[(m[1], m[2])] += [float(m[3]), float(m[4])] + + print( + "| variant | scenario | runs | wall s (median) | wall s (all) | user+sys s (median) " + "| peak RSS MiB (median) | exit codes | path | 1-min load before/after |" + ) + print("|---|---|---|---|---|---|---|---|---|---|") + found = {variant for variant, _ in runs} + variants = [v for v in VARIANT_ORDER if v in found] + sorted(found - set(VARIANT_ORDER)) + for variant in variants: + for scenario in SCENARIOS: + cell = runs.get((variant, scenario)) + if not cell: + continue + walls = [r[0] for r in cell] + load = loads.get((variant, scenario), [0.0]) + print( + f"| {variant} | {scenario} | {len(cell)} | {statistics.median(walls):.2f} " + f"| {', '.join(f'{w:.1f}' for w in walls)} " + f"| {statistics.median(r[1] for r in cell):.1f} " + f"| {statistics.median(r[2] for r in cell) / 1024:.0f} " + f"| {','.join(sorted({str(r[3]) for r in cell}))} " + f"| {','.join(sorted({r[4] for r in cell}))} " + f"| {min(load):.2f}-{max(load):.2f} |" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py new file mode 100644 index 0000000000..aa0801bd76 --- /dev/null +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -0,0 +1,342 @@ +# ruff: noqa: E501, PLR0915 +"""Write the same dense project for SQLBuild (DuckDB) and dbt (DuckDB). + +Usage, from the repository root: + + python benchmarks/dbt-compare/write_dbt_dense.py --models N --out DIR [--seed-error ref|column] + +Creates under DIR: + sqlbuild/ SQLBuild dense project (the CI dense generator, adapter switched to duckdb) + sqlbuild-norules/ the same project with `[rules] select = []` + dbt/ dbt project derived file by file from `sqlbuild/` + warehouse.duckdb DuckDB file holding every source table (empty) + profiles/profiles.yml + +The SQLBuild project comes unchanged from the generator the compile-performance CI jobs use. The +dbt project is derived from the generated SQLBuild files, so SQL, DAG, names and folders match. +""" + +import argparse +import re +import shutil +import sys +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPOSITORY_ROOT)) + +from scripts.cold_compile_performance._helpers.dense_project import ( # noqa: E402 + dense_column_count, + dense_model_name, + write_dense_compile_project, +) + +SOURCE_NAME = "raw" +SOURCE_SCHEMA = "raw" +DATABASE = "warehouse" +ERROR_MODEL_INDEX = 2 +ERROR_COLUMN = "missing_col" +ERROR_REF = "sales000__int_clean__orders_missing" + +COLUMN_RE = re.compile( + r'^\s+(\w+) \(type (\w+), nullable (true|false), description "([^"]*)"(, audits \[not_null\])?\)' +) +MACRO_CALL_RE = re.compile(r'@(macro_\d{3}_\d{5})\("([^"]*)"\)') +CAST_RE = re.compile(r"CAST\(([^ ]+) AS (\w+)\) AS (\w+)") + + +def q(text: str) -> str: + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def write_sqlbuild(out: Path, n: int) -> Path: + project = out / "sqlbuild" + write_dense_compile_project(project_dir=project, model_count=n) + toml = project / "sqlbuild_project.toml" + text = toml.read_text() + assert 'adapter = "snowflake"' in text + toml.write_text(text.replace('adapter = "snowflake"', 'adapter = "duckdb"')) + return project + + +def write_norules(project: Path) -> Path: + target = project.parent / "sqlbuild-norules" + shutil.copytree(project, target) + toml = target / "sqlbuild_project.toml" + text = toml.read_text() + new = text.replace('select = ["SQBR", "XSQBR"]', "select = []") + assert new != text + toml.write_text(new) + return target + + +def model_header_and_body(text: str) -> tuple[list[tuple[str, str, bool, str, bool]], str]: + header, body = text.split("\n));\n", 1) + columns = [] + for line in header.splitlines()[1:]: + m = COLUMN_RE.match(line) + assert m, line + columns.append((m[1], m[2], m[3] == "false", m[4], bool(m[5]))) + return columns, body + + +def to_jinja_sql(body: str) -> str: + body = re.sub(r'__ref\("([^"]+)"\)', r"{{ ref('\1') }}", body) + body = re.sub(r'__source\("([^"]+)"\)', rf"{{{{ source('{SOURCE_NAME}', '\1') }}}}", body) + body = re.sub(r'__udf\("([^"]+)"\)\(', r"{{ function('\1') }}(", body) + body = MACRO_CALL_RE.sub(lambda m: "{{ " + m[1] + "(" + q(m[2]) + ") }}", body) + assert not re.search(r"__(ref|source|udf)\(", body), body[:400] + assert "@" not in body + return body.lstrip("\n") + + +def parse_cast_row(select_list: str) -> dict[str, str]: + return {col: value for value, _type, col in CAST_RE.findall(select_list)} + + +def parse_test(text: str) -> tuple[dict[str, dict[str, str]], dict[str, str]]: + inputs: dict[str, dict[str, str]] = {} + expected: dict[str, str] = {} + for line in text.splitlines(): + line = line.strip().removeprefix("WITH ").rstrip(",") + m = re.match(r"(__(ref|source|expected)__(\w+)) AS \(SELECT (.*)\)$", line) + if not m: + continue + row = parse_cast_row(m[4]) + if m[2] == "expected": + expected = row + elif m[2] == "ref": + inputs[f"ref('{m[3]}')"] = row + else: + inputs[f"source('{SOURCE_NAME}', '{m[3]}')"] = row + assert expected and inputs, text[:300] + return inputs, expected + + +def flow_row(row: dict[str, str]) -> str: + return "{" + ", ".join(f"{k}: {v}" for k, v in row.items()) + "}" + + +def write_dbt(sqb: Path, out: Path, n: int) -> Path: + dbt = out / "dbt" + dbt.mkdir() + (dbt / "dbt_project.yml").write_text( + "name: dense_orders\n" + "version: '1.0.0'\n" + "profile: dense_orders\n" + "model-paths: [models]\n" + "macro-paths: [macros]\n" + "seed-paths: [seeds]\n" + "function-paths: [functions]\n" + "test-paths: [tests]\n" + "models:\n dense_orders:\n +materialized: table\n" + ) + # Sources + source_count = max(1, n * 24 // 100) + lines = [ + "sources:", + f" - name: {SOURCE_NAME}", + f" database: {DATABASE}", + f" schema: {SOURCE_SCHEMA}", + " tables:", + ] + for index in range(source_count): + width = dense_column_count(index % 240) + lines += [ + f" - name: orders_{index:05d}", + f" description: Generated source orders_{index:05d}.", + " columns:", + " - name: id", + " data_type: integer", + " constraints: [{type: not_null}]", + " - name: amount", + " data_type: double", + ] + for column in range(width - 2): + lines += [f" - name: quantity_{column:03d}", " data_type: double"] + (dbt / "models").mkdir() + (dbt / "models" / "_sources.yml").write_text("\n".join(lines) + "\n") + # Seeds + (dbt / "seeds").mkdir() + seed_yml = ["seeds:"] + for csv in sorted((sqb / "seeds").glob("*.csv")): + shutil.copy(csv, dbt / "seeds" / csv.name) + seed_yml += [ + f" - name: {csv.stem}", + f" description: Test seed {csv.stem}.", + " config:", + " column_types: {id: integer, label: varchar}", + " columns:", + " - name: id", + " - name: label", + ] + (dbt / "seeds" / "_seeds.yml").write_text("\n".join(seed_yml) + "\n") + # Functions (dbt UDF resources) + (dbt / "functions").mkdir() + fn_yml = ["functions:"] + for fn in sorted((sqb / "functions" / "sql").glob("*.sql")): + body = fn.read_text().split(");\n", 1)[1].strip() + (dbt / "functions" / fn.name).write_text(body + "\n") + fn_yml += [ + f" - name: {fn.stem}", + " description: Test function.", + " arguments:", + " - name: input_value", + " data_type: double", + " returns:", + " data_type: double", + ] + (dbt / "functions" / "_functions.yml").write_text("\n".join(fn_yml) + "\n") + # Macros: SQLBuild Python macros -> Jinja macros, same per-cohort bucket layout + macro_def_re = re.compile( + r"def (macro_\d{3}_\d{5})\(expression: str\) -> str:\n return (.*)\n" + ) + offset_re = re.compile( + r'def _offset\(expression: str\) -> str:\n return f"\(\{expression\} \+ (\d+)\)"' + ) + base_re = re.compile( + r'def (base_offset_\d{3})\(expression: str\) -> str:\n return f"\(\{expression\} \+ 1\)"' + ) + for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/*/*.py")): + text = macro.read_text() + name, ret = macro_def_re.search(text).groups() + offset = offset_re.search(text)[1] + parts = [] + base = base_re.search(text) + if base: + parts.append( + f"{{% macro {base[1]}(expression) -%}}({{{{ expression }}}} + 1){{%- endmacro %}}" + ) + assert ret == f"{base[1]}(_offset(expression))" + parts.append( + f"{{% macro {name}(expression) -%}}" + f"{{{{ {base[1]}('(' ~ expression ~ ' + {offset})') }}}}{{%- endmacro %}}" + ) + else: + assert ret == "_offset(expression)" + parts.append( + f"{{% macro {name}(expression) -%}}({{{{ expression }}}} + {offset}){{%- endmacro %}}" + ) + domain = macro.relative_to(sqb / "models").parts[0] + target = dbt / "macros" / domain / macro.parent.name / (macro.stem + ".sql") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("\n".join(parts) + "\n") + # Models + properties + unit tests, one YAML per batch folder + for index in range(n): + name = dense_model_name(index) + domain = f"sales{index // 1000:03d}" + batch = f"batch{index // 5:05d}" + rel = Path(domain) / "intermediate" / "clean" / batch + src = sqb / "models" / rel / f"{name}.sql" + columns, body = model_header_and_body(src.read_text()) + out_dir = dbt / "models" / rel + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / f"{name}.sql").write_text(to_jinja_sql(body)) + yml_path = out_dir / "_models.yml" + if not yml_path.exists(): + yml_path.write_text("models:\n") + entry = [ + f" - name: {name}", + " description: Generated dense model.", + " config:", + " materialized: table", + " contract: {enforced: true}", + " columns:", + ] + for col, typ, not_null, desc, audit in columns: + entry += [ + f" - name: {col}", + f" data_type: {typ.lower()}", + f" description: {desc}", + ] + if not_null: + entry.append(" constraints: [{type: not_null}]") + if audit: + entry.append(" data_tests: [not_null]") + with yml_path.open("a") as fh: + fh.write("\n".join(entry) + "\n") + test_src = ( + sqb / "tests" / "unit" / rel / f"test_{name.replace('__', '_')}__preserves_order_id.sql" + ) + inputs, expected = parse_test(test_src.read_text()) + ut_path = out_dir / "_unit_tests.yml" + if not ut_path.exists(): + ut_path.write_text("unit_tests:\n") + ut = [f" - name: {name}__preserves_order_id", f" model: {name}", " given:"] + for inp, row in inputs.items(): + ut += [f" - input: {inp}", " rows:", f" - {flow_row(row)}"] + ut += [" expect:", " rows:", f" - {flow_row(expected)}"] + with ut_path.open("a") as fh: + fh.write("\n".join(ut) + "\n") + (dbt / "tests").mkdir() + return dbt + + +def write_warehouse(out: Path, n: int) -> None: + import duckdb + + path = (out / f"{DATABASE}.duckdb").resolve() + if path.exists(): + path.unlink() + con = duckdb.connect(str(path)) + con.execute(f"create schema {SOURCE_SCHEMA}") + con.execute("create schema analytics") + for index in range(max(1, n * 24 // 100)): + width = dense_column_count(index % 240) + cols = ["id integer not null", "amount double"] + [ + f"quantity_{c:03d} double" for c in range(width - 2) + ] + con.execute(f"create table {SOURCE_SCHEMA}.orders_{index:05d} ({', '.join(cols)})") + con.close() + profiles = out / "profiles" + profiles.mkdir(exist_ok=True) + (profiles / "profiles.yml").write_text( + "dense_orders:\n target: dev\n outputs:\n dev:\n type: duckdb\n" + f" path: {path}\n schema: analytics\n threads: 4\n" + ) + + +def seed_error(out: Path, kind: str) -> None: + """Break model ERROR_MODEL_INDEX the same way in every project. + + `column` selects a column the upstream model does not have; `ref` references a model that does + not exist. + """ + name = dense_model_name(ERROR_MODEL_INDEX) + upstream = dense_model_name(ERROR_MODEL_INDEX - 1) + for tool in ("sqlbuild", "sqlbuild-norules", "dbt"): + for path in (out / tool / "models").rglob(f"{name}.sql"): + text = path.read_text() + if kind == "column": + new = text.replace( + "COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + f"COALESCE(b.{ERROR_COLUMN}, 0) + CAST(b.id AS DOUBLE)", + 1, + ) + else: + new = text.replace(f'__ref("{upstream}")', f'__ref("{ERROR_REF}")', 1).replace( + f"{{{{ ref('{upstream}') }}}}", f"{{{{ ref('{ERROR_REF}') }}}}", 1 + ) + assert new != text + path.write_text(new) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--models", type=int, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--seed-error", choices=("ref", "column")) + args = parser.parse_args() + if args.out.exists(): + shutil.rmtree(args.out) + args.out.mkdir(parents=True) + sqb = write_sqlbuild(args.out, args.models) + write_dbt(sqb, args.out, args.models) + write_norules(sqb) + write_warehouse(args.out, args.models) + if args.seed_error: + seed_error(args.out, args.seed_error) + + +if __name__ == "__main__": + main()