feat(benchmarks): add the DuckShop 10K generator (#1248)

Author: kvlonge

.github/workflows/verify.yml

@@ -163,7 +163,7 @@ jobs:
                 performance=true
                 rules_performance=true
                 ;;
-              src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
+              benchmarks/duckshop-10k/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
                 broad=true
                 performance=true
                 ;;

Makefile

@@ -299,6 +299,14 @@ test-e2e-dense-compile-performance:
 		-n auto --dist loadfile \
 		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \
 		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
+# DuckShop 10K runs after the dense guards, alone, so it never shares the memory limit with them.
+	if [ "$(SQLBUILD_BENCHMARK_MODELS)" = "10000" ]; then \
+		env PYTHONUNBUFFERED=1 uv run pytest benchmarks/duckshop-10k/tests -q --color=yes && \
+		env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
+			tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py \
+			-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \
+			-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes; \
+	fi
 
 test-e2e-cache-compile-performance:
 	test -n "$(SQLBUILD_BENCHMARK_MODELS)"

benchmarks/duckshop-10k/README.md

@@ -0,0 +1,66 @@
+# DuckShop 10K
+
+A deterministic generator for a ten-thousand-model SQLBuild project in Snowflake SQL. It describes
+Mallard & Co., the fictional duck-themed home goods business of the
+[DuckShop](../duckshop/README.md) project, after a decade of growth: 40 storefronts in 10 regions,
+marketplace partners, carriers, warehouses, retail stores, suppliers, payment gateways, ad
+networks, messaging, affiliates, support tools, ledgers, wholesale desks, returns portals and a
+product catalogue. Every name and value is invented.
+
+Only the generator, its tests and `checksum.txt` are committed. The generated project is written
+to a directory you choose.
+
+## Usage
+
+```bash
+python benchmarks/duckshop-10k/generate.py --out /path/to/duckshop-10k   # write the project
+python benchmarks/duckshop-10k/generate.py --verify /path/to/duckshop-10k  # check a written copy
+python benchmarks/duckshop-10k/generate.py --check            # check an in-memory build
+cd /path/to/duckshop-10k && sqb compile                        # no warehouse connection needed
+```
+
+`--profile smoke` writes a small project of about 1,100 models with the same templates, for fast
+iteration. `checksum.txt` records the full profile only.
+
+## Determinism and versioning
+
+The generator uses no randomness, clocks or environment: the same generator version always writes
+byte-identical files. `checksum.txt` holds the generator version, the file count and the SHA-256 of
+`MANIFEST.sha256`, a sorted list of the SHA-256 of every generated file.
+
+When a change alters any generated file, bump `GENERATOR_VERSION` in `duckshop10k/__init__.py` and
+run `python benchmarks/duckshop-10k/generate.py --update-checksum`.
+
+## Shape
+
+| | Count |
+|---|---|
+| Models | 10,282 |
+| Staging, intermediate, marts and snapshots | 6,149 (59.8%), 1,961 (19.1%), 2,162 (21.0%), 10 |
+| Sources | 2,466 table sources and one expression source |
+| Unit tests and scenarios | 288 and 2 |
+| Dependency depth | 11 at most; 168 models at depth 7 or more |
+
+Most models read one to four upstreams; a few dozen wide marts, unions and scorecards read ten or
+more. Staging deduplicates change logs with a change-log macro, types three platform layouts
+(legacy columns with local times, typed columns with offset times, and JSON documents with epoch
+times) and flattens arrays into child models. The intermediate layer converts every instance to a
+shared shape (UTC times, major-unit amounts, mapped statuses), enriches headers with child totals,
+unions instances by region and company, converts currencies with `ASOF JOIN`, sessionises
+clickstream events in microbatches and builds customer identity, value and segment models and a
+budget variance chain. Marts give every instance, team and region its reports, and publish facts and
+dimensions with enforced contracts.
+
+The models that use `HASH`, `TIMESTAMP_LTZ` or `CURRENT_TIMESTAMP()` are tagged `known_divergence`,
+because their results depend on the engine or the session.
+
+## Checks
+
+- `benchmarks/duckshop-10k/tests/` checks the shape, the determinism of two builds, the recorded
+  checksum and the generated custom Rule's tests, offline.
+- `tests/e2e/.../compile/test_duckshop_10k_compile.py` generates the project, verifies it against
+  `checksum.txt` and compiles it cold with the checkout's build, requiring zero errors and zero
+  warnings.
+
+CI runs both in the 10,000-model cell of the dense compile job, under its 4 GiB memory limit, after
+the dense guards have finished, so the DuckShop compile never shares the limit with them.

benchmarks/duckshop-10k/checksum.txt

@@ -0,0 +1 @@
+generator 1 files 10874 sha256 4d276ad2428841efa2a1b851c83863bea70505d9e085a71390401399b5667667

benchmarks/duckshop-10k/duckshop10k/__init__.py

@@ -0,0 +1,9 @@
+"""Deterministic generator for DuckShop 10K, a ten-thousand-model SQLBuild project.
+
+The same generator version always writes byte-identical output. Bump GENERATOR_VERSION, and
+regenerate checksum.txt, whenever a change alters any generated file.
+"""
+
+GENERATOR_VERSION = "1"
+SEED = 20261024
+PROJECT_NAME = "duckshop_10k"