perf(compiler): reduce compile memory and latency

.github/workflows/verify.yml

@@ -18,6 +18,8 @@ jobs:
     outputs:
       duckdb: ${{ steps.scope.outputs.duckdb }}
       postgres: ${{ steps.scope.outputs.postgres }}
+      performance: ${{ steps.scope.outputs.performance }}
+      rules_performance: ${{ steps.scope.outputs.rules_performance }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -33,21 +35,43 @@ jobs:
             {
               echo "duckdb=$1"
               echo "postgres=$2"
+              echo "performance=$3"
+              echo "rules_performance=$4"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" ]]; then
-            write_scope true true
+            write_scope true true true true
             exit 0
           fi
 
           duckdb=false
           postgres=false
+          performance=false
+          rules_performance=false
           broad=false
           changed=false
           while IFS= read -r -d '' path; do
             changed=true
             case "${path}" in
+              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
+                broad=true
+                performance=true
+                rules_performance=true
+                ;;
+              src/sqlbuild/compiler/*|src/sqlbuild/adapter/type_system/*)
+                broad=true
+                performance=true
+                rules_performance=true
+                ;;
+              src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
+                broad=true
+                performance=true
+                ;;
+              src/sqlbuild/rule_engine/*|src/sqlbuild/lint/*|scripts/rules_benchmark/*|scripts/benchmark_rules_ci.py|tests/performance/*)
+                broad=true
+                rules_performance=true
+                ;;
               tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/virtual/state/classes/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*)
                 postgres=true
                 ;;
@@ -56,6 +80,10 @@ jobs:
                 ;;
               tests/unit/*|tests/integration/*)
                 ;;
+              .github/workflows/verify.yml|Makefile)
+                performance=true
+                rules_performance=true
+                ;;
               .github/*|docs/*|*.md|.gitignore|LICENSE)
                 ;;
               *)
@@ -64,12 +92,16 @@ jobs:
             esac
           done < <(git diff --name-only -z "${BASE_SHA}" "${HEAD_SHA}")
 
-          if [[ "${changed}" != "true" || "${broad}" == "true" ]]; then
-            write_scope true true
+          if [[ "${changed}" != "true" ]]; then
+            write_scope true true true true
             exit 0
           fi
 
-          write_scope "${duckdb}" "${postgres}"
+          if [[ "${broad}" == "true" ]]; then
+            duckdb=true
+            postgres=true
+          fi
+          write_scope "${duckdb}" "${postgres}" "${performance}" "${rules_performance}"
 
   lockfile:
     name: Lockfile consistency
@@ -202,6 +234,53 @@ jobs:
       - name: Run performance guards
         run: make test-e2e-performance
 
+  cold_compile_performance:
+    name: Fresh-process compile (${{ matrix.models }} models)
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.performance == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        models: [3000, 5000, 10000]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Run fresh-process compile guard under 4 GiB
+        env:
+          BENCHMARK_MODELS: ${{ matrix.models }}
+          MEMORY_LIMIT_BYTES: "4294967296"
+        run: |
+          sudo systemd-run \
+            --unit="sqlbuild-cold-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
+            --uid="$(id -u)" \
+            --gid="$(id -g)" \
+            --same-dir \
+            --setenv="HOME=${HOME}" \
+            --setenv="PATH=${PATH}" \
+            --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
+            --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
+            --property="MemorySwapMax=0" \
+            --wait \
+            --collect \
+            --pipe \
+            make test-e2e-cold-compile-performance \
+              SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
+
   rules_performance:
     name: Rules performance guards
     needs: [changes, lockfile]
@@ -286,7 +365,17 @@ jobs:
   verify:
     name: Verify
     if: always()
-    needs: [changes, lockfile, checks, e2e_shards, performance, rules_performance, postgres]
+    needs:
+      [
+        changes,
+        lockfile,
+        checks,
+        e2e_shards,
+        performance,
+        cold_compile_performance,
+        rules_performance,
+        postgres,
+      ]
     runs-on: ubuntu-24.04
     steps:
       - name: Confirm all required verification jobs passed
@@ -296,6 +385,7 @@ jobs:
           CHECKS_RESULT: ${{ needs.checks.result }}
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
+          COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
         run: |
@@ -308,6 +398,7 @@ jobs:
           for result in \
             "${E2E_SHARDS_RESULT}" \
             "${PERFORMANCE_RESULT}" \
+            "${COLD_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then

Cargo.toml

@@ -11,11 +11,11 @@ publish = false
 rust-version = "1.88"
 
 [workspace.dependencies]
-fensu-policy = "=0.13.0"
+fensu-policy = "=0.22.0"
 globset = "0.4"
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "afd2c73a2fe40f59e275aa918264c41ae83cbe94", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "190b1f8265b6135a9a6a100356599dd99487c1b4", default-features = false, features = [
     "ast-tools",
     "builder",
     "dialect-bigquery",
@@ -28,6 +28,8 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "afd2c73a2
     "stacker",
 ] }
 quote = "1"
+rayon = "1"
+regex = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha2 = "0.10"

Makefile

@@ -4,7 +4,8 @@ SHELL := /bin/bash
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
 	test-e2e-duckdb-build-incremental test-e2e-duckdb-build-virtual \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \
-	test-e2e-duckdb-integrations test-e2e-performance
+	test-e2e-duckdb-integrations test-e2e-performance \
+	test-e2e-cold-compile-performance
 
 format:
 	uv run ruff format .
@@ -223,13 +224,21 @@ test-e2e-performance:
 	@log="/tmp/opencode/test-e2e-performance-$$(date +%Y%m%d-%H%M%S).log"; \
 	echo "Logging to $$log"; \
 	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests/e2e \
-		-m "performance and not real_warehouse and not dbt" \
+		-m "performance and not cold_compile_performance and not real_warehouse and not dbt" \
 		-vv --log-cli-level=INFO --color=yes 2>&1 | tee "$$log"; \
 	status=$${PIPESTATUS[0]}; \
 	echo "TEST_E2E_PERFORMANCE_EXIT=$$status (log: $$log)" | tee -a "$$log"; \
 	exit $$status
 
 
+test-e2e-cold-compile-performance:
+	test -n "$(SQLBUILD_BENCHMARK_MODELS)"
+	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
+		tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py \
+		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
+		-vv --log-cli-level=INFO --color=yes
+
+
 test-virtual:
 	@mkdir -p /tmp/opencode
 	@log="/tmp/opencode/test-virtual-$$(date +%Y%m%d-%H%M%S).log"; \

crates/sqlbuild-rules-native/Cargo.toml

@@ -19,6 +19,8 @@ fensu-policy.workspace = true
 globset.workspace = true
 polyglot-sql.workspace = true
 pyo3.workspace = true
+rayon.workspace = true
+regex.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true