perf(compile): close the fresh compile gap with ordered native scheduling and mimalloc (#889)

* perf(analysis): schedule native query analysis largest first

Hand out each native analysis batch through a shared queue ordered by SQL size, so the widest queries start first instead of becoming a late critical path. Results keep request order. Fresh 3000 native analysis wall falls from about 2.24 s to 1.91 s with identical output.

* perf(native): use mimalloc as the native extension allocator

Native analysis spends about 40% of its time in the system allocator building and dropping syntax trees. mimalloc (local-dynamic TLS for the dlopen-loaded extension) cuts fresh 3000 native analysis from about 6.4 to 4.8 CPU-s and 1.9 to 1.4 s wall with identical output. Fresh 3000 peak memory falls from about 830 to 690 MB; dense 3000 CPU falls about 7% at similar peak memory.

* perf(tests): reuse parsed model SQL when rendering SQL-test comparisons

Tests in one project render the same model SQL many times while lifting its CTEs into the comparison query; fresh 3000 splits about 1,150 statements from only 50 distinct SQL strings. Keep a parse of SQL once it repeats within a render batch, bounded to 1 MiB of SQL text, and clone it for each later use; per-test CTE renaming is unchanged. Unrepeated SQL is never retained, so dense projects keep their peak memory. Fresh 3000 native planning and rendering falls from about 455 to 380 ms with identical artifacts.

* ci: restore the fresh same-runner compile ratio limit to 1.10

Fresh 3000 --no-cache is back within 1.10x of 0.119.1 on the same machine (medians of 3 interleaved runs: 8.22 s vs 7.86 s wall, 12.56 vs 12.54 CPU-s), so the temporary 1.30 limit is no longer needed.

* ci: build every wheel platform on pull requests that change native inputs

Build the same Linux, macOS and Windows wheel matrix as publish.yml, without uploading, whenever crates, Cargo manifests, pyproject.toml or the verify workflow change, and smoke-import sqlbuild._native from each built wheel. The aggregate Verify job treats the job like the other path-selected suites.

* test: allow the 10,000-model cache-write CPU overhead the same 1.15 as wall

Author: kvlonge

.github/workflows/publish.yml

@@ -34,6 +34,7 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
   wheels:
+    # Keep the platform matrix and maturin settings in sync with verify.yml.
     name: Build wheel (${{ matrix.platform.name }})
     needs: release-tag
     runs-on: ${{ matrix.platform.runner }}

.github/workflows/verify.yml

@@ -22,6 +22,7 @@ jobs:
       performance: ${{ steps.scope.outputs.performance }}
       format_performance: ${{ steps.scope.outputs.format_performance }}
       rules_performance: ${{ steps.scope.outputs.rules_performance }}
+      wheels: ${{ steps.scope.outputs.wheels }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -41,11 +42,12 @@ jobs:
               echo "format_performance=$4"
               echo "rules_performance=$5"
               echo "checks=$6"
+              echo "wheels=$7"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" ]]; then
-            write_scope true true true true true true
+            write_scope true true true true true true true
             exit 0
           fi
 
@@ -54,6 +56,7 @@ jobs:
           performance=false
           format_performance=false
           rules_performance=false
+          wheels=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -69,6 +72,11 @@ jobs:
               *.md) ;;
               *) docs_only=false ;;
             esac
+            case "${path}" in
+              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml)
+                wheels=true
+                ;;
+            esac
             case "${path}" in
               crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
                 broad=true
@@ -123,7 +131,7 @@ jobs:
           done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}")
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true
+            write_scope true true true true true true true
             exit 0
           fi
 
@@ -143,7 +151,8 @@ jobs:
             "${performance}" \
             "${format_performance}" \
             "${rules_performance}" \
-            "${checks}"
+            "${checks}" \
+            "${wheels}"
 
   lockfile:
     name: Lockfile consistency
@@ -414,7 +423,7 @@ jobs:
           - kind: dense
             max_ratio: "1.10"
           - kind: fresh
-            max_ratio: "1.30"
+            max_ratio: "1.10"
     steps:
       - uses: actions/checkout@v4
       - name: Check out the comparison base
@@ -601,6 +610,59 @@ jobs:
           -m "real_warehouse and postgres"
           -vv --color=yes -n auto --dist loadfile
 
+  wheels:
+    # Keep the platform matrix and maturin settings in sync with publish.yml.
+    name: Build wheel (${{ matrix.platform.name }})
+    needs: [changes]
+    if: needs.changes.outputs.wheels == 'true'
+    runs-on: ${{ matrix.platform.runner }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - name: linux-x86_64
+            runner: ubuntu-24.04
+            target: x86_64
+          - name: linux-aarch64
+            runner: ubuntu-24.04-arm
+            target: aarch64
+          - name: macos-x86_64
+            runner: macos-15-intel
+            target: x86_64
+          - name: macos-arm64
+            runner: macos-15
+            target: aarch64
+          - name: windows-x86_64
+            runner: windows-2025
+            target: x86_64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          args: --release --out dist --features extension-module
+          target: ${{ matrix.platform.target }}
+          manylinux: auto
+      - name: Smoke import the native extension from the built wheel
+        shell: bash
+        run: |
+          python -m venv "${RUNNER_TEMP}/wheel-smoke"
+          if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then
+            smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe"
+          else
+            smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python"
+          fi
+          "${smoke_python}" -m pip install --no-deps dist/*.whl
+          cd "${RUNNER_TEMP}"
+          "${smoke_python}" -c "import sqlbuild._native"
+
   verify:
     name: Verify
     if: always()
@@ -618,6 +680,7 @@ jobs:
         cache_compile_performance,
         rules_performance,
         postgres,
+        wheels,
       ]
     runs-on: ubuntu-24.04
     steps:
@@ -636,6 +699,7 @@ jobs:
           CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
+          WHEELS_RESULT: ${{ needs.wheels.result }}
         run: |
           if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then
             echo "Required classification or lockfile checks did not pass" >&2
@@ -655,7 +719,8 @@ jobs:
             "${COMPILE_PERFORMANCE_RATIO_RESULT}" \
             "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
-            "${POSTGRES_RESULT}"; do
+            "${POSTGRES_RESULT}" \
+            "${WHEELS_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
               echo "A selected verification suite did not pass: ${result}" >&2
               exit 1

Cargo.lock

@@ -13,6 +13,8 @@ rust-version = "1.88"
 [workspace.dependencies]
 fensu-policy = "=0.22.0"
 globset = "0.4"
+# Python loads the extension with dlopen, so mimalloc must not use initial-exec TLS.
+mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
 polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad011c49f2c7cd48f4427abb8fc0a6af2e4120", default-features = false, features = [

Cargo.toml

@@ -17,6 +17,7 @@ extension-module = ["pyo3/extension-module"]
 [dependencies]
 fensu-policy.workspace = true
 globset.workspace = true
+mimalloc.workspace = true
 polyglot-sql.workspace = true
 pyo3.workspace = true
 rayon.workspace = true

crates/sqlbuild-rules-native/Cargo.toml

@@ -0,0 +1,4 @@
+//! Process-wide allocator for the many short-lived syntax trees built across analysis threads.
+
+#[global_allocator]
+static GLOBAL_ALLOCATOR: mimalloc::MiMalloc = mimalloc::MiMalloc;

crates/sqlbuild-rules-native/src/bindings/_helpers/allocator.rs

@@ -1,2 +1,3 @@
+mod allocator;
 pub(crate) mod functions;
 pub(crate) mod panics;

crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs

@@ -22,7 +22,7 @@ use crate::compiler::_helpers::sql_tests::markers::{
     replace_dbt_ref_markers, replace_named_markers,
 };
 use crate::compiler::_helpers::sql_tests::rendering::{
-    AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect,
+    AssertionStep, ChainStep, RenderRequest, StatementCache, render_comparison_sql, render_dialect,
     rendered_chain_steps,
 };
 use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE};
@@ -206,6 +206,7 @@ struct ProjectContext {
     analysis_templates: Arc<AnalysisTemplateCache>,
     patterns: SqlTestPatterns,
     render_dialect: Arc<Dialect>,
+    rendered_statements: Arc<StatementCache>,
 }
 
 #[derive(Clone)]
@@ -614,6 +615,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result<String, String>
         analysis_templates: Arc::new(Mutex::new(HashMap::new())),
         patterns: SqlTestPatterns::new()?,
         render_dialect,
+        rendered_statements: Arc::default(),
     };
     let workers = request.workers.clamp(1, MAX_WORKERS);
     let pool = rayon::ThreadPoolBuilder::new()
@@ -631,8 +633,13 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result<String, String>
                 let mut planned = plan_test(test, &context)?;
                 let planning_ns = planning_start.elapsed().as_nanos();
                 let rendering_start = Instant::now();
-                let sql = render_sql
-                    .then(|| render_comparison_sql(&planned.request, &context.render_dialect));
+                let sql = render_sql.then(|| {
+                    render_comparison_sql(
+                        &planned.request,
+                        &context.render_dialect,
+                        Some(&context.rendered_statements),
+                    )
+                });
                 if !include_plan {
                     planned.request.chain.clear();
                     planned.request.assertions.clear();