perf(compile): close the fresh compile gap with ordered native scheduling and mimalloc

.github/workflows/publish.yml

@@ -34,6 +34,7 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
   wheels:
+    # Keep the platform matrix and maturin settings in sync with verify.yml.
     name: Build wheel (${{ matrix.platform.name }})
     needs: release-tag
     runs-on: ${{ matrix.platform.runner }}

.github/workflows/verify.yml

@@ -22,6 +22,7 @@ jobs:
       performance: ${{ steps.scope.outputs.performance }}
       format_performance: ${{ steps.scope.outputs.format_performance }}
       rules_performance: ${{ steps.scope.outputs.rules_performance }}
+      wheels: ${{ steps.scope.outputs.wheels }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -41,11 +42,12 @@ jobs:
               echo "format_performance=$4"
               echo "rules_performance=$5"
               echo "checks=$6"
+              echo "wheels=$7"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" ]]; then
-            write_scope true true true true true true
+            write_scope true true true true true true true
             exit 0
           fi
 
@@ -54,6 +56,7 @@ jobs:
           performance=false
           format_performance=false
           rules_performance=false
+          wheels=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -69,6 +72,11 @@ jobs:
               *.md) ;;
               *) docs_only=false ;;
             esac
+            case "${path}" in
+              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml)
+                wheels=true
+                ;;
+            esac
             case "${path}" in
               crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
                 broad=true
@@ -123,7 +131,7 @@ jobs:
           done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}")
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true
+            write_scope true true true true true true true
             exit 0
           fi
 
@@ -143,7 +151,8 @@ jobs:
             "${performance}" \
             "${format_performance}" \
             "${rules_performance}" \
-            "${checks}"
+            "${checks}" \
+            "${wheels}"
 
   lockfile:
     name: Lockfile consistency
@@ -414,7 +423,7 @@ jobs:
           - kind: dense
             max_ratio: "1.10"
           - kind: fresh
-            max_ratio: "1.30"
+            max_ratio: "1.10"
     steps:
       - uses: actions/checkout@v4
       - name: Check out the comparison base
@@ -601,6 +610,59 @@ jobs:
           -m "real_warehouse and postgres"
           -vv --color=yes -n auto --dist loadfile
 
+  wheels:
+    # Keep the platform matrix and maturin settings in sync with publish.yml.
+    name: Build wheel (${{ matrix.platform.name }})
+    needs: [changes]
+    if: needs.changes.outputs.wheels == 'true'
+    runs-on: ${{ matrix.platform.runner }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - name: linux-x86_64
+            runner: ubuntu-24.04
+            target: x86_64
+          - name: linux-aarch64
+            runner: ubuntu-24.04-arm
+            target: aarch64
+          - name: macos-x86_64
+            runner: macos-15-intel
+            target: x86_64
+          - name: macos-arm64
+            runner: macos-15
+            target: aarch64
+          - name: windows-x86_64
+            runner: windows-2025
+            target: x86_64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          args: --release --out dist --features extension-module
+          target: ${{ matrix.platform.target }}
+          manylinux: auto
+      - name: Smoke import the native extension from the built wheel
+        shell: bash
+        run: |
+          python -m venv "${RUNNER_TEMP}/wheel-smoke"
+          if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then
+            smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe"
+          else
+            smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python"
+          fi
+          "${smoke_python}" -m pip install --no-deps dist/*.whl
+          cd "${RUNNER_TEMP}"
+          "${smoke_python}" -c "import sqlbuild._native"
+
   verify:
     name: Verify
     if: always()
@@ -618,6 +680,7 @@ jobs:
         cache_compile_performance,
         rules_performance,
         postgres,
+        wheels,
       ]
     runs-on: ubuntu-24.04
     steps:
@@ -636,6 +699,7 @@ jobs:
           CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
+          WHEELS_RESULT: ${{ needs.wheels.result }}
         run: |
           if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then
             echo "Required classification or lockfile checks did not pass" >&2
@@ -655,7 +719,8 @@ jobs:
             "${COMPILE_PERFORMANCE_RATIO_RESULT}" \
             "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
-            "${POSTGRES_RESULT}"; do
+            "${POSTGRES_RESULT}" \
+            "${WHEELS_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
               echo "A selected verification suite did not pass: ${result}" >&2
               exit 1

Cargo.toml

@@ -13,6 +13,8 @@ rust-version = "1.88"
 [workspace.dependencies]
 fensu-policy = "=0.22.0"
 globset = "0.4"
+# Python loads the extension with dlopen, so mimalloc must not use initial-exec TLS.
+mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
 polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad011c49f2c7cd48f4427abb8fc0a6af2e4120", default-features = false, features = [

crates/sqlbuild-rules-native/Cargo.toml

@@ -17,6 +17,7 @@ extension-module = ["pyo3/extension-module"]
 [dependencies]
 fensu-policy.workspace = true
 globset.workspace = true
+mimalloc.workspace = true
 polyglot-sql.workspace = true
 pyo3.workspace = true
 rayon.workspace = true

crates/sqlbuild-rules-native/src/bindings/_helpers/allocator.rs

@@ -0,0 +1,4 @@
+//! Process-wide allocator for the many short-lived syntax trees built across analysis threads.
+
+#[global_allocator]
+static GLOBAL_ALLOCATOR: mimalloc::MiMalloc = mimalloc::MiMalloc;

crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs

@@ -1,2 +1,3 @@
+mod allocator;
 pub(crate) mod functions;
 pub(crate) mod panics;

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs

@@ -22,7 +22,7 @@ use crate::compiler::_helpers::sql_tests::markers::{
     replace_dbt_ref_markers, replace_named_markers,
 };
 use crate::compiler::_helpers::sql_tests::rendering::{
-    AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect,
+    AssertionStep, ChainStep, RenderRequest, StatementCache, render_comparison_sql, render_dialect,
     rendered_chain_steps,
 };
 use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE};
@@ -206,6 +206,7 @@ struct ProjectContext {
     analysis_templates: Arc<AnalysisTemplateCache>,
     patterns: SqlTestPatterns,
     render_dialect: Arc<Dialect>,
+    rendered_statements: Arc<StatementCache>,
 }
 
 #[derive(Clone)]
@@ -614,6 +615,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result<String, String>
         analysis_templates: Arc::new(Mutex::new(HashMap::new())),
         patterns: SqlTestPatterns::new()?,
         render_dialect,
+        rendered_statements: Arc::default(),
     };
     let workers = request.workers.clamp(1, MAX_WORKERS);
     let pool = rayon::ThreadPoolBuilder::new()
@@ -631,8 +633,13 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result<String, String>
                 let mut planned = plan_test(test, &context)?;
                 let planning_ns = planning_start.elapsed().as_nanos();
                 let rendering_start = Instant::now();
-                let sql = render_sql
-                    .then(|| render_comparison_sql(&planned.request, &context.render_dialect));
+                let sql = render_sql.then(|| {
+                    render_comparison_sql(
+                        &planned.request,
+                        &context.render_dialect,
+                        Some(&context.rendered_statements),
+                    )
+                });
                 if !include_plan {
                     planned.request.chain.clear();
                     planned.request.assertions.clear();