perf(compiler): bound cold analysis and tighten regression budgets

.github/workflows/verify.yml

@@ -73,6 +73,7 @@ jobs:
                 ;;
               src/sqlbuild/rule_engine/*|src/sqlbuild/lint/*|scripts/rules_benchmark/*|scripts/benchmark_rules_ci.py|tests/performance/*)
                 broad=true
+                performance=true
                 format_performance=true
                 rules_performance=true
                 ;;
@@ -324,6 +325,53 @@ jobs:
             make test-e2e-cold-compile-performance \
               SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
 
+  dense_compile_performance:
+    name: Dense all-rules compile (${{ matrix.models }} models)
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.performance == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        models: [1000, 3000, 5000, 10000]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Run dense all-rules guard under 4 GiB
+        env:
+          BENCHMARK_MODELS: ${{ matrix.models }}
+          MEMORY_LIMIT_BYTES: "4294967296"
+        run: |
+          sudo systemd-run \
+            --unit="sqlbuild-dense-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
+            --uid="$(id -u)" \
+            --gid="$(id -g)" \
+            --same-dir \
+            --setenv="HOME=${HOME}" \
+            --setenv="PATH=${PATH}" \
+            --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
+            --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
+            --property="MemorySwapMax=0" \
+            --wait \
+            --collect \
+            --pipe \
+            make test-e2e-dense-compile-performance \
+              SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
+
   cache_compile_performance:
     name: Fresh-process compile cache (${{ matrix.models }} models)
     needs: [changes, lockfile]
@@ -464,6 +512,7 @@ jobs:
         format_performance,
         performance,
         cold_compile_performance,
+        dense_compile_performance,
         cache_compile_performance,
         rules_performance,
         postgres,
@@ -479,6 +528,7 @@ jobs:
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
+          DENSE_COMPILE_PERFORMANCE_RESULT: ${{ needs.dense_compile_performance.result }}
           CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
@@ -494,6 +544,7 @@ jobs:
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \
+            "${DENSE_COMPILE_PERFORMANCE_RESULT}" \
             "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}"; do

Cargo.toml

@@ -15,7 +15,7 @@ fensu-policy = "=0.22.0"
 globset = "0.4"
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "190b1f8265b6135a9a6a100356599dd99487c1b4", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b", default-features = false, features = [
     "ast-tools",
     "builder",
     "dialect-bigquery",

Makefile

@@ -5,7 +5,8 @@ SHELL := /bin/bash
 	test-e2e-duckdb-build-incremental test-e2e-duckdb-build-virtual \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \
 	test-e2e-duckdb-integrations test-e2e-performance \
-	test-e2e-cold-compile-performance test-e2e-cache-compile-performance
+	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
+	test-e2e-dense-compile-performance
 
 format:
 	uv run ruff format .
@@ -238,6 +239,14 @@ test-e2e-cold-compile-performance:
 		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
 		-vv --log-cli-level=INFO --color=yes
 
+test-e2e-dense-compile-performance:
+	test -n "$(SQLBUILD_BENCHMARK_MODELS)"
+	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
+		tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py \
+		-n auto --dist loadfile \
+		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \
+		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
+
 test-e2e-cache-compile-performance:
 	test -n "$(SQLBUILD_BENCHMARK_MODELS)"
 	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \

crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs

@@ -21,8 +21,9 @@ fn value_error(error: impl std::fmt::Display) -> PyErr {
 }
 
 #[pyfunction]
-fn evaluate_json(request_json: &str) -> PyResult<String> {
-    evaluate::evaluate_json(request_json).map_err(value_error)
+fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
+    py.detach(|| evaluate::evaluate_json(request_json))
+        .map_err(value_error)
 }
 
 #[pyfunction]
@@ -31,6 +32,31 @@ fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
         .map_err(value_error)
 }
 
+#[pyfunction]
+fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
+    py.detach(|| crate::engine::main::finalize::finalize_findings_json(request_json))
+        .map_err(value_error)
+}
+
+#[pyfunction]
+fn prepare_lint_sql(
+    py: Python<'_>,
+    expanded: &str,
+    before_expansion: &str,
+    prior_sites: Vec<usize>,
+) -> PyResult<Option<crate::sql_lint::types::PreparedSql>> {
+    py.detach(|| {
+        crate::sql_lint::main::preparation::prepare(expanded, before_expansion, &prior_sites)
+    })
+    .map_err(value_error)
+}
+
+#[pyfunction]
+fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
+    py.detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json))
+        .map_err(pyo3::exceptions::PyValueError::new_err)
+}
+
 #[pyfunction]
 fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
     py.detach(|| crate::sql_lint::main::formatter::format_json(request_json))
@@ -63,20 +89,24 @@ fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult<String>
 
 #[pyfunction]
 fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
-    py.detach(|| crate::query_analysis::main::analyze_json(request_json))
+    py.detach(|| crate::query_analysis::main::analyze::analyze_json(request_json))
         .map_err(value_error)
 }
 
 #[pyfunction]
 fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
-    py.detach(|| crate::query_analysis::main::analyze_project_json(request_json))
+    py.detach(|| crate::query_analysis::main::analyze_project::analyze_project_json(request_json))
         .map_err(value_error)
 }
 
 #[pyfunction]
 fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
-    py.detach(|| crate::query_analysis::main::analyze_project_compact_json(request_json))
-        .map_err(value_error)
+    py.detach(|| {
+        crate::query_analysis::main::analyze_project_compact::analyze_project_compact_json(
+            request_json,
+        )
+    })
+    .map_err(value_error)
 }
 
 #[pyfunction]
@@ -215,6 +245,21 @@ fn tokenize_model_header(header: &str) -> PyResult<Vec<(u8, String, usize)>> {
     crate::compiler::main::model_header_tokenizing::tokenize_one(header).map_err(value_error)
 }
 
+#[pyfunction]
+fn substitute_static_project_vars(
+    sqls: Vec<String>,
+    variables: Vec<(String, String)>,
+) -> Vec<(u8, Option<String>)> {
+    crate::compiler::main::sql_interpolation::substitute_batch(&sqls, &variables)
+}
+
+#[pyfunction]
+fn extract_static_sql_references(
+    sql: &str,
+) -> Option<Vec<crate::compiler::_helpers::sql_references::extraction::StaticReference>> {
+    crate::compiler::main::sql_references::extract(sql)
+}
+
 #[pyfunction]
 fn load_config_json(project_dir: &str) -> PyResult<String> {
     load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error)
@@ -259,7 +304,10 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String {
 
 pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(evaluate_json, module)?)?;
+    module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?;
     module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?;
+    module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?;
+    module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?;
     module.add_function(wrap_pyfunction!(format_sql_json, module)?)?;
     module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?;
     module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?;
@@ -276,6 +324,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?;
     module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?;
     module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?;
+    module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?;
+    module.add_function(wrap_pyfunction!(extract_static_sql_references, module)?)?;
     module.add_function(wrap_pyfunction!(load_config_json, module)?)?;
     module.add_function(wrap_pyfunction!(catalogue_json, module)?)?;
     module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?;

crates/sqlbuild-rules-native/src/compiler/_helpers/mod.rs

@@ -1,2 +1,4 @@
 pub(crate) mod model_headers;
+pub(crate) mod sql_interpolation;
+pub(crate) mod sql_references;
 pub(crate) mod sql_tests;

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod substitution;