feat(compiler): add compiler engine switch and differential harness

.github/workflows/compiler-differential.yml

@@ -0,0 +1,44 @@
+name: Compiler Differential
+
+on:
+  schedule:
+    - cron: "43 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: compiler-differential-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  dense:
+    name: Dense compiler differential (${{ matrix.models }} models)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        models: [3000, 10000]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Compare dense projects with and without custom rules
+        env:
+          MODELS: ${{ matrix.models }}
+        run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
+

.github/workflows/verify.yml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-24.04
     outputs:
       checks: ${{ steps.scope.outputs.checks }}
+      compiler_differential: ${{ steps.scope.outputs.compiler_differential }}
       duckdb: ${{ steps.scope.outputs.duckdb }}
       postgres: ${{ steps.scope.outputs.postgres }}
       performance: ${{ steps.scope.outputs.performance }}
@@ -43,11 +44,12 @@ jobs:
               echo "rules_performance=$5"
               echo "checks=$6"
               echo "wheels=$7"
+              echo "compiler_differential=$8"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" ]]; then
-            write_scope true true true true true true true
+            write_scope true true true true true true true true
             exit 0
           fi
 
@@ -57,6 +59,7 @@ jobs:
           format_performance=false
           rules_performance=false
           wheels=false
+          compiler_differential=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -77,6 +80,11 @@ jobs:
                 wheels=true
                 ;;
             esac
+            case "${path}" in
+              scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|.github/workflows/verify.yml|Makefile)
+                compiler_differential=true
+                ;;
+            esac
             case "${path}" in
               crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
                 broad=true
@@ -132,7 +140,7 @@ jobs:
           done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}")
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true true
+            write_scope true true true true true true true true
             exit 0
           fi
 
@@ -145,6 +153,7 @@ jobs:
             duckdb=true
             postgres=true
             format_performance=true
+            compiler_differential=true
           fi
           write_scope \
             "${duckdb}" \
@@ -153,7 +162,8 @@ jobs:
             "${format_performance}" \
             "${rules_performance}" \
             "${checks}" \
-            "${wheels}"
+            "${wheels}" \
+            "${compiler_differential}"
 
   lockfile:
     name: Lockfile consistency
@@ -258,6 +268,31 @@ jobs:
       - name: Run DuckDB E2E shard
         run: make ${{ matrix.target }}
 
+  compiler_differential:
+    name: Compiler engine differential
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.compiler_differential == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Compare fixtures, examples, generated seeds and the failure corpus
+        run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures
+
   format_performance:
     name: Formatter performance guards
     needs: [changes, lockfile]
@@ -675,6 +710,7 @@ jobs:
         lockfile,
         checks,
         e2e_shards,
+        compiler_differential,
         format_performance,
         performance,
         cold_compile_performance,
@@ -694,6 +730,7 @@ jobs:
           LOCKFILE_RESULT: ${{ needs.lockfile.result }}
           CHECKS_RESULT: ${{ needs.checks.result }}
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
+          COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
@@ -715,6 +752,7 @@ jobs:
           fi
           for result in \
             "${E2E_SHARDS_RESULT}" \
+            "${COMPILER_DIFFERENTIAL_RESULT}" \
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \

Makefile

@@ -6,7 +6,8 @@ SHELL := /bin/bash
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
-	test-e2e-dense-compile-performance test-e2e-varied-cache-performance
+	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
+	compiler-differential compiler-differential-dense
 
 format:
 	uv run ruff format .
@@ -130,6 +131,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \
 E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/scripts/cli_preview \
 	tests/e2e/scripts/compile_performance_ratio \
+	tests/e2e/scripts/compiler_differential \
 	tests/e2e/scripts/release_performance \
 	tests/e2e/src/sqlbuild/cli/commands/main/adapters \
 	tests/e2e/src/sqlbuild/cli/commands/main/audit \
@@ -285,6 +287,21 @@ test-e2e-varied-cache-performance:
 		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
 
 
+COMPILER_DIFFERENTIAL_JOBS ?= 4
+COMPILER_DIFFERENTIAL_SEEDS ?= 12
+COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
+COMPILER_DIFFERENTIAL_ARGS ?=
+
+compiler-differential:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
+		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
+		$(COMPILER_DIFFERENTIAL_ARGS)
+
+compiler-differential-dense:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
+		--corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \
+		--jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS)
+
 
 skills:
 	uv run python -m scripts.generate_docs_skill

crates/sqlbuild-rules/src/engine/_helpers/cache.rs

@@ -16,33 +16,32 @@ pub(crate) struct RuleCacheEntry {
 }
 
 pub(crate) struct Cache {
-    root: PathBuf,
+    native_rules_path: PathBuf,
 }
 
 impl Cache {
-    pub(crate) fn open(project_dir: &Path) -> Result<Self, String> {
-        Ok(Self {
-            root: project_dir.join("target/rules-cache"),
-        })
+    pub(crate) fn open(project_dir: &Path, rules_cache_path: &str) -> Result<Self, String> {
+        let native_rules_path = if rules_cache_path.is_empty() {
+            project_dir.join("target/rules-cache/bulk/native.json")
+        } else {
+            PathBuf::from(rules_cache_path)
+        };
+        Ok(Self { native_rules_path })
     }
 
     pub(crate) fn native_rules_bucket(
         &self,
         ruleset_identity: &str,
     ) -> Result<RuleCacheBucket, String> {
-        read_bucket(&self.root.join("bulk/native.json"), ruleset_identity)
+        read_bucket(&self.native_rules_path, ruleset_identity)
     }
 
     pub(crate) fn put_native_rules_bucket(
         &self,
         ruleset_identity: &str,
         bucket: &RuleCacheBucket,
     ) -> Result<(), String> {
-        write_bucket(
-            &self.root.join("bulk/native.json"),
-            ruleset_identity,
-            bucket,
-        )
+        write_bucket(&self.native_rules_path, ruleset_identity, bucket)
     }
 }
 

crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs

@@ -320,7 +320,7 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result<String, Strin
         .transpose()?
         .unwrap_or_default();
     let cache = cache_enabled
-        .then(|| Cache::open(Path::new(&request.project_dir)))
+        .then(|| Cache::open(Path::new(&request.project_dir), &request.rules_cache_path))
         .transpose()?;
     let built_in_started = Instant::now();
     let mut raw_faults = request.initial_findings.clone();

crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs

@@ -330,6 +330,47 @@ fn given_removed_model_file_when_cache_is_rewritten_then_only_its_entry_is_dropp
     Ok(())
 }
 
+#[test]
+fn given_configured_cache_path_when_evaluating_then_only_that_file_is_used() -> Result<(), String> {
+    let test_cases = [test_types::ConfiguredCachePathTestCase {
+        description: "an engine-owned cache file replaces the default bulk cache",
+        cache_file: "target/rules-cache-native-v1/bulk/native.json",
+        expected_default_cache_exists: false,
+        expected_warm_hits: 3,
+    }];
+    for test_case in &test_cases {
+        let project_dir = TempDir::new().map_err(|error| error.to_string())?;
+        let mut rest = split_request(
+            &project_dir,
+            &json!({"select": ["SQBR"], "cache": {"enabled": true}}),
+        );
+        let cache_path = project_dir.path().join(test_case.cache_file);
+        rest["rules_cache_path"] = json!(cache_path);
+        let models = orders_models();
+
+        let cold = evaluate_split(&rest, &models)?;
+        let warm = evaluate_split(&rest, &models)?;
+
+        assert!(cache_path.is_file(), "{}", test_case.description);
+        assert_eq!(
+            project_dir
+                .path()
+                .join("target/rules-cache/bulk/native.json")
+                .exists(),
+            test_case.expected_default_cache_exists,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            warm["cache_hits"], test_case.expected_warm_hits,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(warm["faults"], cold["faults"], "{}", test_case.description);
+    }
+    Ok(())
+}
+
 #[test]
 fn given_corrupt_model_cache_when_evaluating_then_every_model_reevaluates() -> Result<(), String> {
     let test_cases = [test_types::ModelCacheReuseTestCase {

crates/sqlbuild-rules/src/engine/tests/test_types.rs

@@ -242,6 +242,13 @@ pub(crate) struct ModelCacheReuseTestCase {
     pub(crate) expected_misses: u64,
 }
 
+pub(crate) struct ConfiguredCachePathTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) cache_file: &'static str,
+    pub(crate) expected_default_cache_exists: bool,
+    pub(crate) expected_warm_hits: u64,
+}
+
 pub(crate) struct CachePruningTestCase {
     pub(crate) description: &'static str,
     pub(crate) expected_paths: &'static [&'static str],

crates/sqlbuild-rules/src/models.rs

@@ -729,6 +729,7 @@ pub struct EvaluateRequest {
     pub initial_findings: Vec<Fault>,
     pub defer_suppressions: bool,
     pub custom_rules: Vec<CustomRule>,
+    pub rules_cache_path: String,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -765,6 +766,7 @@ impl Default for EvaluateRequest {
             initial_findings: vec![],
             defer_suppressions: false,
             custom_rules: vec![],
+            rules_cache_path: String::new(),
         }
     }
 }

scripts/cold_compile_performance/main/semantic_compile_fingerprint.py

@@ -14,6 +14,7 @@ def semantic_compile_fingerprint(*, payload: dict[str, object], compiled_dir: Pa
 
     semantic_payload: dict[str, object] = dict(payload)
     semantic_payload.pop("compile_timings", None)
+    semantic_payload.pop("compiler_engine", None)
     semantic_payload.pop("version", None)
     fingerprint: Any = hashlib.sha256(orjson.dumps(semantic_payload, option=orjson.OPT_SORT_KEYS))
     for artifact_path in sorted(path for path in compiled_dir.rglob("*") if path.is_file()):

scripts/compiler_differential/__init__.py

@@ -0,0 +1 @@
+"""Differential harness comparing the Python and native compiler engines."""