feat(compiler): add compiler engine switch and differential harness (#987)

* feat(compiler): add compiler engine switch and engine-namespaced caches

Select the compiler engine with SQLBUILD_COMPILER_ENGINE (python by
default, or native) or the hidden --compiler-engine global option. All
frontier stages run through one compile_frontier seam; the native engine
currently falls back to the Python stages. Compile JSON and sqb debug
report the engine, compile reuse keys include it, and the native engine
keeps its compiler, Rules and lineage stores in -native-v1 directories so
the Python stores stay untouched.

* test(compiler): add compiler engine differential harness and corpora

Compare exit codes, normalised stdout and stderr, the compiled tree,
manifest, DAG and per-model query fingerprints between two engines over
the e2e fixtures, website examples, seeded generated projects, a failure
corpus and dense projects, with optional per-stage captures.

* ci: run the compiler engine differential per pull request and nightly

* fix(compiler): defer stage-capture annotations for Python 3.12

itertools.count is not subscriptable at runtime before Python 3.13, so the
ClassVar annotation failed at import on 3.12.

* test(compile): leave the engine label out of semantic compile fingerprints

* test(cli): match the engine choice error across Python versions

Author: kvlonge

.github/workflows/compiler-differential.yml

@@ -0,0 +1,44 @@
+name: Compiler Differential
+
+on:
+  schedule:
+    - cron: "43 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: compiler-differential-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  dense:
+    name: Dense compiler differential (${{ matrix.models }} models)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        models: [3000, 10000]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Compare dense projects with and without custom rules
+        env:
+          MODELS: ${{ matrix.models }}
+        run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
+

.github/workflows/verify.yml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-24.04
     outputs:
       checks: ${{ steps.scope.outputs.checks }}
+      compiler_differential: ${{ steps.scope.outputs.compiler_differential }}
       duckdb: ${{ steps.scope.outputs.duckdb }}
       postgres: ${{ steps.scope.outputs.postgres }}
       performance: ${{ steps.scope.outputs.performance }}
@@ -43,11 +44,12 @@ jobs:
               echo "rules_performance=$5"
               echo "checks=$6"
               echo "wheels=$7"
+              echo "compiler_differential=$8"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" ]]; then
-            write_scope true true true true true true true
+            write_scope true true true true true true true true
             exit 0
           fi
 
@@ -57,6 +59,7 @@ jobs:
           format_performance=false
           rules_performance=false
           wheels=false
+          compiler_differential=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -77,6 +80,11 @@ jobs:
                 wheels=true
                 ;;
             esac
+            case "${path}" in
+              scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|.github/workflows/verify.yml|Makefile)
+                compiler_differential=true
+                ;;
+            esac
             case "${path}" in
               crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
                 broad=true
@@ -132,7 +140,7 @@ jobs:
           done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}")
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true true
+            write_scope true true true true true true true true
             exit 0
           fi
 
@@ -145,6 +153,7 @@ jobs:
             duckdb=true
             postgres=true
             format_performance=true
+            compiler_differential=true
           fi
           write_scope \
             "${duckdb}" \
@@ -153,7 +162,8 @@ jobs:
             "${format_performance}" \
             "${rules_performance}" \
             "${checks}" \
-            "${wheels}"
+            "${wheels}" \
+            "${compiler_differential}"
 
   lockfile:
     name: Lockfile consistency
@@ -258,6 +268,31 @@ jobs:
       - name: Run DuckDB E2E shard
         run: make ${{ matrix.target }}
 
+  compiler_differential:
+    name: Compiler engine differential
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.compiler_differential == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Compare fixtures, examples, generated seeds and the failure corpus
+        run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures
+
   format_performance:
     name: Formatter performance guards
     needs: [changes, lockfile]
@@ -675,6 +710,7 @@ jobs:
         lockfile,
         checks,
         e2e_shards,
+        compiler_differential,
         format_performance,
         performance,
         cold_compile_performance,
@@ -694,6 +730,7 @@ jobs:
           LOCKFILE_RESULT: ${{ needs.lockfile.result }}
           CHECKS_RESULT: ${{ needs.checks.result }}
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
+          COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
@@ -715,6 +752,7 @@ jobs:
           fi
           for result in \
             "${E2E_SHARDS_RESULT}" \
+            "${COMPILER_DIFFERENTIAL_RESULT}" \
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \

Makefile

@@ -6,7 +6,8 @@ SHELL := /bin/bash
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
-	test-e2e-dense-compile-performance test-e2e-varied-cache-performance
+	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
+	compiler-differential compiler-differential-dense
 
 format:
 	uv run ruff format .
@@ -130,6 +131,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \
 E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/scripts/cli_preview \
 	tests/e2e/scripts/compile_performance_ratio \
+	tests/e2e/scripts/compiler_differential \
 	tests/e2e/scripts/release_performance \
 	tests/e2e/src/sqlbuild/cli/commands/main/adapters \
 	tests/e2e/src/sqlbuild/cli/commands/main/audit \
@@ -285,6 +287,21 @@ test-e2e-varied-cache-performance:
 		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
 
 
+COMPILER_DIFFERENTIAL_JOBS ?= 4
+COMPILER_DIFFERENTIAL_SEEDS ?= 12
+COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
+COMPILER_DIFFERENTIAL_ARGS ?=
+
+compiler-differential:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
+		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
+		$(COMPILER_DIFFERENTIAL_ARGS)
+
+compiler-differential-dense:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
+		--corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \
+		--jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS)
+
 
 skills:
 	uv run python -m scripts.generate_docs_skill

crates/sqlbuild-rules/src/engine/_helpers/cache.rs

@@ -16,33 +16,32 @@ pub(crate) struct RuleCacheEntry {
 }
 
 pub(crate) struct Cache {
-    root: PathBuf,
+    native_rules_path: PathBuf,
 }
 
 impl Cache {
-    pub(crate) fn open(project_dir: &Path) -> Result<Self, String> {
-        Ok(Self {
-            root: project_dir.join("target/rules-cache"),
-        })
+    pub(crate) fn open(project_dir: &Path, rules_cache_path: &str) -> Result<Self, String> {
+        let native_rules_path = if rules_cache_path.is_empty() {
+            project_dir.join("target/rules-cache/bulk/native.json")
+        } else {
+            PathBuf::from(rules_cache_path)
+        };
+        Ok(Self { native_rules_path })
     }
 
     pub(crate) fn native_rules_bucket(
         &self,
         ruleset_identity: &str,
     ) -> Result<RuleCacheBucket, String> {
-        read_bucket(&self.root.join("bulk/native.json"), ruleset_identity)
+        read_bucket(&self.native_rules_path, ruleset_identity)
     }
 
     pub(crate) fn put_native_rules_bucket(
         &self,
         ruleset_identity: &str,
         bucket: &RuleCacheBucket,
     ) -> Result<(), String> {
-        write_bucket(
-            &self.root.join("bulk/native.json"),
-            ruleset_identity,
-            bucket,
-        )
+        write_bucket(&self.native_rules_path, ruleset_identity, bucket)
     }
 }
 

crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs

@@ -320,7 +320,7 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result<String, Strin
         .transpose()?
         .unwrap_or_default();
     let cache = cache_enabled
-        .then(|| Cache::open(Path::new(&request.project_dir)))
+        .then(|| Cache::open(Path::new(&request.project_dir), &request.rules_cache_path))
         .transpose()?;
     let built_in_started = Instant::now();
     let mut raw_faults = request.initial_findings.clone();

crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs

@@ -330,6 +330,47 @@ fn given_removed_model_file_when_cache_is_rewritten_then_only_its_entry_is_dropp
     Ok(())
 }
 
+#[test]
+fn given_configured_cache_path_when_evaluating_then_only_that_file_is_used() -> Result<(), String> {
+    let test_cases = [test_types::ConfiguredCachePathTestCase {
+        description: "an engine-owned cache file replaces the default bulk cache",
+        cache_file: "target/rules-cache-native-v1/bulk/native.json",
+        expected_default_cache_exists: false,
+        expected_warm_hits: 3,
+    }];
+    for test_case in &test_cases {
+        let project_dir = TempDir::new().map_err(|error| error.to_string())?;
+        let mut rest = split_request(
+            &project_dir,
+            &json!({"select": ["SQBR"], "cache": {"enabled": true}}),
+        );
+        let cache_path = project_dir.path().join(test_case.cache_file);
+        rest["rules_cache_path"] = json!(cache_path);
+        let models = orders_models();
+
+        let cold = evaluate_split(&rest, &models)?;
+        let warm = evaluate_split(&rest, &models)?;
+
+        assert!(cache_path.is_file(), "{}", test_case.description);
+        assert_eq!(
+            project_dir
+                .path()
+                .join("target/rules-cache/bulk/native.json")
+                .exists(),
+            test_case.expected_default_cache_exists,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            warm["cache_hits"], test_case.expected_warm_hits,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(warm["faults"], cold["faults"], "{}", test_case.description);
+    }
+    Ok(())
+}
+
 #[test]
 fn given_corrupt_model_cache_when_evaluating_then_every_model_reevaluates() -> Result<(), String> {
     let test_cases = [test_types::ModelCacheReuseTestCase {

crates/sqlbuild-rules/src/engine/tests/test_types.rs

@@ -242,6 +242,13 @@ pub(crate) struct ModelCacheReuseTestCase {
     pub(crate) expected_misses: u64,
 }
 
+pub(crate) struct ConfiguredCachePathTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) cache_file: &'static str,
+    pub(crate) expected_default_cache_exists: bool,
+    pub(crate) expected_warm_hits: u64,
+}
+
 pub(crate) struct CachePruningTestCase {
     pub(crate) description: &'static str,
     pub(crate) expected_paths: &'static [&'static str],

crates/sqlbuild-rules/src/models.rs

@@ -729,6 +729,7 @@ pub struct EvaluateRequest {
     pub initial_findings: Vec<Fault>,
     pub defer_suppressions: bool,
     pub custom_rules: Vec<CustomRule>,
+    pub rules_cache_path: String,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -765,6 +766,7 @@ impl Default for EvaluateRequest {
             initial_findings: vec![],
             defer_suppressions: false,
             custom_rules: vec![],
+            rules_cache_path: String::new(),
         }
     }
 }

scripts/cold_compile_performance/main/semantic_compile_fingerprint.py

@@ -14,6 +14,7 @@ def semantic_compile_fingerprint(*, payload: dict[str, object], compiled_dir: Pa
 
     semantic_payload: dict[str, object] = dict(payload)
     semantic_payload.pop("compile_timings", None)
+    semantic_payload.pop("compiler_engine", None)
     semantic_payload.pop("version", None)
     fingerprint: Any = hashlib.sha256(orjson.dumps(semantic_payload, option=orjson.OPT_SORT_KEYS))
     for artifact_path in sorted(path for path in compiled_dir.rglob("*") if path.is_file()):

scripts/compiler_differential/__init__.py

@@ -0,0 +1 @@
+"""Differential harness comparing the Python and native compiler engines."""