refactor(compiler): remove the Python engine and remaining M4 algorithms

.github/workflows/compiler-differential.yml

@@ -37,14 +37,8 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
-      - name: Compare dense projects with and without custom rules on the preview engine
+      - name: Compare dense projects with and without custom rules (native vs native-preview)
         env:
           MODELS: ${{ matrix.models }}
         run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
-      - name: Compare dense projects with and without custom rules on the shipped engine
-        env:
-          MODELS: ${{ matrix.models }}
-        run: >-
-          make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
-          COMPILER_DIFFERENTIAL_ARGS="--engines python native"
 

.github/workflows/verify.yml

@@ -303,7 +303,7 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
-      - name: Compare fixtures, examples, generated seeds and the failure corpus (python vs native-preview)
+      - name: Compare fixtures, examples, generated seeds and the failure corpus (native vs native-preview, goldens)
         run: >-
           make compiler-differential
           COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
@@ -318,12 +318,6 @@ jobs:
           path: ${{ runner.temp }}/compiler-differential-wheel-sites.json
           if-no-files-found: warn
           retention-days: 30
-      - name: Compare fixtures, examples, generated seeds and the failure corpus (python vs shipped native)
-        run: >-
-          make compiler-differential-shipped
-          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
-          --require-render-coverage --require-analysis-coverage
-          --evidence-dir ${{ runner.temp }}/compiler-differential-evidence/shipped"
       - name: Upload differing projects' targets, stderr and compiled-test stats
         if: failure()
         uses: actions/upload-artifact@v4
@@ -370,19 +364,13 @@ jobs:
           uv run --no-sync pytest
           tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py
           -n auto --dist loadfile
-      - name: Compare python with native-preview on fixtures, examples, the failure corpus and a few seeds
+      - name: Compare native with native-preview and the goldens on fixtures, examples, the failure corpus and a few seeds
         env:
           PYTHONUNBUFFERED: "1"
         run: >-
           uv run --no-sync python -m scripts.run_compiler_differential
-          --engines python native-preview
+          --engines native native-preview --goldens check
           --corpus fixtures examples failures seeds --seeds 4 --jobs 4
-      - name: Compare python with shipped native on the failure corpus and a few seeds
-        env:
-          PYTHONUNBUFFERED: "1"
-        run: >-
-          uv run --no-sync python -m scripts.run_compiler_differential
-          --engines python native --corpus failures seeds --seeds 4 --jobs 4
 
   native_discovery_windows:
     name: Native compiler on Windows
@@ -420,14 +408,14 @@ jobs:
           tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py
           tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py
           -n auto --dist loadfile
-      - name: Compile playground projects on the default and Python engines, and build
+      - name: Compile playground projects on the default and preview engines, and build
         run: |
           project="${RUNNER_TEMP}/waffle-shop"
-          oracle="${RUNNER_TEMP}/waffle-shop-python"
+          oracle="${RUNNER_TEMP}/waffle-shop-preview"
           uv run --no-sync sqb playground "${project}"
           cp -r "${project}" "${oracle}"
           uv run --no-sync sqb --project-dir "${project}" --no-color compile
-          SQLBUILD_COMPILER_ENGINE=python uv run --no-sync sqb --project-dir "${oracle}" --no-color compile
+          SQLBUILD_COMPILER_ENGINE=native-preview uv run --no-sync sqb --project-dir "${oracle}" --no-color compile
           diff -r "${oracle}/target/compiled" "${project}/target/compiled"
           if ! test -f "${project}/target/cache/compiler-native-v1/macro-calls.bin"; then
             echo "The default compile did not write the macro-call store; macro bridge debug log:"
@@ -611,24 +599,18 @@ jobs:
           - kind: fresh
             max_ratio: "1.10"
           - kind: dense
-            label: native contracts vs python
-            base_engine: python
-            head_engine: native
+            label: contracts CPU vs base
             gate_phase: contracts_cpu_ms
-            max_ratio: "1.00"
+            max_ratio: "1.10"
           - kind: dense
-            label: native model analysis vs python
-            base_engine: python
-            head_engine: native
+            label: model analysis CPU vs base
             gate_phase: model_analysis_cpu_ms
             compile_args: --compile-arg=--select=*
-            max_ratio: "1.50"
+            max_ratio: "1.10"
           - kind: dense
-            label: native custom rules vs python
-            base_engine: python
-            head_engine: native
+            label: custom rules CPU vs base
             gate_phase: custom_rules_cpu_ms
-            max_ratio: "1.25"
+            max_ratio: "1.10"
     steps:
       - uses: actions/checkout@v4
       - name: Check out the comparison base
@@ -655,13 +637,11 @@ jobs:
       - name: Install base dependencies
         working-directory: base
         run: uv sync --locked --all-groups --all-extras
-      - name: Compare python and native phase CPU on head
+      - name: Compare base and head phase CPU
         if: matrix.gate_phase
         env:
           KIND: ${{ matrix.kind }}
           MAX_RATIO: ${{ matrix.max_ratio }}
-          BASE_ENGINE: ${{ matrix.base_engine }}
-          HEAD_ENGINE: ${{ matrix.head_engine }}
           GATE_PHASE: ${{ matrix.gate_phase }}
           COMPILE_ARGS: ${{ matrix.compile_args }}
         run: |
@@ -670,10 +650,9 @@ jobs:
             --modes cold \
             --max-ratio "${MAX_RATIO}" \
             --models 3000 \
-            --base-python .venv/bin/python \
+            --base-root base \
+            --base-python base/.venv/bin/python \
             --head-python .venv/bin/python \
-            --base-engine "${BASE_ENGINE}" \
-            --head-engine "${HEAD_ENGINE}" \
             --gate-phase "${GATE_PHASE}" \
             ${COMPILE_ARGS:+"${COMPILE_ARGS}"}
       - name: Compare base and head cold, warm and one-model edit compiles

Cargo.toml

@@ -55,12 +55,7 @@ pyo3 = { version = "0.26", features = ["abi3-py312"] }
 polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "ca9e7b6e405fb6f5484472bb08780268789c5e04", default-features = false, features = [
     "ast-tools",
     "builder",
-    "dialect-bigquery",
-    "dialect-databricks",
-    "dialect-duckdb",
-    "dialect-postgresql",
-    "dialect-snowflake",
-    "dialect-tsql",
+    "all-dialects",
     "semantic",
     "stacker",
 ] }

Makefile

@@ -7,7 +7,7 @@ SHELL := /bin/bash
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
-	compiler-differential compiler-differential-shipped compiler-differential-dense \
+	compiler-differential compiler-differential-dense \
 	compiler-baselines
 
 format:
@@ -291,7 +291,6 @@ test-e2e-varied-cache-performance:
 
 COMPILER_DIFFERENTIAL_JOBS ?= 4
 COMPILER_DIFFERENTIAL_SEEDS ?= 12
-COMPILER_DIFFERENTIAL_SHIPPED_SEEDS ?= 12
 COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
 COMPILER_DIFFERENTIAL_ARGS ?=
 # Native-to-Python fallbacks and analysis deferrals must match the allow-list exactly, and every
@@ -303,23 +302,13 @@ compiler-differential:
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
 		$(COMPILER_DIFFERENTIAL_GATES) $(COMPILER_DIFFERENTIAL_ARGS)
 
-# python vs native-preview above covers every native stage; this keeps the shipped `native` default
-# (native discovery, rendering and analysis) and its goldens covered on the same corpus.
-compiler-differential-shipped:
-	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
-		--engines python native --corpus fixtures examples seeds failures \
-		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SHIPPED_SEEDS) \
-		$(COMPILER_DIFFERENTIAL_GATES) $(COMPILER_DIFFERENTIAL_ARGS)
-
-# Rewrite the goldens (from the python oracle) and the fallback allow-list from both CI runs.
+# Rewrite the goldens (from the native oracle) and the fallback allow-list from the CI run.
 # Review the diff like code; see scripts/compiler_differential/README.md.
 COMPILER_DIFFERENTIAL_CI_ARGS := --stage-captures --require-discovery-coverage --require-render-coverage
 compiler-baselines:
 	rm -rf tests/goldens/compiler
 	$(MAKE) compiler-differential COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
 		--require-analysis-coverage --native-fallbacks update --goldens update"
-	$(MAKE) compiler-differential-shipped COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
-		--require-analysis-coverage --native-fallbacks update --goldens update"
 
 compiler-differential-dense:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \

crates/README.md

@@ -58,32 +58,29 @@ stages run natively:
 
 | Engine | Runs |
 |---|---|
-| `python` | The Python compiler only. It is the oracle every native stage must match byte for byte. |
-| `native` | The default: native stages that passed their flip gate (`shipped` tier). |
+| `native` | The default: discovery, rendering and analysis, and every native stage that passed its flip gate (`shipped` tier). |
 | `native-preview` | Opt-in: shipped stages plus stages still in development (`preview` tier). |
 
-The shipped tier covers discovery, rendering and analysis: declaration files and scopes, model
-config, reference extraction, the model loop, macro calls and the macro-call store, attachments,
-the type system, model analysis and its `model-analyses.bin` cache, semantic checks, contracts,
-lineage facts, SQL-test glue and project assembly. A failing
-render raises its first error directly, as the Python stage would, and runs each macro call at
-most once; a divergence the bridge detects is reported as a native stage mismatch. Because the
-macro-call store is shipped, the
+The Python compiler and its `python` engine were removed once discovery, rendering and analysis
+shipped natively; setting `python` is an error that says so. Native stages still hand inputs they
+cannot reproduce exactly back to Python helpers (the allow-list in
+`scripts/compiler_differential/native_fallbacks.toml` lists every such deferral the corpus
+reaches). A failing render raises its first error directly and runs each macro call at most
+once; a divergence the bridge detects is reported as a native stage mismatch. Because the
+macro-call store is always on, the
 [macro determinism contract](../website/src/content/docs/docs/concepts/macros.mdx) applies to
-every default compile; `SQLBUILD_COMPILER_ENGINE=python` runs every macro call each time.
+every compile.
 
 Each native stage declares its tier once, in `NATIVE_STAGE_TIERS` in
 `src/sqlbuild/compiler/frontier/constants.py`. A stage moves from `preview` to `shipped` by
 changing that line, after its flip gate passes: a byte-identical real project, a green
 differential, and no measured slowdown on cold, edit and no-change compiles. Every engine keeps
 its own compiler, Rules and compile-reuse stores, so preview output is never reused by `native`.
 
-`make compiler-differential` compares `python` with `native-preview` on the full per-PR corpus.
-`make compiler-differential-shipped` compares `python` with `native` on the generated seeds and
-the failure corpus, so the shipped default stays covered on its own; CI runs it with stage
-captures and requires full discovery and render coverage. CI runs both, compares `python` with
-`native` and `native-preview` on Python 3.13 and 3.14 as well, and on Windows checks that the
-default compiles a playground project to the same files as `python`.
+`make compiler-differential` compares `native` with `native-preview` on the full per-PR corpus
+and checks both against the goldens in `tests/goldens/compiler`, with stage captures and full
+discovery, render and analysis coverage required. CI also runs it on Python 3.13 and 3.14, and
+on Windows checks that both engines compile a playground project to the same files.
 
 ## Working on the crates
 

crates/sqlbuild-analysis/Cargo.toml

@@ -23,6 +23,7 @@ sha2.workspace = true
 sqlbuild-cache.workspace = true
 sqlbuild-core.workspace = true
 sqlbuild-model-config.workspace = true
+sqlbuild-render.workspace = true
 sqlbuild-scopes.workspace = true
 sqlbuild-sqltext.workspace = true
 

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs

@@ -33,7 +33,6 @@ impl AnalysisSession {
             session_digest: session_digest(&self.request),
             keys: vec![None; models],
             hits: vec![false; models],
-            uncacheable: vec![false; models],
             legacy_tables: vec![None; models],
             stats: AnalysisCacheStats::default(),
         });
@@ -100,6 +99,7 @@ impl AnalysisSession {
             }
         }
         key.flag(self.dependency_ordered);
+        key.flag(self.cyclic);
         key.finish()
     }
 
@@ -411,12 +411,11 @@ impl<'a> Decoder<'a> {
 pub(crate) fn encode_outcome(
     outcome: &ModelOutcome,
     legacy_tables: Option<&ContentDigest>,
-) -> Option<Vec<u8>> {
+) -> Vec<u8> {
     let analysis: &ModelAnalysis = &outcome.analysis;
     let (tag, rows) = match &analysis.lineage {
         LineageFacts::Native(rows) => (LINEAGE_NATIVE, rows),
         LineageFacts::NativeFacts(rows) => (LINEAGE_FACTS, rows),
-        LineageFacts::PythonAnalysis | LineageFacts::PythonEnrichment => return None,
     };
     let mut encoder: Encoder<'_> = Encoder::default();
     match legacy_tables {
@@ -464,7 +463,7 @@ pub(crate) fn encode_outcome(
         encoder.text(severity);
     }
     encoder.flag(analysis.binding_validated);
-    Some(encoder.into_bytes())
+    encoder.into_bytes()
 }
 
 /// The outcome [`encode_outcome`] stored, validated against `schema`, without cleaned SQL.

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs

@@ -1,13 +1,15 @@
-//! A session copy of the compile's binding catalog that records what Python's would gain.
+//! The session's binding catalog copy and Python's published relation shapes.
 
 use std::collections::{HashMap, HashSet};
 
 use crate::assembly::analysis_session::_helpers::compact_batch::{MemoKey, SharedResult};
 use crate::assembly::analysis_session::_helpers::mappings::{
-    ShapeTable, catalog_columns, catalog_relations, same_mapping,
+    ShapeTable, catalog_columns, catalog_relations, same_mapping, with_pair,
 };
+use crate::assembly::analysis_session::constants::{QUOTED_IDENTIFIER_DELIMITER, UNKNOWN_TYPE};
 use crate::assembly::analysis_session::types::{Pairs, Shapes};
-use crate::semantic_validation::models::{Columns, ProjectCatalog};
+use crate::semantic_validation::main::normalize::normalize_analysis_sql;
+use crate::semantic_validation::models::{Columns, NormalizationInput, ProjectCatalog};
 use crate::semantic_validation::types::{BindingRequest, Relations};
 
 /// Python's `BindingCatalog` state that model analysis reads and changes.
@@ -143,3 +145,71 @@ impl SessionCatalog {
         (self.additions, self.analysis_names)
     }
 }
+
+/// How a session publishes relation shapes.
+pub(crate) struct ShapeOptions<'a> {
+    pub(crate) dialect: &'a str,
+    pub(crate) case_sensitive: bool,
+}
+
+/// One relation's inferred columns and the SQL and inputs they came from.
+pub(crate) struct ShapeSource<'a> {
+    pub(crate) sql: &'a str,
+    pub(crate) columns: Pairs,
+    pub(crate) inputs: &'a Shapes,
+    pub(crate) snapshot_columns: Option<&'a (String, String)>,
+}
+
+impl SessionCatalog {
+    /// A model relation's shape, including snapshot validity columns.
+    pub(crate) fn published_model_shape(
+        &self,
+        options: &ShapeOptions<'_>,
+        source: ShapeSource<'_>,
+    ) -> Result<Pairs, String> {
+        let snapshot_columns: Option<&(String, String)> = source.snapshot_columns;
+        let mut shape: Pairs = self.inferred_binding_shape(options, source)?;
+        if let Some((valid_from, valid_to)) = snapshot_columns {
+            shape = with_pair(shape, valid_from, UNKNOWN_TYPE);
+            shape = with_pair(shape, valid_to, UNKNOWN_TYPE);
+        }
+        Ok(shape)
+    }
+
+    /// Column names keep authored quoting only where the dialect binds quoted names exactly.
+    pub(crate) fn inferred_binding_shape(
+        &self,
+        options: &ShapeOptions<'_>,
+        source: ShapeSource<'_>,
+    ) -> Result<Pairs, String> {
+        if !options.case_sensitive {
+            return Ok(source.columns);
+        }
+        let normalized: String = normalize_analysis_sql(NormalizationInput {
+            sql: source.sql.to_owned(),
+            dialect: options.dialect.to_owned(),
+            stubs: HashMap::new(),
+            placeholders: HashMap::new(),
+        })?;
+        if !normalized.contains(QUOTED_IDENTIFIER_DELIMITER) && !has_exact_input(source.inputs) {
+            return Ok(source.columns);
+        }
+        Ok(self
+            .native
+            .inferred_schema(
+                &normalized,
+                catalog_columns(&source.columns),
+                catalog_relations(source.inputs),
+            )?
+            .into_iter()
+            .map(|(name, data_type)| (name, data_type.unwrap_or_default()))
+            .collect())
+    }
+}
+
+fn has_exact_input(inputs: &Shapes) -> bool {
+    inputs
+        .iter()
+        .flat_map(|(_, shape)| shape.iter())
+        .any(|(name, _)| name.contains(QUOTED_IDENTIFIER_DELIMITER))
+}