feat(native): resolve declaration scopes natively behind a native-preview engine tier

.github/workflows/compiler-differential.yml

@@ -16,7 +16,7 @@ jobs:
   dense:
     name: Dense compiler differential (${{ matrix.models }} models)
     runs-on: ubuntu-24.04
-    timeout-minutes: 90
+    timeout-minutes: 150
     strategy:
       fail-fast: false
       matrix:
@@ -37,8 +37,14 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
-      - name: Compare dense projects with and without custom rules
+      - name: Compare dense projects with and without custom rules on the preview engine
         env:
           MODELS: ${{ matrix.models }}
         run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
+      - name: Compare dense projects with and without custom rules on the shipped engine
+        env:
+          MODELS: ${{ matrix.models }}
+        run: >-
+          make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}"
+          COMPILER_DIFFERENTIAL_ARGS="--engines python native"
 

.github/workflows/verify.yml

@@ -290,11 +290,13 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
-      - name: Compare fixtures, examples, generated seeds and the failure corpus
+      - name: Compare fixtures, examples, generated seeds and the failure corpus (python vs native-preview)
         run: >-
           make compiler-differential
           COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
           --require-render-coverage"
+      - name: Compare generated seeds and the failure corpus (python vs shipped native)
+        run: make compiler-differential-shipped
 
   compiler_differential_python_versions:
     name: Native discovery on Python ${{ matrix.python }}
@@ -333,11 +335,12 @@ jobs:
           uv run --no-sync pytest
           tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py
           -n auto --dist loadfile
-      - name: Compare engines on fixtures, examples, the failure corpus and a few seeds
+      - name: Compare python with the shipped native default on fixtures, examples, the failure corpus and a few seeds
         env:
           PYTHONUNBUFFERED: "1"
         run: >-
           uv run --no-sync python -m scripts.run_compiler_differential
+          --engines python native
           --corpus fixtures examples failures seeds --seeds 4 --jobs 4
 
   format_performance:
@@ -758,6 +761,7 @@ jobs:
         checks,
         e2e_shards,
         compiler_differential,
+        compiler_differential_python_versions,
         format_performance,
         performance,
         cold_compile_performance,
@@ -778,6 +782,7 @@ jobs:
           CHECKS_RESULT: ${{ needs.checks.result }}
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
           COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
+          COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }}
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
@@ -800,6 +805,7 @@ jobs:
           for result in \
             "${E2E_SHARDS_RESULT}" \
             "${COMPILER_DIFFERENTIAL_RESULT}" \
+            "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "crates/sqlbuild-sqltext",
     "crates/sqlbuild-config",
     "crates/sqlbuild-discovery",
+    "crates/sqlbuild-scopes",
     "crates/sqlbuild-analysis",
     "crates/sqlbuild-rules",
     "crates/sqlbuild-python",
@@ -19,6 +20,7 @@ order = [
     "sqlbuild-sqltext",
     "sqlbuild-config",
     "sqlbuild-discovery",
+    "sqlbuild-scopes",
     "sqlbuild-analysis",
     "sqlbuild-rules",
     "sqlbuild-python",
@@ -68,6 +70,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" }
 sqlbuild-core = { path = "crates/sqlbuild-core" }
 sqlbuild-discovery = { path = "crates/sqlbuild-discovery" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }
+sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
 sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 stacker = "0.1"
 sqlparser = { version = "0.58", features = ["visitor"] }

Makefile

@@ -7,7 +7,7 @@ SHELL := /bin/bash
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
-	compiler-differential compiler-differential-dense
+	compiler-differential compiler-differential-shipped compiler-differential-dense
 
 format:
 	uv run ruff format .
@@ -289,6 +289,7 @@ test-e2e-varied-cache-performance:
 
 COMPILER_DIFFERENTIAL_JOBS ?= 4
 COMPILER_DIFFERENTIAL_SEEDS ?= 12
+COMPILER_DIFFERENTIAL_SHIPPED_SEEDS ?= 12
 COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
 COMPILER_DIFFERENTIAL_ARGS ?=
 
@@ -297,6 +298,14 @@ compiler-differential:
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
 		$(COMPILER_DIFFERENTIAL_ARGS)
 
+# python vs native-preview above covers every native stage; this keeps the shipped `native` default
+# covered on a bounded corpus.
+compiler-differential-shipped:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
+		--engines python native --corpus seeds failures \
+		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SHIPPED_SEEDS) \
+		$(COMPILER_DIFFERENTIAL_ARGS)
+
 compiler-differential-dense:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
 		--corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \

crates/README.md

@@ -10,6 +10,7 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. |
 | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. |
 | `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. |
+| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. |
 | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. |
 | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. |
 | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. |
@@ -20,13 +21,14 @@ Dependencies point one way, from the top of this graph to the bottom:
 
 ```text
 sqlbuild-python
-  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-discovery -> sqlbuild-config
+  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config
   -> sqlbuild-sqltext -> sqlbuild-core
 ```
 
-`sqlbuild-config` does not depend on the crates below it today; its place in the order only
-fixes which crates may use it. The JSON emitter lives in `sqlbuild-core` so that any layer can
-produce text that must equal Python's `json.dumps` output.
+`sqlbuild-config` does not depend on the crates below it today, and `sqlbuild-scopes` uses only
+`sqlbuild-sqltext`; their place in the order fixes which crates may use them. The JSON emitter
+lives in `sqlbuild-core` so that any layer can produce text that must equal Python's
+`json.dumps` output.
 
 Each crate may depend only on crates below it, and may also skip layers. The order is declared
 in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and
@@ -43,6 +45,28 @@ The same check fails when the workspace version, the `pyproject.toml` version an
 `fensu check` applies the Rust structure rules inside each crate: other crates use items through
 `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`.
 
+## Compiler engines
+
+`SQLBUILD_COMPILER_ENGINE` (or the hidden `--compiler-engine` flag) selects which compiler
+stages run natively:
+
+| Engine | Runs |
+|---|---|
+| `python` | The Python compiler only. It is the oracle every native stage must match byte for byte. |
+| `native` | The default: native stages that passed their flip gate (`shipped` tier). |
+| `native-preview` | Opt-in: shipped stages plus stages still in development (`preview` tier). |
+
+Each native stage declares its tier once, in `NATIVE_STAGE_TIERS` in
+`src/sqlbuild/compiler/frontier/constants.py`. A stage moves from `preview` to `shipped` by
+changing that line, after its flip gate passes: a byte-identical real project, a green
+differential, and no measured slowdown on cold, edit and no-change compiles. Every engine keeps
+its own compiler, Rules and compile-reuse stores, so preview output is never reused by `native`.
+
+`make compiler-differential` compares `python` with `native-preview` on the full per-PR corpus.
+`make compiler-differential-shipped` compares `python` with `native` on the generated seeds and
+the failure corpus, so the shipped default stays covered on its own. CI runs both, and compares
+`python` with `native` on Python 3.13 and 3.14 as well.
+
 ## Working on the crates
 
 - Tests live next to the module they cover; run them for one crate with

crates/sqlbuild-python/Cargo.toml

@@ -25,6 +25,7 @@ sqlbuild-config.workspace = true
 sqlbuild-core.workspace = true
 sqlbuild-discovery.workspace = true
 sqlbuild-rules.workspace = true
+sqlbuild-scopes.workspace = true
 sqlbuild-sqltext.workspace = true
 
 [lints]

crates/sqlbuild-python/src/bindings/_helpers/catalog.rs → crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs

@@ -1,8 +1,8 @@
 //! Python methods of the compile-owned binding catalog and its compact analysis jobs.
 
-use crate::bindings::_helpers::conversions;
-use crate::bindings::_helpers::normalization_results::normalization_results;
-use crate::bindings::_helpers::panics::compiler_error;
+use crate::bindings::_helpers::analysis::conversions;
+use crate::bindings::_helpers::analysis::normalization_results::normalization_results;
+use crate::bindings::_helpers::boundary::panics::compiler_error;
 use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog};
 use crate::bindings::types::CompilerDetach;
 use pyo3::exceptions::PyValueError;

crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs

@@ -0,0 +1,7 @@
+mod catalog;
+pub(crate) mod conversions;
+pub(crate) mod normalization;
+pub(crate) mod normalization_results;
+mod positions;
+pub(crate) mod queries;
+pub(crate) mod sql_tests;

crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs

@@ -0,0 +1,71 @@
+//! Normalize SQL for analysis and recover binding diagnostics.
+
+use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python};
+use pyo3::{pyfunction, wrap_pyfunction};
+
+use crate::bindings::_helpers::analysis::normalization_results::normalization_results;
+use crate::bindings::_helpers::boundary::panics::value_error;
+use crate::bindings::types::CompilerDetach;
+
+#[pyfunction]
+fn normalize_analysis_sql(
+    py: Python<'_>,
+    request: crate::bindings::models::NormalizationInput,
+) -> PyResult<String> {
+    py.compiler_detach(|| {
+        sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql(
+            request.into(),
+        )
+    })
+    .map_err(value_error)
+}
+
+#[pyfunction]
+fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult<String> {
+    py.compiler_detach(|| {
+        sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql(
+            sql, dialect,
+        )
+    })
+    .map_err(value_error)
+}
+
+/// Normalize a batch in one detached call; a failed member is returned as its exception.
+#[pyfunction]
+fn normalize_analysis_sqls(
+    py: Python<'_>,
+    dialect: &str,
+    requests: Vec<sqlbuild_analysis::semantic_validation::types::NormalizationRequest>,
+) -> PyResult<Vec<Py<PyAny>>> {
+    let results = py.compiler_detach(|| {
+        Ok(
+            sqlbuild_analysis::semantic_validation::main::normalize_batch::normalize_analysis_sqls(
+                dialect, requests, None,
+            ),
+        )
+    });
+    normalization_results(py, results.map_err(value_error)?)
+}
+
+#[pyfunction]
+fn binding_diagnostics(
+    py: Python<'_>,
+    sql: &str,
+    dialect: &str,
+    rows: Vec<sqlbuild_analysis::semantic_validation::types::DiagnosticRow>,
+) -> PyResult<Vec<sqlbuild_analysis::semantic_validation::types::DiagnosticRow>> {
+    py.compiler_detach(|| {
+        sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics(
+            sql, dialect, rows,
+        )
+    })
+    .map_err(value_error)
+}
+
+pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?;
+    module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?;
+    module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?;
+    module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?;
+    Ok(())
+}

crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs → crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs

@@ -9,9 +9,11 @@ pub(crate) fn normalization_results(
         .into_iter()
         .map(|result| match result {
             Ok(sql) => Ok(sql.into_pyobject(py)?.into_any().unbind()),
-            Err(error) => Ok(crate::bindings::_helpers::panics::compiler_error(error)
-                .into_value(py)
-                .into_any()),
+            Err(error) => Ok(
+                crate::bindings::_helpers::boundary::panics::compiler_error(error)
+                    .into_value(py)
+                    .into_any(),
+            ),
         })
         .collect()
 }

crates/sqlbuild-python/src/bindings/_helpers/positions.rs → crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs

@@ -1,6 +1,6 @@
 //! Python methods of authored binding positions.
 
-use crate::bindings::_helpers::panics::compiler_guard;
+use crate::bindings::_helpers::boundary::panics::compiler_guard;
 use crate::bindings::models::{BindingPositions, PositionInput};
 use pyo3::exceptions::PyValueError;
 use pyo3::{PyResult, pymethods};