refactor(native): split the native crate into a layered workspace

.github/workflows/release-please.yml

@@ -186,7 +186,7 @@ jobs:
           done
 
           unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \
-            --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "crates/sqlbuild-rules-native/Cargo.toml")] | join("\n")')"
+            --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "Cargo.toml")] | join("\n")')"
           if [[ -n "${unexpected_files}" ]]; then
             echo "Release pull request contains non-metadata changes:" >&2
             echo "${unexpected_files}" >&2

Cargo.toml

@@ -1,10 +1,29 @@
 [workspace]
 members = [
-    "crates/sqlbuild-rules-native",
+    "crates/sqlbuild-core",
+    "crates/sqlbuild-sqltext",
+    "crates/sqlbuild-analysis",
+    "crates/sqlbuild-rules",
+    "crates/sqlbuild-python",
 ]
 resolver = "2"
 
+# Native crates from the bottom up: each crate may depend only on crates listed before it.
+# `make check-ci` enforces this, keeps PyO3 in the Python boundary crate and keeps polyglot
+# out of every crate below the polyglot floor. See crates/README.md.
+[workspace.metadata.native-layers]
+order = [
+    "sqlbuild-core",
+    "sqlbuild-sqltext",
+    "sqlbuild-analysis",
+    "sqlbuild-rules",
+    "sqlbuild-python",
+]
+python-boundary = "sqlbuild-python"
+polyglot-floor = "sqlbuild-analysis"
+
 [workspace.package]
+version = "0.136.0"
 edition = "2024"
 license = "Apache-2.0"
 publish = false
@@ -36,8 +55,13 @@ quote = "1"
 rayon = "1"
 regex = "1"
 serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+# Every crate relies on insertion-ordered JSON objects, which the extension has always built with.
+serde_json = { version = "1", features = ["preserve_order"] }
 sha2 = "0.10"
+sqlbuild-analysis = { path = "crates/sqlbuild-analysis" }
+sqlbuild-core = { path = "crates/sqlbuild-core" }
+sqlbuild-rules = { path = "crates/sqlbuild-rules" }
+sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 stacker = "0.1"
 sqlparser = { version = "0.58", features = ["visitor"] }
 syn = { version = "2", features = ["full", "visit", "extra-traits"] }

Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 
-.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check \
+.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering \
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
 	test-e2e-duckdb-build-incremental \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
@@ -566,14 +566,18 @@ verify-quick:
 	exit $$status
 
 
-check-ci: check-e2e-shards
+check-ci: check-e2e-shards check-native-layering
 	uv run ruff format --check .
 	uv run ruff check .
 	uv run ty check src tests
 	uv run pytest tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py -q
 	uv run fensu check
 
 
+check-native-layering:
+	uv run python -m scripts.check_native_layering
+
+
 check-pr-metadata:
 	@test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; }
 	@test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; }

crates/README.md

@@ -0,0 +1,48 @@
+# Native crates
+
+SQLBuild's native code is a Cargo workspace that builds one Python extension module,
+`sqlbuild._native`. Only `sqlbuild-python` knows about Python; every other crate is plain Rust
+that can be tested and benchmarked with `cargo test` alone.
+
+| Crate | Responsibility |
+|---|---|
+| `sqlbuild-core` | Constants shared by several crates and the panic boundary that turns an unwinding native failure into a named compiler error. |
+| `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. |
+| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. |
+| `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. |
+| `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. |
+
+## Layering
+
+Dependencies point one way, from the top of this graph to the bottom:
+
+```text
+sqlbuild-python
+  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-sqltext -> sqlbuild-core
+```
+
+Each crate may depend only on crates below it, and may also skip layers. The order is declared
+in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and
+`make check-native-layering` (part of `make check-ci`) fails when:
+
+- a crate depends on a crate that is not in an earlier layer, or a workspace crate is missing
+  from the order;
+- any crate other than `sqlbuild-python` reaches `pyo3`, directly or through another package;
+- a crate below `sqlbuild-analysis` reaches polyglot.
+
+The same check fails when the workspace version, the `pyproject.toml` version and
+`.release-please-manifest.json` disagree, because the version is part of the build identity.
+
+`fensu check` applies the Rust structure rules inside each crate: other crates use items through
+`main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`.
+
+## Working on the crates
+
+- Tests live next to the module they cover; run them for one crate with
+  `cargo test -p <crate>` or for all crates with `cargo test --workspace`.
+- `sqlbuild-analysis` has a `test-support` feature that exposes test-only helpers to the tests of
+  dependent crates. Enable it only from `[dev-dependencies]`.
+- The build identity (`sqlbuild._native.BUILD_IDENTITY`) hashes the sources of every crate in
+  this directory, so editing any crate invalidates the native caches keyed on it.
+- maturin builds `crates/sqlbuild-python`; the release version lives in
+  `[workspace.package]` and is shared by every crate.

crates/sqlbuild-analysis/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "sqlbuild-analysis"
+version.workspace = true
+description = "Polyglot-based SQL analysis, binding and semantic validation for SQLBuild."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[features]
+default = []
+# Expose test-only helpers to the tests of dependent crates.
+test-support = []
+
+[dependencies]
+polyglot-sql.workspace = true
+polyglot-sql-function-catalogs.workspace = true
+rayon.workspace = true
+regex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+sqlbuild-core.workspace = true
+sqlbuild-sqltext.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-analysis/src/column_references/main/analyze.rs

@@ -0,0 +1,3 @@
+pub fn analyze_json(request_json: &str) -> Result<String, String> {
+    crate::column_references::references::analyze_json(request_json)
+}

crates/sqlbuild-analysis/src/column_references/main/mod.rs

@@ -0,0 +1 @@
+pub mod analyze;

crates/sqlbuild-rules-native/src/column_references/mod.rs → crates/sqlbuild-analysis/src/column_references/mod.rs

@@ -1,4 +1,4 @@
-pub(crate) mod main;
+pub mod main;
 #[path = "_helpers/references.rs"]
 mod references;
 

crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs

@@ -0,0 +1 @@
+pub mod sql_tests;

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_rename.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_rename.rs

@@ -3,9 +3,9 @@
 use crate::compiler::_helpers::sql_tests::cte_slices::{
     SliceDialect, WithSlices, opaque_end, read_identifier,
 };
-use crate::sql_scan::main::comment_end::comment_end;
-use crate::sql_scan::main::skip_whitespace::skip_whitespace;
-use crate::sql_scan::models::Unclosed;
+use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end;
+use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
+use sqlbuild_sqltext::sql_scan::models::Unclosed;
 
 const DOT: &str = ".";
 

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_slices.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_slices.rs

@@ -3,11 +3,11 @@
 use std::collections::HashSet;
 
 use crate::compiler::_helpers::sql_tests::cte_sql::keyword_end;
-use crate::rules::main::quote_policy::quote_policy;
-use crate::sql_scan::main::comment_end::comment_end;
-use crate::sql_scan::main::non_code_end::non_code_end;
-use crate::sql_scan::main::skip_whitespace::skip_whitespace;
-use crate::sql_scan::models::{QuotePolicy, Unclosed};
+use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end;
+use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end;
+use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy;
+use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
+use sqlbuild_sqltext::sql_scan::models::{QuotePolicy, Unclosed};
 
 /// Dialect lexical rules for slicing authored SQL.
 #[derive(Clone, Copy, Debug)]

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_sql.rs

@@ -1,7 +1,7 @@
 //! CTE definition and leading-WITH assembly shared by SQL-test planning and rendering.
 
-use crate::sql_scan::main::comment_end::comment_end;
-use crate::sql_scan::main::skip_whitespace::skip_whitespace;
+use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end;
+use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
 
 /// Render one `name AS (body)` definition, closing a trailing line comment first.
 pub(crate) fn cte_definition_sql(name: &str, sql: &str) -> String {

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs

@@ -7,13 +7,13 @@ use serde::{Deserialize, Serialize};
 use crate::constants::{
     DIRECT_DEPENDENCY_PATH_LENGTH, MACRO_TEST_MODE, TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE,
 };
-use crate::sql_scan::main::comment_end::comment_end;
-use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren;
-use crate::sql_scan::main::non_code_end::non_code_end;
-use crate::sql_scan::main::quote_end::quote_end;
-use crate::sql_scan::main::skip_whitespace::skip_whitespace;
-use crate::sql_scan::models::QuotePolicy;
-use crate::sql_scan::models::Unclosed;
+use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end;
+use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren as scan_matching_paren;
+use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end;
+use sqlbuild_sqltext::sql_scan::main::quote_end::quote_end;
+use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
+use sqlbuild_sqltext::sql_scan::models::QuotePolicy;
+use sqlbuild_sqltext::sql_scan::models::Unclosed;
 
 const MODEL_PREFIXES: [&str; 8] = [
     "__macro__",

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs

@@ -5,10 +5,10 @@ use std::collections::HashSet;
 use regex::{Captures, Regex};
 
 use crate::compiler::_helpers::sql_tests::planning::compile_error;
-use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
-use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
-use crate::sql_scan::main::skip_whitespace::skip_whitespace;
-use crate::sql_scan::models::{LexicalSyntax, Unclosed};
+use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
+use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
+use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
+use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed};
 
 pub(crate) fn replace_callable_markers<F>(
     sql: &str,

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs

@@ -0,0 +1,10 @@
+pub mod cte_rename;
+pub mod cte_slices;
+pub mod cte_sql;
+pub mod expected_columns;
+pub mod extraction;
+pub mod helper_scope;
+pub mod markers;
+pub mod planning;
+pub mod relation_markers;
+pub mod rendering;

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs → crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs

@@ -27,7 +27,7 @@ use crate::compiler::_helpers::sql_tests::rendering::{
     rendered_chain_steps,
 };
 use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE};
-use crate::sql_scan::models::LexicalSyntax;
+use sqlbuild_sqltext::sql_scan::models::LexicalSyntax;
 
 const DEFAULT_WORKERS: usize = 4;
 const MAX_WORKERS: usize = 4;

crates/sqlbuild-analysis/src/compiler/main/mod.rs

@@ -0,0 +1,6 @@
+pub mod sql_test_chain_resolution;
+pub mod sql_test_difference_sampling;
+pub mod sql_test_extraction;
+pub mod sql_test_fixture_facts;
+pub mod sql_test_planning;
+pub mod sql_test_rendering;

crates/sqlbuild-rules-native/src/compiler/main/sql_test_chain_resolution.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_chain_resolution.rs

@@ -1,5 +1,5 @@
 use crate::compiler::_helpers;
 
-pub(crate) fn resolve_chains_json(request_json: &str) -> Result<String, String> {
+pub fn resolve_chains_json(request_json: &str) -> Result<String, String> {
     _helpers::sql_tests::planning::resolve_chains_json(request_json)
 }

crates/sqlbuild-rules-native/src/compiler/main/sql_test_difference_sampling.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_difference_sampling.rs

@@ -1,5 +1,5 @@
 use crate::compiler::_helpers;
 
-pub(crate) fn render_difference_sample_json(request_json: &str) -> Result<String, String> {
+pub fn render_difference_sample_json(request_json: &str) -> Result<String, String> {
     _helpers::sql_tests::rendering::render_difference_sample_json(request_json)
 }

crates/sqlbuild-rules-native/src/compiler/main/sql_test_extraction.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_extraction.rs

@@ -1,5 +1,5 @@
 use crate::compiler::_helpers;
 
-pub(crate) fn extract_batch_json(request_json: &str) -> Result<String, String> {
+pub fn extract_batch_json(request_json: &str) -> Result<String, String> {
     _helpers::sql_tests::extraction::extract_batch_json(request_json)
 }

crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_fixture_facts.rs

@@ -1,7 +1,7 @@
 use crate::compiler::_helpers::sql_tests::{extraction, planning};
 use crate::compiler::models::SqlTestFixtureFacts;
 
-pub(crate) fn fixture_facts(name: &str, sql: &str) -> SqlTestFixtureFacts {
+pub fn fixture_facts(name: &str, sql: &str) -> SqlTestFixtureFacts {
     SqlTestFixtureFacts {
         mock: planning::MOCK_CTE_PREFIXES.iter().any(|prefix| {
             name.strip_prefix(prefix)

crates/sqlbuild-rules-native/src/compiler/main/sql_test_planning.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_planning.rs

@@ -1,5 +1,5 @@
 use crate::compiler::_helpers;
 
-pub(crate) fn plan_and_render_json(request_json: &str) -> Result<String, String> {
+pub fn plan_and_render_json(request_json: &str) -> Result<String, String> {
     _helpers::sql_tests::planning::plan_and_render_json(request_json)
 }

crates/sqlbuild-rules-native/src/compiler/main/sql_test_rendering.rs → crates/sqlbuild-analysis/src/compiler/main/sql_test_rendering.rs

@@ -1,5 +1,5 @@
 use crate::compiler::_helpers;
 
-pub(crate) fn render_json(request_json: &str) -> Result<String, String> {
+pub fn render_json(request_json: &str) -> Result<String, String> {
     _helpers::sql_tests::rendering::render_json(request_json)
 }