perf: build the sqb binary and the extension from one set of compiled dependencies (#1179)

* perf: build the crates sqb shares with the extension once

The sqb build resolved several dependency features differently from the
extension build, so Cargo compiled polyglot-sql, sqlbuild-analysis,
sqlbuild-rules and their dependencies twice in every wheel and uv sync.
A new code-free sqlbuild-build-features crate enables the missing features
in the sqb build, and make check-build-features fails when the two builds
drift apart again.

* fix: read cargo tree without colour in the build features check

* ci: let the macOS sqb wheel build reuse the extension's crates

Author: kvlonge

.github/workflows/build-distributions.yml

@@ -135,6 +135,13 @@ jobs:
           manylinux: auto
       - uses: PyO3/maturin-action@v1
         if: inputs.prebuilt-run-id == ''
+        # On macOS maturin links the extension with these flags for every crate, which changes
+        # their build hash. Passing the same flags here lets sqb reuse the crates the extension
+        # build just compiled instead of compiling them again; sqb has no undefined symbols, and
+        # the smoke test below runs it.
+        env:
+          RUSTFLAGS: >-
+            ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
         with:
           command: build
           args: >-

Cargo.lock

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "crates/sqlbuild-build-features",
     "crates/sqlbuild-core",
     "crates/sqlbuild-cursor",
     "crates/sqlbuild-state",
@@ -27,6 +28,7 @@ resolver = "2"
 # out of every crate below the polyglot floor. See crates/README.md.
 [workspace.metadata.native-layers]
 order = [
+    "sqlbuild-build-features",
     "sqlbuild-core",
     "sqlbuild-cursor",
     "sqlbuild-state",
@@ -79,6 +81,17 @@ polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.
     "dialect-duckdb",
 ] }
 blake3 = { version = "1", features = ["rayon"] }
+# Only sqlbuild-build-features uses these: each enables in the sqb build exactly the features the
+# extension build resolves for that crate, so both builds share one compile of every common crate.
+arrayvec = "0.7"
+bstr = "1"
+either = { version = "1", default-features = false, features = ["use_std"] }
+memchr = "2"
+once_cell = "1"
+regex-automata = { version = "0.4", default-features = false, features = ["dfa-search"] }
+syn-v2-macro-features = { package = "syn", version = "2", default-features = false, features = ["extra-traits", "fold", "visit-mut"] }
+syn-v3-macro-features = { package = "syn", version = "3", default-features = false, features = ["extra-traits", "full"] }
+thiserror = "2"
 # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it.
 ctrlc = "3"
 quote = "1"
@@ -101,6 +114,7 @@ serde_json = { version = "1", features = ["preserve_order"] }
 sha2 = "0.10"
 sqlbuild-analysis = { path = "crates/sqlbuild-analysis" }
 sqlbuild-attachments = { path = "crates/sqlbuild-attachments" }
+sqlbuild-build-features = { path = "crates/sqlbuild-build-features" }
 sqlbuild-cache = { path = "crates/sqlbuild-cache" }
 sqlbuild-cli = { path = "crates/sqlbuild-cli" }
 sqlbuild-config = { path = "crates/sqlbuild-config" }

Cargo.toml

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 
-.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate check-docs-skill \
+.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-build-features check-python-boundary check-release-gate check-docs-skill \
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
 	test-e2e-duckdb-build-incremental \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
@@ -641,7 +641,7 @@ verify-quick:
 	exit $$status
 
 
-check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate check-docs-skill rust-lint
+check-ci: check-e2e-shards check-native-layering check-build-features check-python-boundary check-release-gate check-docs-skill rust-lint
 	uv run ruff format --check .
 	uv run ruff check .
 	uv run ty check src tests
@@ -653,6 +653,10 @@ check-native-layering:
 	uv run python -m scripts.check_native_layering
 
 
+check-build-features:
+	uv run python -m scripts.check_build_features
+
+
 check-python-boundary:
 	uv run python -m scripts.check_python_boundary
 

Makefile

@@ -6,6 +6,7 @@ that can be tested and benchmarked with `cargo test` alone.
 
 | Crate | Responsibility |
 |---|---|
+| `sqlbuild-build-features` | No code. Enables, in the `sqb` build, the dependency features the extension build resolves, so Cargo compiles the crates both builds share (polyglot-sql, `sqlbuild-analysis`, `sqlbuild-rules` and the rest) once. `make check-build-features` (part of `make check-ci`) fails when the two builds resolve a shared crate differently and names the features to add. |
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. |
 | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. |
 | `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. It also owns execution history: lifecycle event batch digests and conflicts, run projection, event and run cursors, filter and page validation, and the SQLite and PostgreSQL statements the native history backends run. |
@@ -34,7 +35,7 @@ sqlbuild-python
   -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
   -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
   -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
-sqlbuild-cli -> sqlbuild-cache, sqlbuild-core
+sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features
 ```
 
 `sqlbuild-scopes` reuses the macro scanner in `sqlbuild-render`; render itself depends only on

crates/README.md

@@ -0,0 +1,32 @@
+[package]
+name = "sqlbuild-build-features"
+version.workspace = true
+description = "No code: makes the sqb build resolve dependency features exactly as the extension build does, so both builds share one compile of every common crate."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+# Cargo resolves features separately for each build, so without these entries the sqb build
+# hashes the crates it shares with the extension differently and compiles polyglot-sql,
+# sqlbuild-analysis and sqlbuild-rules a second time. The workspace entries enable in the sqb
+# build exactly the features the extension build already resolves; sqlbuild-cli depends on this
+# crate, so the extension build sees them too and stays unchanged. `make check-build-features`
+# fails when the two builds resolve a shared crate differently.
+[dependencies]
+arrayvec.workspace = true
+bstr.workspace = true
+either.workspace = true
+memchr.workspace = true
+once_cell.workspace = true
+regex-automata.workspace = true
+thiserror.workspace = true
+
+# Proc-macro dependencies resolve with build dependencies, so the syn features PyO3's macros use
+# are enabled here. The empty build script makes Cargo resolve them.
+[build-dependencies]
+syn-v2-macro-features.workspace = true
+syn-v3-macro-features.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-build-features/Cargo.toml

@@ -0,0 +1,3 @@
+//! Empty: its presence makes Cargo resolve the build dependencies that align proc-macro features.
+
+fn main() {}

crates/sqlbuild-build-features/build.rs

@@ -0,0 +1,3 @@
+//! No code: aligns dependency features between the sqb and extension builds (see Cargo.toml).
+
+#![forbid(unsafe_code)]

crates/sqlbuild-build-features/src/lib.rs

@@ -20,6 +20,7 @@ blake3.workspace = true
 getrandom.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+sqlbuild-build-features.workspace = true
 sqlbuild-cache.workspace = true
 sqlbuild-config.workspace = true
 sqlbuild-core.workspace = true

crates/sqlbuild-cli/Cargo.toml

@@ -1567,6 +1567,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types"
 crate_names = [
     "sqlbuild-analysis",
     "sqlbuild-attachments",
+    "sqlbuild-build-features",
     "sqlbuild-cache",
     "sqlbuild-cli",
     "sqlbuild-config",