perf: build the sqb binary and the extension from one set of compiled dependencies

.github/workflows/build-distributions.yml

@@ -135,6 +135,13 @@ jobs:
           manylinux: auto
       - uses: PyO3/maturin-action@v1
         if: inputs.prebuilt-run-id == ''
+        # On macOS maturin links the extension with these flags for every crate, which changes
+        # their build hash. Passing the same flags here lets sqb reuse the crates the extension
+        # build just compiled instead of compiling them again; sqb has no undefined symbols, and
+        # the smoke test below runs it.
+        env:
+          RUSTFLAGS: >-
+            ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
         with:
           command: build
           args: >-

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "crates/sqlbuild-build-features",
     "crates/sqlbuild-core",
     "crates/sqlbuild-cursor",
     "crates/sqlbuild-state",
@@ -27,6 +28,7 @@ resolver = "2"
 # out of every crate below the polyglot floor. See crates/README.md.
 [workspace.metadata.native-layers]
 order = [
+    "sqlbuild-build-features",
     "sqlbuild-core",
     "sqlbuild-cursor",
     "sqlbuild-state",
@@ -79,6 +81,17 @@ polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.
     "dialect-duckdb",
 ] }
 blake3 = { version = "1", features = ["rayon"] }
+# Only sqlbuild-build-features uses these: each enables in the sqb build exactly the features the
+# extension build resolves for that crate, so both builds share one compile of every common crate.
+arrayvec = "0.7"
+bstr = "1"
+either = { version = "1", default-features = false, features = ["use_std"] }
+memchr = "2"
+once_cell = "1"
+regex-automata = { version = "0.4", default-features = false, features = ["dfa-search"] }
+syn-v2-macro-features = { package = "syn", version = "2", default-features = false, features = ["extra-traits", "fold", "visit-mut"] }
+syn-v3-macro-features = { package = "syn", version = "3", default-features = false, features = ["extra-traits", "full"] }
+thiserror = "2"
 # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it.
 ctrlc = "3"
 quote = "1"
@@ -101,6 +114,7 @@ serde_json = { version = "1", features = ["preserve_order"] }
 sha2 = "0.10"
 sqlbuild-analysis = { path = "crates/sqlbuild-analysis" }
 sqlbuild-attachments = { path = "crates/sqlbuild-attachments" }
+sqlbuild-build-features = { path = "crates/sqlbuild-build-features" }
 sqlbuild-cache = { path = "crates/sqlbuild-cache" }
 sqlbuild-cli = { path = "crates/sqlbuild-cli" }
 sqlbuild-config = { path = "crates/sqlbuild-config" }

Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 
-.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate check-docs-skill \
+.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-build-features check-python-boundary check-release-gate check-docs-skill \
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
 	test-e2e-duckdb-build-incremental \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
@@ -641,7 +641,7 @@ verify-quick:
 	exit $$status
 
 
-check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate check-docs-skill rust-lint
+check-ci: check-e2e-shards check-native-layering check-build-features check-python-boundary check-release-gate check-docs-skill rust-lint
 	uv run ruff format --check .
 	uv run ruff check .
 	uv run ty check src tests
@@ -653,6 +653,10 @@ check-native-layering:
 	uv run python -m scripts.check_native_layering
 
 
+check-build-features:
+	uv run python -m scripts.check_build_features
+
+
 check-python-boundary:
 	uv run python -m scripts.check_python_boundary
 

crates/README.md

@@ -6,6 +6,7 @@ that can be tested and benchmarked with `cargo test` alone.
 
 | Crate | Responsibility |
 |---|---|
+| `sqlbuild-build-features` | No code. Enables, in the `sqb` build, the dependency features the extension build resolves, so Cargo compiles the crates both builds share (polyglot-sql, `sqlbuild-analysis`, `sqlbuild-rules` and the rest) once. `make check-build-features` (part of `make check-ci`) fails when the two builds resolve a shared crate differently and names the features to add. |
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. |
 | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. |
 | `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. It also owns execution history: lifecycle event batch digests and conflicts, run projection, event and run cursors, filter and page validation, and the SQLite and PostgreSQL statements the native history backends run. |
@@ -34,7 +35,7 @@ sqlbuild-python
   -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
   -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
   -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
-sqlbuild-cli -> sqlbuild-cache, sqlbuild-core
+sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features
 ```
 
 `sqlbuild-scopes` reuses the macro scanner in `sqlbuild-render`; render itself depends only on

crates/sqlbuild-build-features/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "sqlbuild-build-features"
+version.workspace = true
+description = "No code: makes the sqb build resolve dependency features exactly as the extension build does, so both builds share one compile of every common crate."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+# Cargo resolves features separately for each build, so without these entries the sqb build
+# hashes the crates it shares with the extension differently and compiles polyglot-sql,
+# sqlbuild-analysis and sqlbuild-rules a second time. The workspace entries enable in the sqb
+# build exactly the features the extension build already resolves; sqlbuild-cli depends on this
+# crate, so the extension build sees them too and stays unchanged. `make check-build-features`
+# fails when the two builds resolve a shared crate differently.
+[dependencies]
+arrayvec.workspace = true
+bstr.workspace = true
+either.workspace = true
+memchr.workspace = true
+once_cell.workspace = true
+regex-automata.workspace = true
+thiserror.workspace = true
+
+# Proc-macro dependencies resolve with build dependencies, so the syn features PyO3's macros use
+# are enabled here. The empty build script makes Cargo resolve them.
+[build-dependencies]
+syn-v2-macro-features.workspace = true
+syn-v3-macro-features.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-build-features/build.rs

@@ -0,0 +1,3 @@
+//! Empty: its presence makes Cargo resolve the build dependencies that align proc-macro features.
+
+fn main() {}

crates/sqlbuild-build-features/src/lib.rs

@@ -0,0 +1,3 @@
+//! No code: aligns dependency features between the sqb and extension builds (see Cargo.toml).
+
+#![forbid(unsafe_code)]

crates/sqlbuild-cli/Cargo.toml

@@ -20,6 +20,7 @@ blake3.workspace = true
 getrandom.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+sqlbuild-build-features.workspace = true
 sqlbuild-cache.workspace = true
 sqlbuild-config.workspace = true
 sqlbuild-core.workspace = true

fensu.toml

@@ -1567,6 +1567,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types"
 crate_names = [
     "sqlbuild-analysis",
     "sqlbuild-attachments",
+    "sqlbuild-build-features",
     "sqlbuild-cache",
     "sqlbuild-cli",
     "sqlbuild-config",

scripts/build_features/__init__.py

@@ -0,0 +1 @@
+"""Check that the sqb and extension builds resolve shared crates identically."""

scripts/build_features/_helpers/__init__.py

@@ -0,0 +1 @@
+"""Build feature alignment helpers."""

scripts/build_features/_helpers/drift.py

@@ -0,0 +1,24 @@
+"""Compare the crates the sqb and extension builds share."""
+
+from __future__ import annotations
+
+from scripts.build_features.models import FeatureDrift, ResolvedCrate
+
+
+def find_feature_drift(
+    *,
+    target: str,
+    sqb: dict[ResolvedCrate, frozenset[str]],
+    extension: dict[ResolvedCrate, frozenset[str]],
+) -> tuple[FeatureDrift, ...]:
+    """Return every crate both builds compile on the same side with different features."""
+    return tuple(
+        FeatureDrift(
+            target=target,
+            crate=crate,
+            sqb_features=sqb[crate],
+            extension_features=extension[crate],
+        )
+        for crate in sorted(sqb.keys() & extension.keys(), key=lambda c: (c.package, c.side))
+        if sqb[crate] != extension[crate]
+    )

scripts/build_features/_helpers/tree.py

@@ -0,0 +1,80 @@
+"""Read the features Cargo resolves for each crate of one build."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from scripts.build_features.constants import (
+    BUILD_DEPENDENCIES_DEPTH_OFFSET,
+    BUILD_DEPENDENCIES_HEADING,
+    HOST_SIDE,
+    PROC_MACRO_MARKER,
+    TARGET_SIDE,
+    TREE_FIELD_SEPARATOR,
+    TREE_FORMAT,
+    TREE_GLYPHS,
+    TREE_INDENT_WIDTH,
+)
+from scripts.build_features.models import ResolvedCrate
+
+
+def run_cargo_tree(*, root: Path, selection: tuple[str, ...], target: str) -> str:
+    """Return `cargo tree` output for the selected package on one target, without dedupe."""
+    completed: subprocess.CompletedProcess[str] = subprocess.run(
+        (
+            "cargo",
+            "tree",
+            "--locked",
+            *selection,
+            "--target",
+            target,
+            "--edges",
+            "normal,build",
+            "--format",
+            TREE_FORMAT,
+            "--prefix",
+            "indent",
+            "--no-dedupe",
+            "--color",
+            "never",
+        ),
+        cwd=root,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return completed.stdout
+
+
+def parse_cargo_tree(output: str) -> dict[ResolvedCrate, frozenset[str]]:
+    """Map each crate to its resolved feature lists, with host compiles apart from target ones."""
+    resolved: dict[ResolvedCrate, set[str]] = {}
+    open_ancestors: tuple[tuple[float, bool], ...] = ()
+    for line in output.splitlines():
+        body: str = line.lstrip(TREE_GLYPHS)
+        depth: int = (len(line) - len(body)) // TREE_INDENT_WIDTH
+        if body.startswith(BUILD_DEPENDENCIES_HEADING):
+            heading_depth: float = depth + BUILD_DEPENDENCIES_DEPTH_OFFSET
+            open_ancestors = (
+                *_enclosing(ancestors=open_ancestors, depth=heading_depth),
+                (heading_depth, True),
+            )
+            continue
+        if TREE_FIELD_SEPARATOR not in body:
+            continue
+        open_ancestors = _enclosing(ancestors=open_ancestors, depth=depth)
+        package, features = body.split(TREE_FIELD_SEPARATOR, 1)
+        host: bool = PROC_MACRO_MARKER in package or any(on_host for _, on_host in open_ancestors)
+        name: str = " ".join(package.replace(PROC_MACRO_MARKER, "").split()[:2])
+        crate: ResolvedCrate = ResolvedCrate(package=name, side=HOST_SIDE if host else TARGET_SIDE)
+        resolved.setdefault(crate, set()).add(features.strip())
+        open_ancestors = (*open_ancestors, (depth, host))
+    return {crate: frozenset(features) for crate, features in resolved.items()}
+
+
+def _enclosing(
+    *, ancestors: tuple[tuple[float, bool], ...], depth: float
+) -> tuple[tuple[float, bool], ...]:
+    """Keep the ancestors above `depth`; a build-dependencies heading sits half a level down."""
+    return tuple(ancestor for ancestor in ancestors if ancestor[0] < depth)

scripts/build_features/constants.py

@@ -0,0 +1,22 @@
+"""Stable build feature alignment constants."""
+
+SQB_PACKAGE: str = "sqlbuild-cli"
+EXTENSION_PACKAGE: str = "sqlbuild-python"
+EXTENSION_FEATURES: str = "extension-module"
+WHEEL_TARGETS: tuple[str, ...] = (
+    "x86_64-unknown-linux-gnu",
+    "aarch64-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "aarch64-apple-darwin",
+    "x86_64-pc-windows-msvc",
+)
+TREE_FORMAT: str = "{p}|{f}"
+TREE_INDENT_WIDTH: int = 4
+TREE_GLYPHS: str = "│├└─ "
+TREE_FIELD_SEPARATOR: str = "|"
+BUILD_DEPENDENCIES_DEPTH_OFFSET: float = 0.5
+BUILD_DEPENDENCIES_HEADING: str = "[build-dependencies]"
+PROC_MACRO_MARKER: str = " (proc-macro)"
+HOST_SIDE: str = "host"
+TARGET_SIDE: str = "target"
+FEATURE_CRATE: str = "crates/sqlbuild-build-features/Cargo.toml"

scripts/build_features/main/__init__.py

@@ -0,0 +1 @@
+"""Build feature alignment entrypoints."""

scripts/build_features/main/check.py

@@ -0,0 +1,70 @@
+"""Build feature alignment command entrypoint."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from scripts.build_features._helpers.drift import find_feature_drift
+from scripts.build_features._helpers.tree import parse_cargo_tree, run_cargo_tree
+from scripts.build_features.constants import (
+    EXTENSION_FEATURES,
+    EXTENSION_PACKAGE,
+    FEATURE_CRATE,
+    SQB_PACKAGE,
+    WHEEL_TARGETS,
+)
+from scripts.build_features.models import FeatureDrift
+
+
+def run_build_features_check(argv: list[str] | None = None) -> int:
+    """Fail when the sqb build and the extension build would compile a shared crate twice."""
+    parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    arguments: argparse.Namespace = parser.parse_args(argv)
+    drift: list[FeatureDrift] = []
+    for target in WHEEL_TARGETS:
+        drift.extend(
+            find_feature_drift(
+                target=target,
+                sqb=parse_cargo_tree(
+                    run_cargo_tree(
+                        root=arguments.root, selection=("--package", SQB_PACKAGE), target=target
+                    )
+                ),
+                extension=parse_cargo_tree(
+                    run_cargo_tree(
+                        root=arguments.root,
+                        selection=(
+                            "--package",
+                            EXTENSION_PACKAGE,
+                            "--features",
+                            EXTENSION_FEATURES,
+                        ),
+                        target=target,
+                    )
+                ),
+            )
+        )
+    if drift:
+        print(
+            f"The {SQB_PACKAGE} and {EXTENSION_PACKAGE} builds resolve shared crates with "
+            f"different features, so Cargo compiles them twice. Enable the missing features "
+            f"in {FEATURE_CRATE}:",
+            file=sys.stderr,
+        )
+        for entry in drift:
+            print(
+                f"- {entry.target} {entry.crate.side} {entry.crate.package}: "
+                f"sqb {_features(entry.sqb_features)}, "
+                f"extension {_features(entry.extension_features)}",
+                file=sys.stderr,
+            )
+        return 1
+    print(f"The {SQB_PACKAGE} and {EXTENSION_PACKAGE} builds share every common crate.")
+    return 0
+
+
+def _features(features: frozenset[str]) -> str:
+    return " | ".join(sorted(entry or "(none)" for entry in features))