feat(native): batch and cache macro calls behind the preview engine

Cargo.toml

@@ -1,11 +1,13 @@
 [workspace]
 members = [
     "crates/sqlbuild-core",
+    "crates/sqlbuild-cache",
     "crates/sqlbuild-sqltext",
     "crates/sqlbuild-config",
     "crates/sqlbuild-discovery",
     "crates/sqlbuild-scopes",
     "crates/sqlbuild-model-config",
+    "crates/sqlbuild-render",
     "crates/sqlbuild-analysis",
     "crates/sqlbuild-rules",
     "crates/sqlbuild-python",
@@ -18,11 +20,13 @@ resolver = "2"
 [workspace.metadata.native-layers]
 order = [
     "sqlbuild-core",
+    "sqlbuild-cache",
     "sqlbuild-sqltext",
     "sqlbuild-config",
     "sqlbuild-discovery",
     "sqlbuild-scopes",
     "sqlbuild-model-config",
+    "sqlbuild-render",
     "sqlbuild-analysis",
     "sqlbuild-rules",
     "sqlbuild-python",
@@ -61,6 +65,7 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0a
 polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0af282829d92efeeb36b6a820be2271e1da49", default-features = false, features = [
     "dialect-duckdb",
 ] }
+blake3 = { version = "1", features = ["rayon"] }
 quote = "1"
 rayon = "1"
 saphyr-parser = "0.1"
@@ -70,9 +75,11 @@ serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["preserve_order"] }
 sha2 = "0.10"
 sqlbuild-analysis = { path = "crates/sqlbuild-analysis" }
+sqlbuild-cache = { path = "crates/sqlbuild-cache" }
 sqlbuild-config = { path = "crates/sqlbuild-config" }
 sqlbuild-core = { path = "crates/sqlbuild-core" }
 sqlbuild-discovery = { path = "crates/sqlbuild-discovery" }
+sqlbuild-render = { path = "crates/sqlbuild-render" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }
 sqlbuild-model-config = { path = "crates/sqlbuild-model-config" }
 sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }

crates/README.md

@@ -7,11 +7,13 @@ that can be tested and benchmarked with `cargo test` alone.
 | Crate | Responsibility |
 |---|---|
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. |
+| `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. |
 | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. |
 | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. |
 | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. |
 | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. |
 | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. |
+| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. |
 | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. |
 | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. |
 | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. |
@@ -22,8 +24,9 @@ Dependencies point one way, from the top of this graph to the bottom:
 
 ```text
 sqlbuild-python
-  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-scopes
-  -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core
+  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-render
+  -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache
+  -> sqlbuild-core
 ```
 
 `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and

crates/sqlbuild-cache/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "sqlbuild-cache"
+version.workspace = true
+description = "The shared native store and fingerprinting scheme for SQLBuild's compiler caches."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+blake3.workspace = true
+rayon.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+walkdir.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-cache/src/digest/_helpers/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod walk;

crates/sqlbuild-cache/src/digest/_helpers/walk.rs

@@ -0,0 +1,72 @@
+//! List the project files a fingerprint covers, in a stable order.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+use walkdir::{DirEntry, WalkDir};
+
+use crate::digest::constants::{
+    EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES,
+};
+use crate::digest::errors::FingerprintError;
+
+/// One covered file: its project-relative path, location and whether only presence counts.
+pub(crate) struct CoveredFile {
+    pub(crate) relative_path: String,
+    pub(crate) path: PathBuf,
+    pub(crate) presence_only: bool,
+}
+
+/// Every file below `root` outside the excluded folders and `excluded_files`, links followed.
+pub(crate) fn covered_files(
+    root: &Path,
+    excluded_files: &HashSet<String>,
+) -> Result<Vec<CoveredFile>, FingerprintError> {
+    let mut files: Vec<CoveredFile> = Vec::new();
+    let walk = WalkDir::new(root)
+        .follow_links(true)
+        .sort_by_file_name()
+        .into_iter()
+        .filter_entry(|entry| !excluded_directory(entry));
+    for entry in walk {
+        let entry: DirEntry = entry.map_err(FingerprintError::Walk)?;
+        if !entry.file_type().is_file() {
+            continue;
+        }
+        let relative_path: String = relative_text(root, entry.path())?;
+        if excluded_files.contains(&relative_path) {
+            continue;
+        }
+        let presence_only: bool = PRESENCE_ONLY_FILE_SUFFIXES
+            .iter()
+            .any(|suffix| relative_path.ends_with(suffix));
+        files.push(CoveredFile {
+            relative_path,
+            path: entry.into_path(),
+            presence_only,
+        });
+    }
+    Ok(files)
+}
+
+fn excluded_directory(entry: &DirEntry) -> bool {
+    if !entry.file_type().is_dir() || entry.depth() == 0 {
+        return false;
+    }
+    let Some(name) = entry.file_name().to_str() else {
+        return false;
+    };
+    EXCLUDED_DIRECTORIES.contains(&name)
+        || (entry.depth() == 1 && EXCLUDED_ROOT_DIRECTORIES.contains(&name))
+}
+
+fn relative_text(root: &Path, path: &Path) -> Result<String, FingerprintError> {
+    let relative: &Path = path
+        .strip_prefix(root)
+        .map_err(|_| FingerprintError::OutsideRoot(path.to_path_buf()))?;
+    let non_utf8 = || FingerprintError::NonUtf8Path(path.to_path_buf());
+    let parts: Vec<&str> = relative
+        .components()
+        .map(|component| component.as_os_str().to_str().ok_or_else(non_utf8))
+        .collect::<Result<Vec<&str>, FingerprintError>>()?;
+    Ok(parts.join("/"))
+}

crates/sqlbuild-cache/src/digest/constants.rs

@@ -0,0 +1,32 @@
+//! What the project fingerprint walk skips; the same folders project reuse leaves out.
+
+/// Folders skipped only directly below the project root.
+pub const EXCLUDED_ROOT_DIRECTORIES: &[&str] = &[
+    "logs",
+    "target",
+    "venv",
+    ".cache",
+    ".fensu",
+    ".hg",
+    ".idea",
+    ".mypy_cache",
+    ".nox",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".sqlbuild",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+];
+
+/// Folders skipped at any depth.
+pub const EXCLUDED_DIRECTORIES: &[&str] = &[".git", "__pycache__"];
+
+/// Files that count only by presence because every build rewrites them.
+pub const PRESENCE_ONLY_FILE_SUFFIXES: &[&str] = &[".duckdb", ".duckdb.wal"];
+
+/// Digest tag of a file whose bytes count.
+pub const CONTENT_TAG: &[u8] = b"content";
+/// Digest tag of a file that counts only by presence.
+pub const PRESENCE_TAG: &[u8] = b"presence";

crates/sqlbuild-cache/src/digest/errors.rs

@@ -0,0 +1,25 @@
+//! Why a project fingerprint could not cover every file.
+
+use std::fmt;
+
+/// A file the fingerprint must cover could not be listed or read.
+#[derive(Debug)]
+pub enum FingerprintError {
+    Walk(walkdir::Error),
+    NonUtf8Path(std::path::PathBuf),
+    OutsideRoot(std::path::PathBuf),
+    Read(std::path::PathBuf, std::io::Error),
+}
+
+impl fmt::Display for FingerprintError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Walk(error) => write!(formatter, "cannot list project files: {error}"),
+            Self::NonUtf8Path(path) => write!(formatter, "path is not UTF-8: {}", path.display()),
+            Self::OutsideRoot(path) => {
+                write!(formatter, "path is outside the project: {}", path.display())
+            }
+            Self::Read(path, error) => write!(formatter, "cannot read {}: {error}", path.display()),
+        }
+    }
+}