diff --git a/Cargo.lock b/Cargo.lock index ff615f69eb..03a1b44287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "autocfg" version = "1.5.1" @@ -38,6 +44,20 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", + "rayon-core", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -83,6 +103,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -92,6 +118,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -561,6 +596,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "saphyr-parser" version = "0.1.0" @@ -637,7 +681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -662,6 +706,17 @@ dependencies = [ "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-cache" +version = "0.139.0" +dependencies = [ + "blake3", + "rayon", + "sha2", + "tempfile", + "walkdir", +] + [[package]] name = "sqlbuild-config" version = "0.139.0" @@ -701,15 +756,26 @@ dependencies = [ "pyo3", "serde_json", "sqlbuild-analysis", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", + "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-render" +version = "0.139.0" +dependencies = [ + "sqlbuild-cache", + "sqlbuild-core", + "tempfile", +] + [[package]] name = "sqlbuild-rules" version = "0.139.0" @@ -954,6 +1020,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "winapi" version = "0.3.9" @@ -970,6 +1046,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 9b4efa3107..2f610cd372 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,13 @@ [workspace] members = [ "crates/sqlbuild-core", + "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", + "crates/sqlbuild-render", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -18,11 +20,13 @@ resolver = "2" [workspace.metadata.native-layers] order = [ "sqlbuild-core", + "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", "sqlbuild-discovery", "sqlbuild-scopes", "sqlbuild-model-config", + "sqlbuild-render", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -61,6 +65,7 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0a polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0af282829d92efeeb36b6a820be2271e1da49", default-features = false, features = [ "dialect-duckdb", ] } +blake3 = { version = "1", features = ["rayon"] } quote = "1" rayon = "1" saphyr-parser = "0.1" @@ -70,9 +75,11 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-cache = { path = "crates/sqlbuild-cache" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } +sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index 6993d60495..40d973f153 100644 --- a/crates/README.md +++ b/crates/README.md @@ -7,11 +7,13 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | +| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -22,8 +24,9 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-scopes - -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-render + -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache + -> sqlbuild-core ``` `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and diff --git a/crates/sqlbuild-cache/Cargo.toml b/crates/sqlbuild-cache/Cargo.toml new file mode 100644 index 0000000000..c24885ac4b --- /dev/null +++ b/crates/sqlbuild-cache/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sqlbuild-cache" +version.workspace = true +description = "The shared native store and fingerprinting scheme for SQLBuild's compiler caches." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +blake3.workspace = true +rayon.workspace = true +sha2.workspace = true +tempfile.workspace = true +walkdir.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-cache/src/digest/_helpers/mod.rs b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs new file mode 100644 index 0000000000..208de6bb67 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod walk; diff --git a/crates/sqlbuild-cache/src/digest/_helpers/walk.rs b/crates/sqlbuild-cache/src/digest/_helpers/walk.rs new file mode 100644 index 0000000000..af89bd61eb --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/_helpers/walk.rs @@ -0,0 +1,72 @@ +//! List the project files a fingerprint covers, in a stable order. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use walkdir::{DirEntry, WalkDir}; + +use crate::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use crate::digest::errors::FingerprintError; + +/// One covered file: its project-relative path, location and whether only presence counts. +pub(crate) struct CoveredFile { + pub(crate) relative_path: String, + pub(crate) path: PathBuf, + pub(crate) presence_only: bool, +} + +/// Every file below `root` outside the excluded folders and `excluded_files`, links followed. +pub(crate) fn covered_files( + root: &Path, + excluded_files: &HashSet, +) -> Result, FingerprintError> { + let mut files: Vec = Vec::new(); + let walk = WalkDir::new(root) + .follow_links(true) + .sort_by_file_name() + .into_iter() + .filter_entry(|entry| !excluded_directory(entry)); + for entry in walk { + let entry: DirEntry = entry.map_err(FingerprintError::Walk)?; + if !entry.file_type().is_file() { + continue; + } + let relative_path: String = relative_text(root, entry.path())?; + if excluded_files.contains(&relative_path) { + continue; + } + let presence_only: bool = PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .any(|suffix| relative_path.ends_with(suffix)); + files.push(CoveredFile { + relative_path, + path: entry.into_path(), + presence_only, + }); + } + Ok(files) +} + +fn excluded_directory(entry: &DirEntry) -> bool { + if !entry.file_type().is_dir() || entry.depth() == 0 { + return false; + } + let Some(name) = entry.file_name().to_str() else { + return false; + }; + EXCLUDED_DIRECTORIES.contains(&name) + || (entry.depth() == 1 && EXCLUDED_ROOT_DIRECTORIES.contains(&name)) +} + +fn relative_text(root: &Path, path: &Path) -> Result { + let relative: &Path = path + .strip_prefix(root) + .map_err(|_| FingerprintError::OutsideRoot(path.to_path_buf()))?; + let non_utf8 = || FingerprintError::NonUtf8Path(path.to_path_buf()); + let parts: Vec<&str> = relative + .components() + .map(|component| component.as_os_str().to_str().ok_or_else(non_utf8)) + .collect::, FingerprintError>>()?; + Ok(parts.join("/")) +} diff --git a/crates/sqlbuild-cache/src/digest/constants.rs b/crates/sqlbuild-cache/src/digest/constants.rs new file mode 100644 index 0000000000..d0a53f5c3a --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/constants.rs @@ -0,0 +1,32 @@ +//! What the project fingerprint walk skips; the same folders project reuse leaves out. + +/// Folders skipped only directly below the project root. +pub const EXCLUDED_ROOT_DIRECTORIES: &[&str] = &[ + "logs", + "target", + "venv", + ".cache", + ".fensu", + ".hg", + ".idea", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".sqlbuild", + ".svn", + ".tox", + ".venv", + ".vscode", +]; + +/// Folders skipped at any depth. +pub const EXCLUDED_DIRECTORIES: &[&str] = &[".git", "__pycache__"]; + +/// Files that count only by presence because every build rewrites them. +pub const PRESENCE_ONLY_FILE_SUFFIXES: &[&str] = &[".duckdb", ".duckdb.wal"]; + +/// Digest tag of a file whose bytes count. +pub const CONTENT_TAG: &[u8] = b"content"; +/// Digest tag of a file that counts only by presence. +pub const PRESENCE_TAG: &[u8] = b"presence"; diff --git a/crates/sqlbuild-cache/src/digest/errors.rs b/crates/sqlbuild-cache/src/digest/errors.rs new file mode 100644 index 0000000000..87a9b1757b --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/errors.rs @@ -0,0 +1,25 @@ +//! Why a project fingerprint could not cover every file. + +use std::fmt; + +/// A file the fingerprint must cover could not be listed or read. +#[derive(Debug)] +pub enum FingerprintError { + Walk(walkdir::Error), + NonUtf8Path(std::path::PathBuf), + OutsideRoot(std::path::PathBuf), + Read(std::path::PathBuf, std::io::Error), +} + +impl fmt::Display for FingerprintError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Walk(error) => write!(formatter, "cannot list project files: {error}"), + Self::NonUtf8Path(path) => write!(formatter, "path is not UTF-8: {}", path.display()), + Self::OutsideRoot(path) => { + write!(formatter, "path is outside the project: {}", path.display()) + } + Self::Read(path, error) => write!(formatter, "cannot read {}: {error}", path.display()), + } + } +} diff --git a/crates/sqlbuild-cache/src/digest/main/content_digest.rs b/crates/sqlbuild-cache/src/digest/main/content_digest.rs new file mode 100644 index 0000000000..d7076d9eba --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/content_digest.rs @@ -0,0 +1,16 @@ +//! The one digest every native cache key is built from. + +use sha2::{Digest, Sha256}; + +use crate::digest::types::ContentDigest; + +/// Digest `parts` with each part length-prefixed, so part boundaries cannot shift. +pub fn content_digest>(parts: &[T]) -> ContentDigest { + let mut hasher: Sha256 = Sha256::new(); + for part in parts { + let bytes: &[u8] = part.as_ref(); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + hasher.finalize().into() +} diff --git a/crates/sqlbuild-cache/src/digest/main/digest_files.rs b/crates/sqlbuild-cache/src/digest/main/digest_files.rs new file mode 100644 index 0000000000..a659f8c0b0 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/digest_files.rs @@ -0,0 +1,18 @@ +//! BLAKE3 content digests of individual files, read and hashed in parallel. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use std::path::PathBuf; + +use crate::digest::types::ContentDigest; + +/// The digest of each file in `paths`; BLAKE3 keeps large compiled modules cheap to hash. +pub fn digest_files(paths: &[PathBuf]) -> Vec> { + paths.par_iter().map(file_digest).collect() +} + +fn file_digest(path: &PathBuf) -> std::io::Result { + let bytes: Vec = std::fs::read(path)?; + let mut hasher = blake3::Hasher::new(); + let _ = hasher.update_rayon(&bytes); + Ok(*hasher.finalize().as_bytes()) +} diff --git a/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs b/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs new file mode 100644 index 0000000000..9c544f71cf --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs @@ -0,0 +1,38 @@ +//! Fingerprint the content of a project's files, leaving out the files a cache key covers itself. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::path::Path; + +use crate::digest::_helpers::walk::{CoveredFile, covered_files}; +use crate::digest::constants::{CONTENT_TAG, PRESENCE_TAG}; +use crate::digest::errors::FingerprintError; +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; + +/// Digest every covered file's project-relative path and bytes; fails if any file is unreadable. +pub fn fingerprint_project_files( + root: &Path, + excluded_files: &HashSet, +) -> Result { + let files: Vec = covered_files(root, excluded_files)?; + let digests: Vec = files + .par_iter() + .map(file_digest) + .collect::, FingerprintError>>()?; + let mut hasher: Sha256 = Sha256::new(); + for (file, digest) in files.iter().zip(digests) { + hasher.update(content_digest(&[file.relative_path.as_bytes(), &digest])); + } + Ok(hasher.finalize().into()) +} + +fn file_digest(file: &CoveredFile) -> Result { + if file.presence_only { + return Ok(content_digest(&[PRESENCE_TAG])); + } + let bytes: Vec = std::fs::read(&file.path) + .map_err(|error| FingerprintError::Read(file.path.clone(), error))?; + Ok(content_digest(&[CONTENT_TAG, &bytes])) +} diff --git a/crates/sqlbuild-cache/src/digest/main/hex_digest.rs b/crates/sqlbuild-cache/src/digest/main/hex_digest.rs new file mode 100644 index 0000000000..1ef1c12e40 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/hex_digest.rs @@ -0,0 +1,8 @@ +//! Text form of a content digest. + +use crate::digest::types::ContentDigest; + +/// Lowercase hexadecimal text of a digest. +pub fn hex_digest(digest: &ContentDigest) -> String { + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs new file mode 100644 index 0000000000..c738ef926d --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -0,0 +1,4 @@ +pub mod content_digest; +pub mod digest_files; +pub mod fingerprint_project_files; +pub mod hex_digest; diff --git a/crates/sqlbuild-cache/src/digest/mod.rs b/crates/sqlbuild-cache/src/digest/mod.rs new file mode 100644 index 0000000000..49b58ce941 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/digest/tests.rs b/crates/sqlbuild-cache/src/digest/tests.rs new file mode 100644 index 0000000000..7ab98d75a0 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/test_content_digest.rs"] +mod content_digest; +#[path = "tests/test_digest_files.rs"] +mod digest_files; +#[path = "tests/test_fingerprint_project_files.rs"] +mod fingerprint_project_files; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/digest/tests/helpers.rs b/crates/sqlbuild-cache/src/digest/tests/helpers.rs new file mode 100644 index 0000000000..179519088f --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/helpers.rs @@ -0,0 +1,29 @@ +use std::collections::HashSet; +use std::path::Path; + +/// The model file the fingerprint test project leaves to the cache key. +pub(crate) const EXCLUDED_MODEL: &str = "models/orders.sql"; + +/// Write a small project: config, a macro, a model and files in skipped folders. +pub(crate) fn write_project(root: &Path) { + for (relative_path, contents) in [ + ("sqlbuild_project.toml", "name = \"orders\"\n"), + ("macros/cents.py", "def cents(amount):\n return amount\n"), + (EXCLUDED_MODEL, "SELECT 1 AS id\n"), + ("target/cache/store.bin", "old"), + ("macros/__pycache__/cents.pyc", "bytecode"), + ("warehouse.duckdb", "pages"), + ] { + write_file(root, relative_path, contents); + } +} + +pub(crate) fn write_file(root: &Path, relative_path: &str, contents: &str) { + let path = root.join(relative_path); + std::fs::create_dir_all(path.parent().expect("parent")).expect("parent folder"); + std::fs::write(path, contents).expect("project file"); +} + +pub(crate) fn excluded_files() -> HashSet { + HashSet::from([EXCLUDED_MODEL.to_owned()]) +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs b/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs new file mode 100644 index 0000000000..e55864a887 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs @@ -0,0 +1,69 @@ +use crate::digest::main::content_digest::content_digest; +use crate::digest::main::hex_digest::hex_digest; +use crate::digest::tests::test_types::{ContentDigestTestCase, HexDigestTestCase}; + +#[test] +fn given_parts_when_digesting_then_only_identical_parts_match() { + let test_cases = [ + ContentDigestTestCase { + description: "identical parts", + left: &["cents", "@cents(amount)"], + right: &["cents", "@cents(amount)"], + expected_equal: true, + }, + ContentDigestTestCase { + description: "a boundary moved between parts", + left: &["cents", "@cents(amount)"], + right: &["cents@", "cents(amount)"], + expected_equal: false, + }, + ContentDigestTestCase { + description: "an empty part added", + left: &["cents"], + right: &["cents", ""], + expected_equal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + content_digest(test_case.left) == content_digest(test_case.right), + test_case.expected_equal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_digest_when_formatting_then_text_is_lowercase_hexadecimal() { + let test_cases = [ + HexDigestTestCase { + description: "one part", + parts: &["orders"], + expected_length: 64, + }, + HexDigestTestCase { + description: "no parts", + parts: &[], + expected_length: 64, + }, + ]; + + for test_case in test_cases { + let text = hex_digest(&content_digest(test_case.parts)); + + assert_eq!( + text.len(), + test_case.expected_length, + "{}", + test_case.description + ); + assert!( + text.chars() + .all(|character| matches!(character, '0'..='9' | 'a'..='f')), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs b/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs new file mode 100644 index 0000000000..2497649562 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use crate::digest::main::digest_files::digest_files; +use crate::digest::tests::helpers::write_file; +use crate::digest::tests::test_types::DigestFilesTestCase; + +#[test] +fn given_paths_when_digesting_files_then_readable_files_digest_their_bytes() { + let test_cases = [ + DigestFilesTestCase { + description: "regular file", + path: |root| root.join("macros/cents.py"), + expected_digest_of: Some("def cents(amount):\n return amount\n"), + }, + DigestFilesTestCase { + description: "missing file", + path: |root| root.join("macros/missing.py"), + expected_digest_of: None, + }, + DigestFilesTestCase { + description: "member path inside an archive", + path: |root| root.join("macros/cents.py/inner.py"), + expected_digest_of: None, + }, + DigestFilesTestCase { + description: "directory", + path: |root| root.join("macros"), + expected_digest_of: None, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_file( + project.path(), + "macros/cents.py", + "def cents(amount):\n return amount\n", + ); + let path: PathBuf = (test_case.path)(project.path()); + + let digests: Vec> = + digest_files(&[path]).into_iter().map(Result::ok).collect(); + + assert_eq!( + digests, + vec![ + test_case + .expected_digest_of + .map(|text| *blake3::hash(text.as_bytes()).as_bytes()) + ], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs b/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs new file mode 100644 index 0000000000..c6d80d70fd --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs @@ -0,0 +1,124 @@ +use std::path::Path; + +use crate::digest::main::fingerprint_project_files::fingerprint_project_files; +use crate::digest::tests::helpers::{EXCLUDED_MODEL, excluded_files, write_file, write_project}; +use crate::digest::tests::test_types::{ + FingerprintFailureTestCase, FingerprintProjectFilesTestCase, +}; + +#[test] +fn given_project_edit_when_fingerprinting_then_only_covered_content_changes_it() { + let test_cases = [ + FingerprintProjectFilesTestCase { + description: "no edit", + edit: |_| {}, + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "macro body edit", + edit: |root| { + write_file( + root, + "macros/cents.py", + "def cents(amount):\n return 1\n", + ) + }, + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "helper module added", + edit: |root| write_file(root, "macros/_rounding.py", "SCALE = 2\n"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "macro file removed", + edit: |root| std::fs::remove_file(root.join("macros/cents.py")).expect("removed"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "file renamed with the same content", + edit: |root| { + std::fs::rename(root.join("macros/cents.py"), root.join("macros/money.py")) + .expect("renamed"); + }, + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "excluded model edit", + edit: |root| write_file(root, EXCLUDED_MODEL, "SELECT 2 AS id\n"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "new model is covered until the caller excludes it", + edit: |root| write_file(root, "models/customers.sql", "SELECT 1 AS id\n"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "root target folder edit", + edit: |root| write_file(root, "target/cache/store.bin", "new"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "nested target folder edit", + edit: |root| write_file(root, "macros/target/notes.txt", "covered"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "bytecode cache edit", + edit: |root| write_file(root, "macros/__pycache__/cents.pyc", "other"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "database pages rewritten", + edit: |root| write_file(root, "warehouse.duckdb", "other pages"), + expected_changed: false, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_project(project.path()); + let before = fingerprint_project_files(project.path(), &excluded_files()); + (test_case.edit)(project.path()); + let after = fingerprint_project_files(project.path(), &excluded_files()); + + assert!(before.is_ok(), "{}", test_case.description); + assert_eq!( + before.ok() != after.ok(), + test_case.expected_changed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unlistable_root_when_fingerprinting_then_it_fails() { + let test_cases = [ + FingerprintFailureTestCase { + description: "existing project", + root: Path::to_path_buf, + expected_failure: false, + }, + FingerprintFailureTestCase { + description: "missing project", + root: |root| root.join("missing"), + expected_failure: true, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_project(project.path()); + + let fingerprint = + fingerprint_project_files(&(test_case.root)(project.path()), &excluded_files()); + + assert_eq!( + fingerprint.is_err(), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_types.rs b/crates/sqlbuild-cache/src/digest/tests/test_types.rs new file mode 100644 index 0000000000..3ecb41715d --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_types.rs @@ -0,0 +1,35 @@ +use std::path::{Path, PathBuf}; + +pub(crate) struct ContentDigestTestCase { + pub(crate) description: &'static str, + pub(crate) left: &'static [&'static str], + pub(crate) right: &'static [&'static str], + pub(crate) expected_equal: bool, +} + +pub(crate) struct HexDigestTestCase { + pub(crate) description: &'static str, + pub(crate) parts: &'static [&'static str], + pub(crate) expected_length: usize, +} + +/// An edit applied to a fresh copy of the fingerprint test project. +pub(crate) type ProjectEdit = fn(&Path); + +pub(crate) struct FingerprintProjectFilesTestCase { + pub(crate) description: &'static str, + pub(crate) edit: ProjectEdit, + pub(crate) expected_changed: bool, +} + +pub(crate) struct FingerprintFailureTestCase { + pub(crate) description: &'static str, + pub(crate) root: fn(&Path) -> PathBuf, + pub(crate) expected_failure: bool, +} + +pub(crate) struct DigestFilesTestCase { + pub(crate) description: &'static str, + pub(crate) path: fn(&Path) -> PathBuf, + pub(crate) expected_digest_of: Option<&'static str>, +} diff --git a/crates/sqlbuild-cache/src/digest/types.rs b/crates/sqlbuild-cache/src/digest/types.rs new file mode 100644 index 0000000000..bab2f3dc57 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/types.rs @@ -0,0 +1,4 @@ +//! Digest values shared by every native cache. + +/// A SHA-256 content digest. +pub type ContentDigest = [u8; 32]; diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs new file mode 100644 index 0000000000..c1e3b017ca --- /dev/null +++ b/crates/sqlbuild-cache/src/lib.rs @@ -0,0 +1,6 @@ +//! The shared native cache store and the content fingerprints its keys are built from. + +#![forbid(unsafe_code)] + +pub mod digest; +pub mod store; diff --git a/crates/sqlbuild-cache/src/store/_helpers/frame.rs b/crates/sqlbuild-cache/src/store/_helpers/frame.rs new file mode 100644 index 0000000000..91e8028df9 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/_helpers/frame.rs @@ -0,0 +1,74 @@ +//! Encode and verify whole store files. + +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; +use crate::store::constants::{STORE_FORMAT_VERSION, STORE_MAGIC}; +use crate::store::errors::StoreDecodeError; +use crate::store::models::{RecordReader, RecordWriter}; + +/// The decoded contents of one store file. +pub(crate) struct StoreFile { + pub(crate) kind: String, + pub(crate) environment: String, + pub(crate) generation: u64, + pub(crate) metadata: Vec, + pub(crate) entries: Vec<(ContentDigest, u64, Vec)>, +} + +/// Magic, format version, a digest of the body, then the body. +pub(crate) fn encode_store(file: &StoreFile) -> Vec { + let mut body: RecordWriter = RecordWriter::default(); + body.put_str(&file.kind); + body.put_str(&file.environment); + body.put_u64(file.generation); + body.put_bytes(&file.metadata); + body.put_u64(file.entries.len() as u64); + for (key, last_used, value) in &file.entries { + body.put_bytes(key); + body.put_u64(*last_used); + body.put_bytes(value); + } + let body: Vec = body.into_bytes(); + let mut bytes: Vec = Vec::with_capacity(body.len() + 44); + bytes.extend_from_slice(STORE_MAGIC); + bytes.extend_from_slice(&STORE_FORMAT_VERSION.to_le_bytes()); + bytes.extend_from_slice(&content_digest(&[&body])); + bytes.extend_from_slice(&body); + bytes +} + +/// The file's contents; another format, a damaged body or trailing bytes are decode errors. +pub(crate) fn decode_store(bytes: &[u8]) -> Result { + let rest: &[u8] = bytes + .strip_prefix(STORE_MAGIC.as_slice()) + .ok_or(StoreDecodeError)?; + let (version, rest) = rest.split_first_chunk::<4>().ok_or(StoreDecodeError)?; + if u32::from_le_bytes(*version) != STORE_FORMAT_VERSION { + return Err(StoreDecodeError); + } + let (checksum, body) = rest.split_first_chunk::<32>().ok_or(StoreDecodeError)?; + if content_digest(&[body]) != *checksum { + return Err(StoreDecodeError); + } + let mut reader: RecordReader<'_> = RecordReader::new(body); + let kind: String = reader.string()?; + let environment: String = reader.string()?; + let generation: u64 = reader.u64()?; + let metadata: Vec = reader.bytes()?.to_vec(); + let count: u64 = reader.u64()?; + let mut entries: Vec<(ContentDigest, u64, Vec)> = Vec::new(); + for _ in 0..count { + let key: ContentDigest = reader.bytes()?.try_into().map_err(|_| StoreDecodeError)?; + let last_used: u64 = reader.u64()?; + let value: Vec = reader.bytes()?.to_vec(); + entries.push((key, last_used, value)); + } + reader.finish()?; + Ok(StoreFile { + kind, + environment, + generation, + metadata, + entries, + }) +} diff --git a/crates/sqlbuild-cache/src/store/_helpers/mod.rs b/crates/sqlbuild-cache/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..70756e496c --- /dev/null +++ b/crates/sqlbuild-cache/src/store/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod frame; diff --git a/crates/sqlbuild-cache/src/store/constants.rs b/crates/sqlbuild-cache/src/store/constants.rs new file mode 100644 index 0000000000..ffda20c06d --- /dev/null +++ b/crates/sqlbuild-cache/src/store/constants.rs @@ -0,0 +1,8 @@ +//! The native store file format. + +/// The first bytes of every store file. +pub const STORE_MAGIC: &[u8; 8] = b"SQBSTORE"; +/// Bumped whenever the file layout changes; older files are discarded whole. +pub const STORE_FORMAT_VERSION: u32 = 1; +/// Saves an unused entry survives while the store's environment is unchanged. +pub const RETAINED_GENERATIONS: u64 = 64; diff --git a/crates/sqlbuild-cache/src/store/errors.rs b/crates/sqlbuild-cache/src/store/errors.rs new file mode 100644 index 0000000000..c055f393e3 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/errors.rs @@ -0,0 +1,5 @@ +//! Why stored bytes could not be read back. + +/// Stored bytes end early, carry unknown fields or were written by another format. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreDecodeError; diff --git a/crates/sqlbuild-cache/src/store/main/mod.rs b/crates/sqlbuild-cache/src/store/main/mod.rs new file mode 100644 index 0000000000..4960dcf371 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/main/mod.rs @@ -0,0 +1 @@ +pub mod open_native_store; diff --git a/crates/sqlbuild-cache/src/store/main/open_native_store.rs b/crates/sqlbuild-cache/src/store/main/open_native_store.rs new file mode 100644 index 0000000000..5abbecd712 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/main/open_native_store.rs @@ -0,0 +1,46 @@ +//! Open one cache kind's store file. + +use std::collections::HashMap; +use std::path::Path; + +use crate::store::_helpers::frame::{StoreFile, decode_store}; +use crate::store::errors::StoreDecodeError; +use crate::store::models::{NativeStore, StoredValue}; + +/// Load `kind` entries for exactly `environment`; missing, damaged or foreign files open empty. +pub fn open_native_store( + path: &Path, + kind: &str, + environment: &str, +) -> std::io::Result { + let mut store: NativeStore = NativeStore { + kind: kind.to_owned(), + environment: environment.to_owned(), + ..NativeStore::default() + }; + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(store), + Err(error) => return Err(error), + }; + let file: StoreFile = match decode_store(&bytes) { + Ok(file) if file.kind == kind && file.environment == environment => file, + Ok(_) | Err(StoreDecodeError) => return Ok(store), + }; + store.generation = file.generation; + store.metadata = file.metadata; + store.loaded = file.entries.len(); + store.entries = file + .entries + .into_iter() + .map(|(key, last_used, value)| { + let stored: StoredValue = StoredValue { + value, + last_used, + used: false, + }; + (key, stored) + }) + .collect::>(); + Ok(store) +} diff --git a/crates/sqlbuild-cache/src/store/mod.rs b/crates/sqlbuild-cache/src/store/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-cache/src/store/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/store/models.rs b/crates/sqlbuild-cache/src/store/models.rs new file mode 100644 index 0000000000..0b662e9704 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/models.rs @@ -0,0 +1,175 @@ +//! The shared native store: one file per cache kind, keyed by content digests. + +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use crate::digest::types::ContentDigest; +use crate::store::_helpers::frame::{StoreFile, encode_store}; +use crate::store::constants::RETAINED_GENERATIONS; +use crate::store::errors::StoreDecodeError; + +/// Length-prefixed fields of one stored value. +#[derive(Debug, Default)] +pub struct RecordWriter { + bytes: Vec, +} + +impl RecordWriter { + pub fn put_u64(&mut self, value: u64) { + self.bytes.extend_from_slice(&value.to_le_bytes()); + } + + pub fn put_bytes(&mut self, value: &[u8]) { + self.put_u64(value.len() as u64); + self.bytes.extend_from_slice(value); + } + + pub fn put_str(&mut self, value: &str) { + self.put_bytes(value.as_bytes()); + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// Reads the fields a [`RecordWriter`] wrote; reading past the end is a decode error. +#[derive(Debug)] +pub struct RecordReader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> RecordReader<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + pub fn u64(&mut self) -> Result { + let raw: [u8; 8] = self.take(8)?.try_into().map_err(|_| StoreDecodeError)?; + Ok(u64::from_le_bytes(raw)) + } + + pub fn bytes(&mut self) -> Result<&'a [u8], StoreDecodeError> { + let length: usize = usize::try_from(self.u64()?).map_err(|_| StoreDecodeError)?; + self.take(length) + } + + pub fn string(&mut self) -> Result { + String::from_utf8(self.bytes()?.to_vec()).map_err(|_| StoreDecodeError) + } + + pub fn finish(&self) -> Result<(), StoreDecodeError> { + if self.position == self.bytes.len() { + Ok(()) + } else { + Err(StoreDecodeError) + } + } + + fn take(&mut self, length: usize) -> Result<&'a [u8], StoreDecodeError> { + let end: usize = self.position.checked_add(length).ok_or(StoreDecodeError)?; + let taken: &'a [u8] = self.bytes.get(self.position..end).ok_or(StoreDecodeError)?; + self.position = end; + Ok(taken) + } +} + +/// One stored value and the save generation that last used it. +#[derive(Clone, Debug)] +pub(crate) struct StoredValue { + pub(crate) value: Vec, + pub(crate) last_used: u64, + pub(crate) used: bool, +} + +/// One cache kind's entries for one environment; entries are immutable once stored. +#[derive(Debug, Default)] +pub struct NativeStore { + pub(crate) kind: String, + pub(crate) environment: String, + pub(crate) generation: u64, + pub(crate) metadata: Vec, + pub(crate) entries: HashMap, + pub(crate) loaded: usize, + pub(crate) changed: bool, +} + +impl NativeStore { + /// The caller-owned validation data saved with the loaded entries; empty when none loaded. + pub fn metadata(&self) -> &[u8] { + &self.metadata + } + + /// How many entries the store opened with. + pub fn loaded_entries(&self) -> usize { + self.loaded + } + + /// Drop every loaded entry, for a caller whose metadata check failed. + pub fn discard(&mut self) { + self.entries.clear(); + self.metadata.clear(); + self.loaded = 0; + self.generation = 0; + } + + /// The value stored under `key`, which the next save keeps. + pub fn get(&mut self, key: &ContentDigest) -> Option<&[u8]> { + let stored: &mut StoredValue = self.entries.get_mut(key)?; + stored.used = true; + Some(&stored.value) + } + + /// Store a value computed in this process. + pub fn put(&mut self, key: ContentDigest, value: Vec) { + let stored: StoredValue = StoredValue { + value, + last_used: self.generation, + used: true, + }; + let _ = self.entries.insert(key, stored); + self.changed = true; + } + + /// Whether this process stored a value the file does not hold yet. + pub fn needs_save(&self) -> bool { + self.changed + } + + /// Atomically replace the file with used and recently used entries; returns entries written. + pub fn save(&self, path: &Path, metadata: &[u8]) -> std::io::Result { + let generation: u64 = self.generation + 1; + let mut entries: Vec<(ContentDigest, u64, Vec)> = self + .entries + .iter() + .filter(|(_, stored)| { + stored.used || generation - stored.last_used <= RETAINED_GENERATIONS + }) + .map(|(key, stored)| { + let last_used: u64 = if stored.used { + generation + } else { + stored.last_used + }; + (*key, last_used, stored.value.clone()) + }) + .collect(); + entries.sort_unstable_by_key(|(key, _, _)| *key); + let written: usize = entries.len(); + let bytes: Vec = encode_store(&StoreFile { + kind: self.kind.clone(), + environment: self.environment.clone(), + generation, + metadata: metadata.to_vec(), + entries, + }); + let directory: &Path = path.parent().unwrap_or_else(|| Path::new(".")); + std::fs::create_dir_all(directory)?; + let mut file: tempfile::NamedTempFile = tempfile::NamedTempFile::new_in(directory)?; + file.write_all(&bytes)?; + let _ = file.persist(path).map_err(|error| error.error)?; + Ok(written) + } +} diff --git a/crates/sqlbuild-cache/src/store/tests.rs b/crates/sqlbuild-cache/src/store/tests.rs new file mode 100644 index 0000000000..7b76c040a2 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_native_store.rs"] +mod native_store; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/store/tests/helpers.rs b/crates/sqlbuild-cache/src/store/tests/helpers.rs new file mode 100644 index 0000000000..044e993772 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/helpers.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; +use crate::store::errors::StoreDecodeError; +use crate::store::main::open_native_store::open_native_store; +use crate::store::models::{NativeStore, RecordReader, RecordWriter}; + +pub(crate) const KIND: &str = "macro-calls"; +pub(crate) const ENVIRONMENT: &str = "environment-a"; +pub(crate) const METADATA: &[u8] = b"module stamps"; +pub(crate) const VALUE: &[u8] = b"amount * 100"; + +pub(crate) fn key(text: &str) -> ContentDigest { + content_digest(&[text]) +} + +pub(crate) fn open(path: &Path) -> NativeStore { + open_native_store(path, KIND, ENVIRONMENT).expect("opened store") +} + +/// Save a store holding `VALUE` under the `cents` key. +pub(crate) fn saved_store(path: &Path) { + let mut store = open(path); + store.put(key("cents"), VALUE.to_vec()); + assert_eq!(store.save(path, METADATA).expect("saved"), 1); +} + +/// Flip one byte near the end of the file. +pub(crate) fn damage(path: &Path) { + let mut bytes = std::fs::read(path).expect("store bytes"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(path, bytes).expect("damaged store"); +} + +/// A record holding a string and a number. +pub(crate) fn record() -> Vec { + let mut writer = RecordWriter::default(); + writer.put_str("café"); + writer.put_u64(7); + writer.into_bytes() +} + +/// Read a string and a number, then require the end of the record. +pub(crate) fn read_record(bytes: &[u8]) -> Result<(String, u64), StoreDecodeError> { + let mut reader = RecordReader::new(bytes); + let text = reader.string()?; + let number = reader.u64()?; + reader.finish()?; + Ok((text, number)) +} diff --git a/crates/sqlbuild-cache/src/store/tests/test_native_store.rs b/crates/sqlbuild-cache/src/store/tests/test_native_store.rs new file mode 100644 index 0000000000..bd6128a871 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/test_native_store.rs @@ -0,0 +1,197 @@ +use crate::store::errors::StoreDecodeError; +use crate::store::main::open_native_store::open_native_store; +use crate::store::tests::helpers::{ + ENVIRONMENT, KIND, METADATA, VALUE, damage, key, open, read_record, record, saved_store, +}; +use crate::store::tests::test_types::{ + NativeStoreDiscardTestCase, NativeStoreOpenTestCase, NativeStoreRetentionTestCase, + RecordReaderTestCase, +}; + +#[test] +fn given_saved_store_when_opening_then_only_the_same_kind_and_environment_load() { + let test_cases = [ + NativeStoreOpenTestCase { + description: "same kind and environment", + kind: KIND, + environment: ENVIRONMENT, + edit: |_| {}, + expected_value: Some(VALUE), + expected_metadata: METADATA, + }, + NativeStoreOpenTestCase { + description: "another environment", + kind: KIND, + environment: "environment-b", + edit: |_| {}, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "another kind", + kind: "inventory", + environment: ENVIRONMENT, + edit: |_| {}, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "damaged file", + kind: KIND, + environment: ENVIRONMENT, + edit: damage, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "truncated file", + kind: KIND, + environment: ENVIRONMENT, + edit: |path| std::fs::write(path, b"SQBSTORE").expect("truncated"), + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "missing file", + kind: KIND, + environment: ENVIRONMENT, + edit: |path| std::fs::remove_file(path).expect("removed"), + expected_value: None, + expected_metadata: b"", + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("cache/macro-calls.bin"); + saved_store(&path); + (test_case.edit)(&path); + + let mut store = + open_native_store(&path, test_case.kind, test_case.environment).expect("opened store"); + + assert_eq!( + store.get(&key("cents")).map(<[u8]>::to_vec), + test_case.expected_value.map(<[u8]>::to_vec), + "{}", + test_case.description + ); + assert_eq!( + store.metadata(), + test_case.expected_metadata, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_entry_unused_for_many_saves_when_saving_then_it_is_dropped_after_the_limit() { + let test_cases = [ + NativeStoreRetentionTestCase { + description: "unused for the retained number of saves", + unused_saves: 64, + expected_retained: true, + }, + NativeStoreRetentionTestCase { + description: "unused for one save more", + unused_saves: 65, + expected_retained: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + saved_store(&path); + for _ in 0..test_case.unused_saves { + let mut store = open(&path); + store.put(key("fresh"), VALUE.to_vec()); + let _ = store.save(&path, METADATA).expect("saved"); + } + + let mut store = open(&path); + + assert_eq!( + store.get(&key("cents")).is_some(), + test_case.expected_retained, + "{}", + test_case.description + ); + assert!( + store.get(&key("fresh")).is_some(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_loaded_store_when_saving_new_entries_then_only_discarded_entries_are_lost() { + let test_cases = [ + NativeStoreDiscardTestCase { + description: "kept", + prepare: |_| {}, + expected_old_entry: true, + }, + NativeStoreDiscardTestCase { + description: "discarded", + prepare: |store| store.discard(), + expected_old_entry: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + saved_store(&path); + let mut store = open(&path); + (test_case.prepare)(&mut store); + store.put(key("fresh"), VALUE.to_vec()); + let _ = store.save(&path, b"").expect("saved"); + + let mut reopened = open(&path); + + assert_eq!( + reopened.get(&key("cents")).is_some(), + test_case.expected_old_entry, + "{}", + test_case.description + ); + assert!( + reopened.get(&key("fresh")).is_some(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_record_bytes_when_reading_then_only_complete_records_decode() { + let test_cases = [ + RecordReaderTestCase { + description: "complete record", + bytes: record, + expected_record: Ok(("café".to_owned(), 7)), + }, + RecordReaderTestCase { + description: "truncated record", + bytes: || record()[..10].to_vec(), + expected_record: Err(StoreDecodeError), + }, + RecordReaderTestCase { + description: "trailing byte", + bytes: || [record(), vec![0]].concat(), + expected_record: Err(StoreDecodeError), + }, + ]; + + for test_case in test_cases { + assert_eq!( + read_record(&(test_case.bytes)()), + test_case.expected_record, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/store/tests/test_types.rs b/crates/sqlbuild-cache/src/store/tests/test_types.rs new file mode 100644 index 0000000000..01b77fa4c0 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/test_types.rs @@ -0,0 +1,34 @@ +use std::path::Path; + +use crate::store::errors::StoreDecodeError; +use crate::store::models::NativeStore; + +/// A change applied to a saved store file before it is opened again. +pub(crate) type StoreFileEdit = fn(&Path); + +pub(crate) struct NativeStoreOpenTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) environment: &'static str, + pub(crate) edit: StoreFileEdit, + pub(crate) expected_value: Option<&'static [u8]>, + pub(crate) expected_metadata: &'static [u8], +} + +pub(crate) struct NativeStoreRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) unused_saves: u64, + pub(crate) expected_retained: bool, +} + +pub(crate) struct NativeStoreDiscardTestCase { + pub(crate) description: &'static str, + pub(crate) prepare: fn(&mut NativeStore), + pub(crate) expected_old_entry: bool, +} + +pub(crate) struct RecordReaderTestCase { + pub(crate) description: &'static str, + pub(crate) bytes: fn() -> Vec, + pub(crate) expected_record: Result<(String, u64), StoreDecodeError>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 7eab2dc618..840752e5fa 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -21,10 +21,12 @@ mimalloc.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs new file mode 100644 index 0000000000..31c5e16613 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs @@ -0,0 +1 @@ +pub(crate) mod native_store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs new file mode 100644 index 0000000000..0cd6e1c9a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -0,0 +1,58 @@ +//! Digests and project fingerprints of the shared native store, with the GIL released. + +use pyo3::exceptions::{PyOSError, PyRuntimeError}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::content_digest::content_digest as digest; +use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each; +use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use std::collections::HashSet; +use std::path::PathBuf; + +use crate::bindings::types::CompilerDetach; + +/// Hexadecimal digest of length-prefixed UTF-8 `parts`, the digest every native cache key uses. +#[pyfunction] +pub(crate) fn content_digest(parts: Vec) -> String { + hex_digest(&digest(&parts)) +} + +/// Digest of every project file outside skipped folders and `excluded_files`, else `OSError`. +#[pyfunction] +pub(crate) fn fingerprint_project_files( + py: Python<'_>, + root: PathBuf, + excluded_files: Vec, +) -> PyResult { + let excluded: HashSet = excluded_files.into_iter().collect(); + let fingerprinted: ContentDigest = py + .compiler_detach(|| Ok(fingerprint(&root, &excluded))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string()))?; + Ok(hex_digest(&fingerprinted)) +} + +/// Hexadecimal content digest of each file in `paths`, or `None` where it cannot be read. +#[pyfunction] +pub(crate) fn digest_files(py: Python<'_>, paths: Vec) -> PyResult>> { + let digests: Vec> = py + .compiler_detach(|| Ok(digest_each(&paths))) + .map_err(PyRuntimeError::new_err)?; + Ok(digests.iter().map(readable_digest).collect()) +} + +fn readable_digest(digest: &std::io::Result) -> Option { + match digest { + Ok(digest) => Some(hex_digest(digest)), + Err(_unreadable) => None, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(content_digest, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; + module.add_function(wrap_pyfunction!(digest_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e5d5e03f58..7863a42fdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,7 +1,9 @@ pub(crate) mod analysis; pub(crate) mod boundary; +pub(crate) mod cache; pub(crate) mod discovery; pub(crate) mod model_config; +pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; pub(crate) mod sqltext; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs new file mode 100644 index 0000000000..718dcff75f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -0,0 +1,131 @@ +//! The macro call memo class: one compile's recorded calls and its optional native store. + +use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::prelude::Python; +use pyo3::{PyResult, pyclass, pymethods}; +use sqlbuild_cache::store::main::open_native_store::open_native_store; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_render::macro_calls::models::MacroCallEntry; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::bindings::types::CompilerDetach; + +use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row}; + +/// The native store kind holding recorded macro calls. +const MACRO_CALL_STORE_KIND: &str = "macro-calls"; + +/// The macro call results one compile records and replays, keyed by call class and call text. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug, Default)] +pub(crate) struct MacroCallMemo { + pub(crate) inner: Mutex, +} + +fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("macro call memo is poisoned") +} + +#[pymethods] +impl MacroCallMemo { + #[new] + fn new() -> Self { + Self::default() + } + + /// The recorded result of a call in `class_id`, or `None` when it must execute. + fn lookup(&self, class_id: u64, call_text: &str) -> PyResult> { + let found = self + .inner + .lock() + .map_err(poisoned)? + .lookup(class_id, call_text); + Ok(found.map(|entry| { + ( + entry.sql.clone(), + entry.relations.clone(), + entry.events.iter().map(event_row).collect(), + ) + })) + } + + /// Record the `(sql, relations, events)` result of one executed call. + fn record(&self, class_id: u64, call_text: String, entry: EntryRow) -> PyResult<()> { + let (sql, relations, events) = entry; + let entry = MacroCallEntry { + sql, + relations, + events: events + .into_iter() + .map(event_from_row) + .collect::>()?, + }; + self.inner + .lock() + .map_err(poisoned)? + .record(class_id, call_text, entry); + Ok(()) + } + + /// Hits, misses and recorded results so far. + fn stats(&self) -> PyResult<(usize, usize, usize)> { + Ok(self.inner.lock().map_err(poisoned)?.stats()) + } + + /// Open the store at `path` for `environment`; returns its saved metadata and entry count. + fn attach_store( + &self, + py: Python<'_>, + path: PathBuf, + environment: &str, + ) -> PyResult<(Vec, usize)> { + let store: NativeStore = py + .compiler_detach(|| Ok(open_native_store(&path, MACRO_CALL_STORE_KIND, environment))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string()))?; + let opened = (store.metadata().to_vec(), store.loaded_entries()); + self.inner.lock().map_err(poisoned)?.attach_store(store); + Ok(opened) + } + + /// Drop the attached store's loaded entries after its metadata failed validation. + fn discard_store(&self) -> PyResult<()> { + if let Some(store) = self.inner.lock().map_err(poisoned)?.store_mut() { + store.discard(); + } + Ok(()) + } + + /// Store results of `class_id` under `class_text` and everything the store key covers. + fn set_persistent_class(&self, class_id: u64, class_text: &str) -> PyResult<()> { + self.inner + .lock() + .map_err(poisoned)? + .set_persistent_class(class_id, class_text); + Ok(()) + } + + /// Calls found in the store and calls recorded into it. + fn store_stats(&self) -> PyResult<(usize, usize)> { + Ok(self.inner.lock().map_err(poisoned)?.store_stats()) + } + + /// Atomically save the store when this compile recorded into it; returns entries written. + fn save_store( + &self, + py: Python<'_>, + path: PathBuf, + metadata: &[u8], + ) -> PyResult> { + let mut memo = self.inner.lock().map_err(poisoned)?; + let Some(store) = memo.store_mut().filter(|store| store.needs_save()) else { + return Ok(None); + }; + let store: &NativeStore = store; + py.compiler_detach(|| Ok(store.save(&path, metadata))) + .map_err(PyRuntimeError::new_err)? + .map(Some) + .map_err(|error| PyOSError::new_err(error.to_string())) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs new file mode 100644 index 0000000000..78ccae8219 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -0,0 +1,75 @@ +//! Python rows of macro call sites, splice spans and recorded call events. + +use pyo3::PyResult; +use pyo3::exceptions::PyValueError; +use sqlbuild_render::macro_calls::models::{MacroCallEvent, MacroCallSite, MacroSpliceSpan}; + +/// Event tags shared with `sqlbuild.compiler.macro_bridge.constants`. +const MACRO_USE: u8 = 0; +const DECLARATION_READ: u8 = 1; +const GENERATED_SQL: u8 = 2; +const ARGUMENT_REFERENCE: u8 = 3; + +/// One call site: start, end, name, tree names and whether typed reference text appears. +pub(crate) type SiteRow = (usize, usize, String, Vec, bool); +/// One substitution span: source start and end, output start and end. +pub(crate) type SpanRow = (usize, usize, usize, usize); +/// One recorded event: its tag and two text fields. +pub(crate) type EventRow = (u8, String, String); +/// One recorded call: its SQL, added relations and events. +pub(crate) type EntryRow = (String, Vec<(String, String)>, Vec); + +pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { + ( + site.start, + site.end, + site.name, + site.tree_names, + site.typed_reference_text, + ) +} + +pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { + ( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ) +} + +pub(crate) fn event_row(event: &MacroCallEvent) -> EventRow { + match event { + MacroCallEvent::MacroUse { name } => (MACRO_USE, name.clone(), String::new()), + MacroCallEvent::DeclarationRead { kind, name } => { + (DECLARATION_READ, kind.clone(), name.clone()) + } + MacroCallEvent::GeneratedSql { macro_name, sql } => { + (GENERATED_SQL, macro_name.clone(), sql.clone()) + } + MacroCallEvent::ArgumentReference { kind, name } => { + (ARGUMENT_REFERENCE, kind.clone(), name.clone()) + } + } +} + +pub(crate) fn event_from_row((tag, first, second): EventRow) -> PyResult { + match tag { + MACRO_USE => Ok(MacroCallEvent::MacroUse { name: first }), + DECLARATION_READ => Ok(MacroCallEvent::DeclarationRead { + kind: first, + name: second, + }), + GENERATED_SQL => Ok(MacroCallEvent::GeneratedSql { + macro_name: first, + sql: second, + }), + ARGUMENT_REFERENCE => Ok(MacroCallEvent::ArgumentReference { + kind: first, + name: second, + }), + _ => Err(PyValueError::new_err(format!( + "unknown macro call event tag {tag}" + ))), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs new file mode 100644 index 0000000000..1d4677fa4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -0,0 +1,57 @@ +//! Macro call-site scanning and splicing for the Python macro bridge, with the GIL released. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; +use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; +use sqlbuild_render::macro_calls::models::ScanDeferral; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::render::macro_call_memo::MacroCallMemo; +use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_row, span_row}; +use crate::bindings::types::CompilerDetach; + +/// Every top-level macro call site of `sql`, or `None` when Python must expand the string. +#[pyfunction] +pub(crate) fn scan_macro_call_sites( + py: Python<'_>, + sql: &str, + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult>> { + let Some(python) = python_text(python_version, unicode_version) else { + return Ok(None); + }; + let scanned = py + .compiler_detach(|| Ok(scan_sites(python, sql))) + .map_err(compiler_error)?; + match scanned { + Ok(sites) => Ok(Some(sites.into_iter().map(site_row).collect())), + Err(ScanDeferral) => Ok(None), + } +} + +/// Splice rendered macro outputs over their call sites, returning the SQL and substitution spans. +#[pyfunction] +pub(crate) fn splice_macro_calls( + py: Python<'_>, + sql: &str, + sites: Vec<(usize, usize)>, + outputs: Vec, +) -> PyResult<(String, Vec)> { + let spliced = py + .compiler_detach(|| Ok(splice_calls(sql, &sites, &outputs))) + .map_err(compiler_error)?; + let (rendered, spans) = + spliced.ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; + Ok((rendered, spans.into_iter().map(span_row).collect())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; + module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs new file mode 100644 index 0000000000..2a04cacf7b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod macro_call_memo; +pub(crate) mod macro_call_rows; +pub(crate) mod macro_calls; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 6e226b6956..64ff38a3ba 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,7 +4,7 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, boundary, discovery, model_config, rules, scopes, sqltext, + analysis, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, }; use crate::bindings::models; @@ -29,6 +29,8 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; + render::macro_calls::register(module)?; + cache::native_store::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-render/Cargo.toml b/crates/sqlbuild-render/Cargo.toml new file mode 100644 index 0000000000..ace3d2d9c6 --- /dev/null +++ b/crates/sqlbuild-render/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqlbuild-render" +version.workspace = true +description = "Native rendering for SQLBuild: macro call-site scanning, splicing and the in-compile macro call memo." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sqlbuild-cache.workspace = true +sqlbuild-core.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-render/src/lib.rs b/crates/sqlbuild-render/src/lib.rs new file mode 100644 index 0000000000..d2567e5538 --- /dev/null +++ b/crates/sqlbuild-render/src/lib.rs @@ -0,0 +1,5 @@ +//! Native rendering: macro call sites, splicing of macro results and the in-compile call memo. + +#![forbid(unsafe_code)] + +pub mod macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs new file mode 100644 index 0000000000..29d46d32c7 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs @@ -0,0 +1,80 @@ +//! Stored bytes of one recorded macro call. + +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +use crate::macro_calls::models::{MacroCallEntry, MacroCallEvent}; + +const MACRO_USE: u64 = 0; +const DECLARATION_READ: u64 = 1; +const GENERATED_SQL: u64 = 2; +const ARGUMENT_REFERENCE: u64 = 3; + +pub(crate) fn encode_entry(entry: &MacroCallEntry) -> Vec { + let mut writer: RecordWriter = RecordWriter::default(); + writer.put_str(&entry.sql); + writer.put_u64(entry.relations.len() as u64); + for (kind, name) in &entry.relations { + writer.put_str(kind); + writer.put_str(name); + } + writer.put_u64(entry.events.len() as u64); + for event in &entry.events { + let (tag, first, second) = match event { + MacroCallEvent::MacroUse { name } => (MACRO_USE, name.as_str(), ""), + MacroCallEvent::DeclarationRead { kind, name } => { + (DECLARATION_READ, kind.as_str(), name.as_str()) + } + MacroCallEvent::GeneratedSql { macro_name, sql } => { + (GENERATED_SQL, macro_name.as_str(), sql.as_str()) + } + MacroCallEvent::ArgumentReference { kind, name } => { + (ARGUMENT_REFERENCE, kind.as_str(), name.as_str()) + } + }; + writer.put_u64(tag); + writer.put_str(first); + writer.put_str(second); + } + writer.into_bytes() +} + +/// The entry stored by [`encode_entry`]; bytes this build did not write are a decode error. +pub(crate) fn decode_entry(bytes: &[u8]) -> Result { + let mut reader: RecordReader<'_> = RecordReader::new(bytes); + let sql: String = reader.string()?; + let relation_count: u64 = reader.u64()?; + let mut relations: Vec<(String, String)> = Vec::new(); + for _ in 0..relation_count { + relations.push((reader.string()?, reader.string()?)); + } + let event_count: u64 = reader.u64()?; + let mut events: Vec = Vec::new(); + for _ in 0..event_count { + let tag: u64 = reader.u64()?; + let first: String = reader.string()?; + let second: String = reader.string()?; + events.push(match tag { + MACRO_USE if second.is_empty() => MacroCallEvent::MacroUse { name: first }, + DECLARATION_READ => MacroCallEvent::DeclarationRead { + kind: first, + name: second, + }, + GENERATED_SQL => MacroCallEvent::GeneratedSql { + macro_name: first, + sql: second, + }, + ARGUMENT_REFERENCE => MacroCallEvent::ArgumentReference { + kind: first, + name: second, + }, + _ => return Err(StoreDecodeError), + }); + } + reader.finish()?; + Ok(MacroCallEntry { + sql, + relations, + events, + }) +} diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs new file mode 100644 index 0000000000..92e1d6be79 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod entry_codec; +pub(crate) mod offsets; +pub(crate) mod scanner; diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs new file mode 100644 index 0000000000..6dbc61a808 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs @@ -0,0 +1,45 @@ +//! Conversions between UTF-8 byte offsets and Python code-point offsets of one text. + +/// Return the code-point offset of each ascending byte offset in `byte_offsets`. +pub(crate) fn char_offsets(text: &str, byte_offsets: &[usize]) -> Vec { + if text.is_ascii() { + return byte_offsets.to_vec(); + } + let mut converted = Vec::with_capacity(byte_offsets.len()); + let mut byte = 0; + let mut chars = 0; + for offset in byte_offsets { + chars += text[byte..*offset].chars().count(); + byte = *offset; + converted.push(chars); + } + converted +} + +/// Return the byte offset of each ascending code-point offset, or `None` past the end of `text`. +pub(crate) fn byte_offsets(text: &str, char_offsets: &[usize]) -> Option> { + if text.is_ascii() { + return char_offsets + .iter() + .all(|offset| *offset <= text.len()) + .then(|| char_offsets.to_vec()); + } + let mut converted = Vec::with_capacity(char_offsets.len()); + let mut indices = text + .char_indices() + .map(|(byte, _)| byte) + .chain([text.len()]); + let mut position = 0; + let mut current = indices.next()?; + for offset in char_offsets { + if *offset < position { + return None; + } + while position < *offset { + current = indices.next()?; + position += 1; + } + converted.push(current); + } + Some(converted) +} diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs new file mode 100644 index 0000000000..71cef2c882 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -0,0 +1,277 @@ +//! Byte offset port of the Python macro call scanner; unsure or rejected text defers to Python. + +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; + +use crate::macro_calls::models::ScanDeferral; + +const TYPED_REFERENCE_NAMES: [&str; 3] = ["__ref", "__source", "__seed"]; + +/// One call found by the scanner, in byte offsets of the scanned text. +pub(crate) struct ScannedCall { + pub(crate) start: usize, + pub(crate) open: usize, + pub(crate) close: usize, + pub(crate) name: String, +} + +/// Return every top-level call of `text` in order. +pub(crate) fn top_level_calls( + python: PythonText, + text: &str, +) -> Result, ScanDeferral> { + let mut calls: Vec = Vec::new(); + let mut cursor = 0; + while cursor < text.len() { + let Some(start) = next_macro_start(python, text, cursor)? else { + break; + }; + let call = call_at(python, text, start)?; + cursor = call.close + 1; + calls.push(call); + } + Ok(calls) +} + +/// Names of every call nested in `args` in source order, in one pass without recursion. +pub(crate) fn nested_names(python: PythonText, args: &str) -> Result, ScanDeferral> { + let bytes: &[u8] = args.as_bytes(); + let mut names: Vec = Vec::new(); + let mut open_call_depths: Vec = Vec::new(); + let mut depth: usize = 0; + let mut index: usize = 0; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, + b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), + b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), + b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, + b'@' if macro_call_starts_at(python, args, index)? => { + let (name, open) = call_header(python, args, index)?; + names.push(name); + depth += 1; + open_call_depths.push(depth); + index = open + 1; + } + b'(' => { + depth += 1; + index += 1; + } + b')' => { + if open_call_depths.last() == Some(&depth) { + let _ = open_call_depths.pop(); + } + depth = depth.checked_sub(1).ok_or(ScanDeferral)?; + index += 1; + } + _ => index += 1, + } + } + if open_call_depths.is_empty() { + Ok(names) + } else { + Err(ScanDeferral) + } +} + +/// Whether call arguments mention a typed reference function name anywhere. +pub(crate) fn mentions_typed_reference(args: &str) -> bool { + TYPED_REFERENCE_NAMES.iter().any(|name| args.contains(name)) +} + +fn call_at(python: PythonText, text: &str, start: usize) -> Result { + let (name, open) = call_header(python, text, start)?; + let close = matching_paren(text.as_bytes(), open)?; + Ok(ScannedCall { + start, + open, + close, + name, + }) +} + +fn call_header( + python: PythonText, + text: &str, + start: usize, +) -> Result<(String, usize), ScanDeferral> { + let name_end = identifier_end(python, text, start + 1); + let open = skip_whitespace(text, name_end); + if text.as_bytes().get(open) != Some(&b'(') { + return Err(ScanDeferral); + } + Ok((text[start + 1..name_end].to_owned(), open)) +} + +fn next_macro_start( + python: PythonText, + text: &str, + from: usize, +) -> Result, ScanDeferral> { + let bytes = text.as_bytes(); + let mut index = from; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, + b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), + b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), + b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, + b'@' => { + if macro_call_starts_at(python, text, index)? { + return Ok(Some(index)); + } + index += 1; + } + _ => index += 1, + } + } + Ok(None) +} + +fn macro_call_starts_at(python: PythonText, text: &str, at: usize) -> Result { + let Some(first) = char_at(text, at + 1) else { + return Ok(false); + }; + if !first.is_ascii() { + return Err(ScanDeferral); + } + if !(first.is_ascii_alphabetic() || first == '_') { + return Ok(false); + } + let mut cursor = at + 2; + while let Some(character) = char_at(text, cursor) { + if !is_identifier_continue(python, character) { + break; + } + cursor = skip_whitespace(text, cursor + character.len_utf8()); + } + Ok(text.as_bytes().get(cursor) == Some(&b'(')) +} + +fn identifier_end(python: PythonText, text: &str, from: usize) -> usize { + let mut cursor = from; + while let Some(character) = char_at(text, cursor) { + if !is_identifier_continue(python, character) { + break; + } + cursor += character.len_utf8(); + } + cursor +} + +fn is_identifier_continue(python: PythonText, character: char) -> bool { + character == '_' || is_python_alnum(python, character) +} + +fn skip_whitespace(text: &str, from: usize) -> usize { + let mut cursor = from; + while let Some(character) = char_at(text, cursor) { + if !is_python_space(character) { + break; + } + cursor += character.len_utf8(); + } + cursor +} + +fn char_at(text: &str, index: usize) -> Option { + text.get(index..).and_then(|rest| rest.chars().next()) +} + +fn matching_paren(bytes: &[u8], open: usize) -> Result { + let mut depth = 1usize; + let mut index = open + 1; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => { + index = quote_end(bytes, index)?; + continue; + } + b'$' => { + if let Some(end) = dollar_quote_end(bytes, index)? { + index = end; + continue; + } + } + b'-' if bytes.get(index + 1) == Some(&b'-') => { + index = line_comment_end(bytes, index); + continue; + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index = block_comment_end(bytes, index)?; + continue; + } + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + _ => {} + } + index += 1; + } + Err(ScanDeferral) +} + +fn quote_end(bytes: &[u8], start: usize) -> Result { + let quote = bytes[start]; + let doubled_escapes = quote == b'\'' || quote == b'"'; + let mut index = start + 1; + loop { + let offset = bytes[index.min(bytes.len())..] + .iter() + .position(|byte| *byte == quote) + .ok_or(ScanDeferral)?; + index += offset; + if doubled_escapes && bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + 1); + } +} + +fn dollar_quote_end(bytes: &[u8], start: usize) -> Result, ScanDeferral> { + if start + .checked_sub(1) + .is_some_and(|previous| continues_dollar_word(bytes[previous])) + { + return Ok(None); + } + let tag_end = bytes[start + 1..] + .iter() + .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) + .map_or(bytes.len(), |offset| start + 1 + offset); + let tag = &bytes[start + 1..tag_end]; + if bytes.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { + return Ok(None); + } + let delimiter = &bytes[start..=tag_end]; + bytes[tag_end + 1..] + .windows(delimiter.len()) + .position(|window| window == delimiter) + .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) + .ok_or(ScanDeferral) +} + +fn continues_dollar_word(byte: u8) -> bool { + !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') +} + +fn line_comment_end(bytes: &[u8], start: usize) -> usize { + bytes[start..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(bytes.len(), |offset| start + offset + 1) +} + +fn block_comment_end(bytes: &[u8], start: usize) -> Result { + bytes[start + 2..] + .windows(2) + .position(|pair| pair == b"*/") + .map(|offset| start + 2 + offset + 2) + .ok_or(ScanDeferral) +} diff --git a/crates/sqlbuild-render/src/macro_calls/main/mod.rs b/crates/sqlbuild-render/src/macro_calls/main/mod.rs new file mode 100644 index 0000000000..2f945c96d5 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/mod.rs @@ -0,0 +1,2 @@ +pub mod scan_macro_call_sites; +pub mod splice_macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs new file mode 100644 index 0000000000..47598cc9dd --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs @@ -0,0 +1,42 @@ +//! Top-level macro call sites of one authored SQL string, as Python's expansion loop visits them. + +use sqlbuild_core::text::models::PythonText; + +use crate::macro_calls::_helpers::offsets::char_offsets; +use crate::macro_calls::_helpers::scanner::{ + mentions_typed_reference, nested_names, top_level_calls, +}; +use crate::macro_calls::models::{MacroCallSite, ScanDeferral}; + +/// Every top-level call site in order under `python`'s character classes, or a deferral. +pub fn scan_macro_call_sites( + python: PythonText, + sql: &str, +) -> Result, ScanDeferral> { + let calls = top_level_calls(python, sql)?; + let boundaries: Vec = calls + .iter() + .flat_map(|call| [call.start, call.close + 1]) + .collect(); + let offsets = char_offsets(sql, &boundaries); + calls + .into_iter() + .zip(offsets.as_chunks::<2>().0) + .map(|(call, offsets)| { + let args = &sql[call.open + 1..call.close]; + let mut tree_names: Vec = vec![call.name.clone()]; + for name in nested_names(python, args)? { + if !tree_names.contains(&name) { + tree_names.push(name); + } + } + Ok(MacroCallSite { + start: offsets[0], + end: offsets[1], + name: call.name, + tree_names, + typed_reference_text: mentions_typed_reference(args), + }) + }) + .collect() +} diff --git a/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs b/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs new file mode 100644 index 0000000000..c1a986e5b1 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs @@ -0,0 +1,45 @@ +//! Replace macro call sites with their rendered SQL and report each substitution span. + +use crate::macro_calls::_helpers::offsets::byte_offsets; +use crate::macro_calls::models::MacroSpliceSpan; + +/// Splice `outputs` over ascending code-point `sites`; `None` when they do not fit `sql`. +pub fn splice_macro_calls( + sql: &str, + sites: &[(usize, usize)], + outputs: &[String], +) -> Option<(String, Vec)> { + if sites.len() != outputs.len() { + return None; + } + let boundaries: Vec = sites + .iter() + .flat_map(|(start, end)| [*start, *end]) + .collect(); + if boundaries.windows(2).any(|pair| pair[0] > pair[1]) { + return None; + } + let bytes = byte_offsets(sql, &boundaries)?; + let mut rendered = + String::with_capacity(sql.len() + outputs.iter().map(String::len).sum::()); + let mut spans = Vec::with_capacity(sites.len()); + let mut cursor_byte = 0; + let mut output_chars = 0; + for ((site, output), range) in sites.iter().zip(outputs).zip(bytes.as_chunks::<2>().0) { + let literal = &sql[cursor_byte..range[0]]; + rendered.push_str(literal); + output_chars += literal.chars().count(); + rendered.push_str(output); + let output_len = output.chars().count(); + spans.push(MacroSpliceSpan { + source_start: site.0, + source_end: site.1, + output_start: output_chars, + output_end: output_chars + output_len, + }); + output_chars += output_len; + cursor_byte = range[1]; + } + rendered.push_str(&sql[cursor_byte..]); + Some((rendered, spans)) +} diff --git a/crates/sqlbuild-render/src/macro_calls/mod.rs b/crates/sqlbuild-render/src/macro_calls/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-render/src/macro_calls/models.rs b/crates/sqlbuild-render/src/macro_calls/models.rs new file mode 100644 index 0000000000..d2d5ada473 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/models.rs @@ -0,0 +1,152 @@ +//! Macro call sites, recorded macro call results and the in-compile memo that replays them. + +use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::NativeStore; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::macro_calls::_helpers::entry_codec::{decode_entry, encode_entry}; + +/// One top-level `@macro(...)` call in an authored SQL string; offsets count code points. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroCallSite { + /// Offset of the `@` that starts the call. + pub start: usize, + /// Offset just past the call's closing parenthesis. + pub end: usize, + /// The called macro's name. + pub name: String, + /// Every macro name in the call, the top-level name first, then nested calls once each. + pub tree_names: Vec, + /// The arguments mention `__ref`, `__source` or `__seed`, so typed references may be passed. + pub typed_reference_text: bool, +} + +/// The scan met text Python reports or classifies differently; Python must expand the string. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScanDeferral; + +/// One consumer-independent fact a macro call produced, replayed for every consumer in order. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum MacroCallEvent { + /// A macro, top-level or nested, was resolved and called. + MacroUse { name: String }, + /// A macro read a constant or enum through its context. + DeclarationRead { kind: String, name: String }, + /// A macro returned SQL that names typed references, a generated-reference candidate. + GeneratedSql { macro_name: String, sql: String }, + /// A typed reference was written as a macro argument. + ArgumentReference { kind: String, name: String }, +} + +/// The recorded result of one top-level macro call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroCallEntry { + /// The macro's SQL before relation placeholders are rendered. + pub sql: String, + /// Typed references the call added to the string's relation table, in order. + pub relations: Vec<(String, String)>, + /// Facts the call produced, in encounter order. + pub events: Vec, +} + +/// Substitution offsets of one spliced macro call, in code points. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MacroSpliceSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + +/// One compile's recorded macro call results by class and call text, plus an optional store. +#[derive(Debug, Default)] +pub struct MacroCallMemo { + entries: HashMap<(u64, String), Arc>, + hits: usize, + misses: usize, + store: Option, + class_digests: HashMap, + store_hits: usize, + store_records: usize, +} + +impl MacroCallMemo { + /// The recorded result of the call, found in this compile or, if persistent, in the store. + pub fn lookup(&mut self, class_id: u64, call_text: &str) -> Option> { + let key: (u64, String) = (class_id, call_text.to_owned()); + if let Some(found) = self.entries.get(&key).cloned() { + self.hits += 1; + return Some(found); + } + match self.stored_entry(class_id, call_text) { + Some(entry) => { + self.store_hits += 1; + let _ = self.entries.insert(key, Arc::clone(&entry)); + Some(entry) + } + None => { + self.misses += 1; + None + } + } + } + + /// Record the result of one executed call, storing it when its class is persistent. + pub fn record(&mut self, class_id: u64, call_text: String, entry: MacroCallEntry) { + if let Some(store_key) = self.store_key(class_id, &call_text) + && let Some(store) = self.store.as_mut() + { + store.put(store_key, encode_entry(&entry)); + self.store_records += 1; + } + let _ = self.entries.insert((class_id, call_text), Arc::new(entry)); + } + + /// Lookups that found a recorded result, lookups that did not, and recorded results. + pub fn stats(&self) -> (usize, usize, usize) { + (self.hits, self.misses, self.entries.len()) + } + + /// Attach a store whose keys already cover everything every persistent class depends on. + pub fn attach_store(&mut self, store: NativeStore) { + self.store = Some(store); + } + + /// The attached store, if any. + pub fn store_mut(&mut self) -> Option<&mut NativeStore> { + self.store.as_mut() + } + + /// Mark `class_id` persistent under `class_text`, which names all its results depend on. + pub fn set_persistent_class(&mut self, class_id: u64, class_text: &str) { + let _ = self + .class_digests + .insert(class_id, content_digest(&[class_text])); + } + + /// Calls found in the store and calls recorded into it. + pub fn store_stats(&self) -> (usize, usize) { + (self.store_hits, self.store_records) + } + + fn stored_entry(&mut self, class_id: u64, call_text: &str) -> Option> { + let store_key: ContentDigest = self.store_key(class_id, call_text)?; + let bytes: &[u8] = self.store.as_mut()?.get(&store_key)?; + match decode_entry(bytes) { + Ok(entry) => Some(Arc::new(entry)), + Err(StoreDecodeError) => None, + } + } + + fn store_key(&self, class_id: u64, call_text: &str) -> Option { + self.store.as_ref()?; + let class_digest = self.class_digests.get(&class_id)?; + Some(content_digest(&[ + class_digest.as_slice(), + call_text.as_bytes(), + ])) + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests.rs b/crates/sqlbuild-render/src/macro_calls/tests.rs new file mode 100644 index 0000000000..6340f8ffc3 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_macro_call_memo.rs"] +mod macro_call_memo; +#[path = "tests/test_scan_macro_call_sites.rs"] +mod scan_macro_call_sites; +#[path = "tests/test_splice_macro_calls.rs"] +mod splice_macro_calls; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs new file mode 100644 index 0000000000..2fa89bc6ec --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs @@ -0,0 +1,76 @@ +use sqlbuild_cache::store::main::open_native_store::open_native_store; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; +use std::path::Path; + +use crate::macro_calls::models::{MacroCallEntry, MacroCallEvent, MacroCallMemo}; + +/// A memo holding one recorded `@cents(amount)` call in class 1, and that call's entry. +pub(crate) fn recorded_memo() -> (MacroCallMemo, MacroCallEntry) { + let mut memo = MacroCallMemo::default(); + let entry = MacroCallEntry { + sql: "amount * 100".to_owned(), + relations: vec![("model".to_owned(), "orders".to_owned())], + events: vec![ + MacroCallEvent::MacroUse { + name: "cents".to_owned(), + }, + MacroCallEvent::DeclarationRead { + kind: "constant".to_owned(), + name: "scale".to_owned(), + }, + ], + }; + memo.record(1, "@cents(amount)".to_owned(), entry.clone()); + (memo, entry) +} + +/// The character classes of Python 3.12. +pub(crate) fn python_312() -> PythonText { + python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") +} + +/// Record `entry` for `@cents(amount)` into a fresh store at `path`, persistent per `class_texts`. +pub(crate) fn saved_memo_store(path: &Path, class_texts: &[&str], entry: &MacroCallEntry) { + let mut memo = MacroCallMemo::default(); + memo.attach_store( + open_native_store(path, STORE_KIND, STORE_ENVIRONMENT).expect("opened store"), + ); + for class_text in class_texts { + memo.set_persistent_class(1, class_text); + } + memo.record(1, "@cents(amount)".to_owned(), entry.clone()); + let _ = memo + .store_mut() + .expect("attached store") + .save(path, b"") + .expect("saved store"); +} + +pub(crate) const STORE_KIND: &str = "macro-calls"; +pub(crate) const STORE_ENVIRONMENT: &str = "environment"; + +/// An entry with one event of every kind. +pub(crate) fn every_event_entry() -> MacroCallEntry { + MacroCallEntry { + sql: "SELECT * FROM __SQLBUILD_RELATION_1__ -- café".to_owned(), + relations: vec![("model".to_owned(), "orders".to_owned())], + events: vec![ + MacroCallEvent::MacroUse { + name: "cents".to_owned(), + }, + MacroCallEvent::DeclarationRead { + kind: "enum".to_owned(), + name: "order_status".to_owned(), + }, + MacroCallEvent::GeneratedSql { + macro_name: "cents".to_owned(), + sql: "__ref(\"orders\")".to_owned(), + }, + MacroCallEvent::ArgumentReference { + kind: "source".to_owned(), + name: "raw.orders".to_owned(), + }, + ], + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs new file mode 100644 index 0000000000..0bae036198 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs @@ -0,0 +1,122 @@ +use sqlbuild_cache::store::main::open_native_store::open_native_store; + +use crate::macro_calls::models::MacroCallMemo; +use crate::macro_calls::tests::helpers::{ + STORE_ENVIRONMENT, STORE_KIND, every_event_entry, recorded_memo, saved_memo_store, +}; +use crate::macro_calls::tests::test_types::{MacroCallMemoTestCase, MacroCallStoreTestCase}; + +#[test] +fn given_recorded_call_when_looking_up_then_only_same_class_and_text_hit() { + let test_cases = [ + MacroCallMemoTestCase { + description: "same class and call text", + class_id: 1, + call_text: "@cents(amount)", + expected_hit: true, + }, + MacroCallMemoTestCase { + description: "another call class", + class_id: 2, + call_text: "@cents(amount)", + expected_hit: false, + }, + MacroCallMemoTestCase { + description: "another call text", + class_id: 1, + call_text: "@cents(total)", + expected_hit: false, + }, + ]; + + for test_case in test_cases { + let (mut memo, entry) = recorded_memo(); + let found = memo.lookup(test_case.class_id, test_case.call_text); + assert_eq!( + found.as_deref() == Some(&entry), + test_case.expected_hit, + "{}", + test_case.description + ); + assert_eq!( + memo.stats(), + ( + usize::from(test_case.expected_hit), + usize::from(!test_case.expected_hit), + 1 + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_call_when_a_new_memo_looks_it_up_then_only_the_same_persistent_class_hits() { + let test_cases = [ + MacroCallStoreTestCase { + description: "same persistent class and call text", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: true, + }, + MacroCallStoreTestCase { + description: "another persistent class", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@models/marts/_sqlbuild/_macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "another call text", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(total)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "class not persistent when recorded", + recorded_class_texts: &[], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "class not persistent when looked up", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &[], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + let entry = every_event_entry(); + saved_memo_store(&path, test_case.recorded_class_texts, &entry); + let mut memo = MacroCallMemo::default(); + memo.attach_store( + open_native_store(&path, STORE_KIND, STORE_ENVIRONMENT).expect("opened store"), + ); + for class_text in test_case.looked_up_class_texts { + memo.set_persistent_class(7, class_text); + } + + let found = memo.lookup(7, test_case.call_text); + + assert_eq!( + found.as_deref() == Some(&entry), + test_case.expected_store_hit, + "{}", + test_case.description + ); + assert_eq!( + memo.store_stats(), + (usize::from(test_case.expected_store_hit), 0), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs new file mode 100644 index 0000000000..0ee44440e9 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -0,0 +1,182 @@ +use crate::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites; +use crate::macro_calls::tests::helpers::python_312; +use std::time::Instant; + +use crate::macro_calls::tests::test_types::{ + DeepNestingTestCase, ExpectedSite, ScanMacroCallSitesTestCase, +}; + +#[test] +fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansion() { + let test_cases = [ + ScanMacroCallSitesTestCase { + description: "one call", + sql: "SELECT @a(1) FROM t", + expected_sites: Some(&[(7, 12, "a", &["a"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "nested calls belong to the enclosing site", + sql: "@outer(@inner(@leaf()), '@quoted()', @inner(2)) y", + expected_sites: Some(&[(0, 47, "outer", &["outer", "inner", "leaf"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "quoted text and comments hide calls", + sql: "'@a(1)' -- @b(2)\n/* @c(3) */ \"@d()\" `@e()` $$@f()$$ @g()", + expected_sites: Some(&[(52, 56, "g", &["g"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a one-letter name followed by whitespace is not a call", + sql: "@a (1)", + expected_sites: Some(&[]), + }, + ScanMacroCallSitesTestCase { + description: "a longer name may be separated from its parenthesis", + sql: "@ab (1)", + expected_sites: Some(&[(0, 7, "ab", &["ab"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "whitespace inside a name defers to Python's error", + sql: "@ab c(1)", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "an unclosed quote after a call defers", + sql: "@a(1) 'oops", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "an unclosed call defers", + sql: "@a(1", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "offsets count code points", + sql: "SELECT 'é', @a(1)", + expected_sites: Some(&[(12, 17, "a", &["a"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a non-ASCII character after @ defers", + sql: "@é(1)", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "typed reference text is flagged", + sql: "@m(__ref('x')) @n('__seed') @o(1)", + expected_sites: Some(&[ + (0, 14, "m", &["m"], true), + (15, 27, "n", &["n"], true), + (28, 33, "o", &["o"], false), + ]), + }, + ScanMacroCallSitesTestCase { + description: "a dollar sign continuing a word does not open a quote", + sql: "a$$ @m()", + expected_sites: Some(&[(4, 8, "m", &["m"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "interpolation markers are not calls", + sql: "x @@var @m()", + expected_sites: Some(&[(8, 12, "m", &["m"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "nested names follow source order across siblings and plain parentheses", + sql: "@a((@b(@c(1)), (@d()) ), \"@x()\", @c())", + expected_sites: Some(&[(0, 38, "a", &["a", "b", "c", "d"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a quoted parenthesis inside a nested call does not close it", + sql: "@a(@b(')'))", + expected_sites: Some(&[(0, 11, "a", &["a", "b"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a nested name without its parenthesis defers", + sql: "@a(@bb c(1))", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "parentheses inside quoted arguments do not close the call", + sql: "@m(')', \"(\", `)`) z", + expected_sites: Some(&[(0, 17, "m", &["m"], false)]), + }, + ]; + + for test_case in test_cases { + let actual: Option> = scan_macro_call_sites(python_312(), test_case.sql) + .ok() + .map(|sites| { + sites + .into_iter() + .map(|site| { + let names: Vec<&'static str> = site + .tree_names + .iter() + .map(|name| &*Box::leak(name.clone().into_boxed_str())) + .collect(); + ( + site.start, + site.end, + &*Box::leak(site.name.into_boxed_str()), + &*Box::leak(names.into_boxed_slice()), + site.typed_reference_text, + ) + }) + .collect() + }); + assert_eq!( + actual, + test_case.expected_sites.map(<[ExpectedSite]>::to_vec), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deeply_nested_calls_when_scanning_then_time_is_linear_and_nothing_overflows() { + let test_cases = [ + DeepNestingTestCase { + description: "one thousand levels", + depth: 1_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + DeepNestingTestCase { + description: "twenty thousand levels", + depth: 20_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + DeepNestingTestCase { + description: "one hundred thousand levels", + depth: 100_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + ]; + + for test_case in test_cases { + let sql = format!( + "{}x{}", + "@m(".repeat(test_case.depth), + ")".repeat(test_case.depth) + ); + let started = Instant::now(); + + let sites = scan_macro_call_sites(python_312(), &sql).expect("scanned"); + + assert!( + started.elapsed().as_secs_f64() < test_case.expected_max_seconds, + "{}", + test_case.description + ); + assert_eq!( + sites + .iter() + .map(|site| site.tree_names.clone()) + .collect::>(), + vec![test_case.expected_tree_names.to_vec()], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs new file mode 100644 index 0000000000..fc43f46ee6 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs @@ -0,0 +1,76 @@ +use crate::macro_calls::main::splice_macro_calls::splice_macro_calls; +use crate::macro_calls::tests::test_types::SpliceMacroCallsTestCase; + +#[test] +fn given_call_sites_and_outputs_when_splicing_then_sql_and_spans_match_python() { + let test_cases = [ + SpliceMacroCallsTestCase { + description: "one call", + sql: "SELECT @a(1) FROM t", + sites: &[(7, 12)], + outputs: &["x+1"], + expected_splice: Some(("SELECT x+1 FROM t", &[(7, 12, 7, 10)])), + }, + SpliceMacroCallsTestCase { + description: "code-point offsets around non-ASCII text", + sql: "é @a() ü @b()", + sites: &[(2, 6), (9, 13)], + outputs: &["ö", "zz"], + expected_splice: Some(("é ö ü zz", &[(2, 6, 2, 3), (9, 13, 6, 8)])), + }, + SpliceMacroCallsTestCase { + description: "no sites leave the text unchanged", + sql: "a @ b", + sites: &[], + outputs: &[], + expected_splice: Some(("a @ b", &[])), + }, + SpliceMacroCallsTestCase { + description: "descending sites are rejected", + sql: "@a() @b()", + sites: &[(5, 9), (0, 4)], + outputs: &["x", "y"], + expected_splice: None, + }, + SpliceMacroCallsTestCase { + description: "out-of-bounds sites are rejected", + sql: "é @a()", + sites: &[(2, 9)], + outputs: &["x"], + expected_splice: None, + }, + ]; + + for test_case in test_cases { + let outputs: Vec = test_case + .outputs + .iter() + .map(|output| (*output).to_owned()) + .collect(); + let actual = + splice_macro_calls(test_case.sql, test_case.sites, &outputs).map(|(sql, spans)| { + ( + sql, + spans + .into_iter() + .map(|span| { + ( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ) + }) + .collect::>(), + ) + }); + assert_eq!( + actual, + test_case + .expected_splice + .map(|(sql, spans)| (sql.to_owned(), spans.to_vec())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs new file mode 100644 index 0000000000..b58c34f5a4 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs @@ -0,0 +1,41 @@ +/// One expected call site: start, end, name, tree names and whether typed reference text appears. +pub(crate) type ExpectedSite = (usize, usize, &'static str, &'static [&'static str], bool); + +/// One expected splice: the rendered SQL and each span's source and output offsets. +pub(crate) type ExpectedSplice = (&'static str, &'static [(usize, usize, usize, usize)]); + +pub(crate) struct ScanMacroCallSitesTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sites: Option<&'static [ExpectedSite]>, +} + +pub(crate) struct SpliceMacroCallsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) sites: &'static [(usize, usize)], + pub(crate) outputs: &'static [&'static str], + pub(crate) expected_splice: Option, +} + +pub(crate) struct MacroCallMemoTestCase { + pub(crate) description: &'static str, + pub(crate) class_id: u64, + pub(crate) call_text: &'static str, + pub(crate) expected_hit: bool, +} + +pub(crate) struct MacroCallStoreTestCase { + pub(crate) description: &'static str, + pub(crate) recorded_class_texts: &'static [&'static str], + pub(crate) looked_up_class_texts: &'static [&'static str], + pub(crate) call_text: &'static str, + pub(crate) expected_store_hit: bool, +} + +pub(crate) struct DeepNestingTestCase { + pub(crate) description: &'static str, + pub(crate) depth: usize, + pub(crate) expected_tree_names: &'static [&'static str], + pub(crate) expected_max_seconds: f64, +} diff --git a/fensu.toml b/fensu.toml index ac01351948..7520b517e9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -305,11 +305,13 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ "sqlbuild-analysis", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-python", + "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index 805f8aa782..b217c7f0c0 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -23,6 +23,7 @@ FIXTURE_ROOT, PLAN, PROJECT_CONFIG_FILE, + STORE_WARM_COMPILE, WARM_COMPILE, ) from scripts.compiler_differential.models import ( @@ -91,7 +92,7 @@ def build_corpus( entries.extend( CorpusProject( name=f"dense/{dense_models}{'-custom-rules' if custom_rules else ''}", - commands=(COLD_COMPILE, WARM_COMPILE), + commands=(COLD_COMPILE, WARM_COMPILE, STORE_WARM_COMPILE), expected=EXPECT_SUCCESS, writer=DenseProject(model_count=dense_models, custom_rules=custom_rules).write, ) @@ -115,7 +116,7 @@ def _authored_project( project_dir: Path = source_dir if subdirectory is None else source_dir / subdirectory return CorpusProject( name=name, - commands=(COLD_COMPILE, WARM_COMPILE, *_plan_commands(project_dir)), + commands=(COLD_COMPILE, WARM_COMPILE, STORE_WARM_COMPILE, *_plan_commands(project_dir)), expected=expected, source_dir=source_dir, project_subdirectory=subdirectory, @@ -137,7 +138,7 @@ def _seed_project(seed: int) -> CorpusProject: generated: GeneratedProject = ProjectBuilder(seed).build() return CorpusProject( name=f"seed/{seed}", - commands=(COLD_COMPILE, PLAN), + commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN), expected=ExpectedOutcome( error_code=generated.expected_error_code, succeeding_commands=generated.succeeding_commands, diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 5e1b13c5c4..3c4b283cb0 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -69,9 +69,10 @@ def run_engine( _run_command( command=command, project_dir=project_dir, - environment=_command_environment( - engine=engine, options=options, capture_dir=capture_dir - ), + environment={ + **_command_environment(engine=engine, options=options, capture_dir=capture_dir), + **dict(command.environment), + }, python=options.python, ) ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 756d0a779f..de07f6da70 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -85,6 +85,11 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: WARM_COMPILE: DifferentialCommand = DifferentialCommand( label="compile-warm", arguments=("compile", "--json") ) +STORE_WARM_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile-store-warm", + arguments=("compile", "--json"), + environment=(("SQLBUILD_DISABLE_COMPILE_REUSE", "1"),), +) PLAN: DifferentialCommand = DifferentialCommand(label=PLAN_LABEL, arguments=("plan", "--json")) GENERATOR_DOMAINS: tuple[str, ...] = ("sales", "inventory", "support", "fulfillment", "billing") diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 5a399c2e9c..3af73b087b 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -9,10 +9,11 @@ @dataclass(frozen=True) class DifferentialCommand: - """One sqb invocation run identically under both engines.""" + """One sqb invocation run identically under both engines, with extra environment values.""" label: str arguments: tuple[str, ...] + environment: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index e28a528a3f..c2e24834a3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -109,7 +109,9 @@ anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.id files count only by presence. Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs -must produce the same output. A compile whose templates read `run.id` is never reused. +must produce the same output. Macro results are also reused within a compile and may be kept between +compiles; see [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). A +compile whose templates read `run.id` is never reused. Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when `SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 7c28c84d6a..808371a6c9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/ - Keep implementation details private - Use macros in tests - Use macros in hooks +- Macros must be deterministic - Where macros work - Next steps @@ -213,6 +214,31 @@ SELECT 1 AS id See [SQL Hooks](models/hooks/sql.md) for hook lifecycle and context syntax. +## Macros must be deterministic + +A macro must return the same result whenever it is called with the same arguments and sees the same +context. Avoid module-level state such as counters, random values, the current time, and reads of +files or environment variables that the call does not receive. + +SQLBuild relies on this contract to reuse macro results: + +- [Compile reuse](../cli/compile.md) replays stored renders without running macros again. +- A compile may run each distinct macro call once and reuse its result for identical calls + elsewhere in the project. +- Macro call results may be kept between compiles under `target/cache/`. They are recomputed when + macro or other Python files, configuration, declarations, seeds, sources, variables, the target, + the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps + them, and a changed call is always run again. Only the first compile in a process uses and saves + them: later compiles in the same process, such as in a notebook or a long-running server, run their + macros again. A compile that has loaded Python code it cannot read from a plain file, such as a + module imported from a zip archive, neither uses nor saves them. + +Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and +`@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, +and visible constants and enums. Environment variables a macro reads directly, for example with +`os.environ`, are compared only when their names start with `SQLBUILD_` or `SQB_`, the same set +compile reuse tracks. A macro whose result changes for any other reason may return a stale result. + ## Where macros work Macros are supported in: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5b64d00c95..83909e19fc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -184,6 +184,34 @@ def substitute_static_project_vars( def extract_static_sql_references( sql: str, ) -> list[tuple[str, str, str | None, int | None]] | None: ... +def scan_macro_call_sites( + sql: str, python_version: tuple[int, int], unicode_version: str +) -> list[tuple[int, int, str, list[str], bool]] | None: ... +def splice_macro_calls( + sql: str, sites: list[tuple[int, int]], outputs: list[str] +) -> tuple[str, list[tuple[int, int, int, int]]]: ... + +class MacroCallMemo: + def __init__(self) -> None: ... + def lookup( + self, class_id: int, call_text: str + ) -> tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None: ... + def record( + self, + class_id: int, + call_text: str, + entry: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]], + ) -> None: ... + def stats(self) -> tuple[int, int, int]: ... + def attach_store(self, path: str, environment: str) -> tuple[bytes, int]: ... + def discard_store(self) -> None: ... + def set_persistent_class(self, class_id: int, class_text: str) -> None: ... + def store_stats(self) -> tuple[int, int]: ... + def save_store(self, path: str, metadata: bytes) -> int | None: ... + +def content_digest(parts: list[str]) -> str: ... +def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... +def digest_files(paths: list[str]) -> list[str | None]: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py new file mode 100644 index 0000000000..8e8c3238f2 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -0,0 +1,58 @@ +"""Resolve the compile target and hand the running macro bridge its call store.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.spec.contracts.models import TargetConfig + + +def target_context_with_macro_call_store( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + no_cache: bool, +) -> tuple[str | None, TargetConfig | None, Path | None]: + """Resolve the target and cache directory; a process's first compile may load stored calls.""" + + first_compile: bool = claim_first_compile() + target_context: tuple[str | None, TargetConfig | None, Path | None] = ( + build_compile_target_context( + discovered_inputs=discovered_inputs, + selected_target=selected_target, + no_cache=no_cache, + ) + ) + bridge: MacroBridge | None = active_macro_bridge() + project_dir: Path | None = discovered_inputs.project_dir + compile_cache_dir: Path | None = target_context[2] + if ( + first_compile + and bridge is not None + and compile_cache_dir is not None + and project_dir is not None + and native_stage_enabled(NativeStage.MACRO_CALL_STORE) + ): + bridge.attach_store( + cache_dir=compile_cache_dir, + project_dir=project_dir, + model_paths=_model_paths(discovered_inputs=discovered_inputs, project_dir=project_dir), + ) + return target_context + + +def _model_paths(*, discovered_inputs: DiscoveredProjectInputs, project_dir: Path) -> list[str]: + paths: list[str] = [] + for model_file in discovered_inputs.model_files: + try: + paths.append(model_file.file_path.relative_to(project_dir).as_posix()) + except ValueError: + continue + return paths diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py new file mode 100644 index 0000000000..6c46bb639c --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -0,0 +1,202 @@ +"""Expand one authored string through the native macro bridge, executing misses in Python.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.render.macros import ( + _evaluate_macro_call, + _expand_resolved_sql_macros, + _expansion_declarations, + _record_macro_declaration_usage, + _record_macro_use, + _report_generated_references, +) +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts +from sqlbuild.compiler.compile.constants import MACRO_TOKEN +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + ExpansionSpan, + MacroExpansionState, +) +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ( + ARGUMENT_REFERENCE_EVENT, + DECLARATION_READ_EVENT, + GENERATED_SQL_EVENT, + MACRO_USE_EVENT, +) +from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.compiler.scopes.types import DeclarationKind +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind + + +def expand_bridged_sql_macros( + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + bridge: MacroBridge, +) -> tuple[str, tuple[ExpansionSpan, ...]]: + """Expand one string, replaying memoized calls and running the rest in Python in order.""" + + if MACRO_TOKEN not in sql: + return sql, () + declarations: DeclarationResolutionContext | None = _expansion_declarations( + state=state, consumer_path=consumer_path + ) + sites: tuple[MacroCallSite, ...] | None = bridge.scan(sql) + call_classes: list[MacroCallClass] = [] + for site in sites or (): + call_class: MacroCallClass | None = bridge.call_class( + site=site, + declarations=declarations, + loaded_macros=state.loaded_macros, + macro_context=state.macro_context, + ) + if call_class is None: + sites = None + break + call_classes.append(call_class) + if sites is None: + return _expand_resolved_sql_macros( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=declarations, + stack=(), + ) + outputs: list[str] = [ + _bridged_call_output( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=declarations, + bridge=bridge, + site=site, + call_class=call_class, + ) + for site, call_class in zip(sites, call_classes, strict=True) + ] + return bridge.splice(sql=sql, sites=sites, outputs=outputs) + + +def _bridged_call_output( # noqa: PLR0913 + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + bridge: MacroBridge, + site: MacroCallSite, + call_class: MacroCallClass, +) -> str: + facts: MacroExpansionFacts = state.facts + call_text: str = sql[site.start : site.end] + prior_relations: tuple[SqlResourceRef, ...] | None = ( + tuple(facts.relations) if site.typed_reference_text else None + ) + class_id: int + recorded: MacroCallRecord | None + class_id, recorded = bridge.lookup( + call_class=call_class, call_text=call_text, prior_relations=prior_relations + ) + if recorded is not None: + recorded_sql, relations, events = recorded + for event in events: + _replay_macro_call_event( + event=event, + consumer_path=consumer_path, + state=state, + declarations=declarations, + ) + for relation in relations: + _ = facts.relations.setdefault(relation, len(facts.relations)) + return facts.render_relation_placeholders(recorded_sql) + known_relations: int = len(facts.relations) + _ = facts.begin_event_log() + try: + macro_result: object + next_index: int + macro_result, next_index = _evaluate_macro_call( + sql=sql, + call_start_index=site.start, + file_path=consumer_path, + state=state, + declarations=declarations, + stack=(), + top_level=True, + ) + finally: + events, replayable = facts.end_event_log() + if not isinstance(macro_result, str): + raise CompileInputError( + f"Macro '@{site.name}' in '{consumer_path}' must return a SQL string when used " + "directly in SQL" + ) + if next_index != site.end: + raise NativeStageMismatchError( + f"Native macro call scan of '{consumer_path}' ended at {site.end}, Python at " + f"{next_index}" + ) + added_relations: tuple[SqlResourceRef, ...] = tuple(facts.relations)[known_relations:] + if replayable and (prior_relations is not None or not added_relations): + bridge.record( + class_id=class_id, + call_text=call_text, + sql=macro_result, + relations=added_relations, + events=events, + ) + return facts.render_relation_placeholders(macro_result) + + +def _replay_macro_call_event( + *, + event: MacroCallEvent, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, +) -> None: + tag, first, second = event + if tag == MACRO_USE_EVENT: + _record_macro_use( + macro_name=first, + identity=( + declarations.macro_records[first].identity + if declarations is not None + else DeclarationIdentity(kind=DeclarationKind.MACRO, name=first) + ), + state=state, + declarations=declarations, + stack=(), + ) + elif tag == DECLARATION_READ_EVENT: + if declarations is None: + raise NativeStageMismatchError("Replayed a declaration read without declarations") + _record_macro_declaration_usage( + name=second, kind=DeclarationKind(first), declarations=declarations, state=state + ) + elif tag == GENERATED_SQL_EVENT: + _report_generated_references( + macro_name=first, + loaded_macro=( + declarations.macros[first] + if declarations is not None + else state.loaded_macros[first] + ), + macro_result=second, + file_path=consumer_path, + state=state, + ) + elif tag == ARGUMENT_REFERENCE_EVENT: + state.facts.record_call_site_refs( + (SqlResourceRef(kind=SqlResourceRefKind(first), name=second),) + ) + else: + raise NativeStageMismatchError(f"Unknown macro call event {tag}") diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index b71c5ef5a6..f011f320a0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -11,7 +11,7 @@ import threading from collections.abc import Callable, Iterator, Mapping from contextvars import Token -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, replace from pathlib import Path from types import MappingProxyType, ModuleType from typing import cast @@ -19,12 +19,10 @@ from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( call_site_sql_references, evaluate_typed_reference, - reference_call_text, reject_macro_generated_references, - relation_placeholder_text, - render_relation_placeholders, report_macro_reference_call_syntax, ) +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import ( DECLARATION_REFERENCE_NAMES, MACRO_CONTEXT_PARAMETER_NAME, @@ -41,6 +39,7 @@ LoadedMacro, MacroContext, MacroExpansionResult, + MacroExpansionState, StaticMacroExport, StaticMacroFault, StaticMacroInventory, @@ -50,6 +49,14 @@ DiscoveredMacroFile, EnumDeclaration, ) +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ( + DECLARATION_READ_EVENT, + GENERATED_REFERENCE_MARKERS, + GENERATED_SQL_EVENT, + MACRO_USE_EVENT, +) +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) @@ -104,83 +111,6 @@ class _MacroAnalysisInputs: package_names: frozenset[str] -@dataclass -class _ExpansionFacts: - dependencies: list[DeclarationIdentity] = field(default_factory=list) - usages: list[UsageRecord] = field(default_factory=list) - relations: dict[SqlResourceRef, int] = field(default_factory=dict) - argument_references: dict[SqlResourceRef, None] = field(default_factory=dict) - call_site_refs: list[set[SqlResourceRef]] = field(default_factory=list) - rendering_approved: list[frozenset[SqlResourceRef]] = field(default_factory=list) - - def add_dependency(self, identity: DeclarationIdentity) -> _ExpansionFacts: - """Record a resolved dependency in encounter order.""" - - self.dependencies.append(identity) - return self - - def add_usage(self, usage: UsageRecord) -> _ExpansionFacts: - """Record a resolved macro-to-macro edge in encounter order.""" - - self.usages.append(usage) - return self - - def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None: - """Record references written as arguments of every macro call being expanded.""" - - self.argument_references.update(dict.fromkeys(refs)) - for approved in self.call_site_refs: - approved.update(refs) - - def open_call_site(self) -> _ExpansionFacts: - """Start collecting the references written as one macro call's arguments.""" - - self.call_site_refs.append(set()) - return self - - def close_call_site(self) -> frozenset[SqlResourceRef]: - """Stop collecting for the innermost macro call and return its argument references.""" - - return frozenset(self.call_site_refs.pop()) - - def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> _ExpansionFacts: - """Approve the call-site argument references of the macro about to run.""" - - self.rendering_approved.append(approved) - return self - - def end_rendering(self) -> _ExpansionFacts: - """Restore the approvals of the enclosing macro after one macro returns.""" - - self.rendering_approved.pop() - return self - - def render_relation(self, ref: object) -> str: - """Render a reference a macro formats; only call-site arguments become placeholders.""" - - if not isinstance(ref, SqlResourceRef): - raise CompileInputError("Only typed resource references render as relations") - if not self.rendering_approved or ref not in self.rendering_approved[-1]: - return reference_call_text(ref) - return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations))) - - def render_relation_placeholders(self, sql: str) -> str: - """Replace typed reference placeholders with the reference call written at the call site.""" - - return render_relation_placeholders(sql=sql, relations=self.relations) - - -@dataclass(frozen=True) -class _ExpansionState: - loaded_macros: dict[str, LoadedMacro] - macro_overrides: dict[str, str] - macro_context: MacroContext - declaration_resolver: DeclarationScopeResolver | None - declarations: DeclarationResolutionContext | None - facts: _ExpansionFacts - consumer: ResourceIdentity | DeclarationIdentity | None = None - - class _MacroDeclarationValues(Mapping[str, object]): def __init__( self, @@ -190,17 +120,20 @@ def __init__( declaration_kind: str, file_path: Path, on_access: Callable[[str], None], + on_miss: Callable[[], None], ) -> None: self._values: Mapping[str, object] = values self._inaccessible: Mapping[str, DeclarationRecord] = inaccessible self._declaration_kind: str = declaration_kind self._file_path: Path = file_path self._on_access: Callable[[str], None] = on_access + self._on_miss: Callable[[], None] = on_miss def __getitem__(self, name: str) -> object: if name in self._values: self._on_access(name) return self._values[name] + self._on_miss() inaccessible: DeclarationRecord | None = self._inaccessible.get(name) if inaccessible is not None: owner: str = inaccessible.owning_path or "global" @@ -227,14 +160,23 @@ def __len__(self) -> int: class _MacroEnumMembers(Mapping[str, str | int]): - def __init__(self, *, enum_name: str, values: Mapping[str, str | int], file_path: Path) -> None: + def __init__( + self, + *, + enum_name: str, + values: Mapping[str, str | int], + file_path: Path, + on_miss: Callable[[], None], + ) -> None: self._enum_name: str = enum_name self._values: Mapping[str, str | int] = values self._file_path: Path = file_path + self._on_miss: Callable[[], None] = on_miss def __getitem__(self, name: str) -> str | int: if name in self._values: return self._values[name] + self._on_miss() available: str = ", ".join(sorted(self._values)) or "none" raise MacroDeclarationLookupError( f"Unknown member '{name}' for enum '{self._enum_name}' in '{self._file_path}'. " @@ -1256,8 +1198,8 @@ def expand_sql_macros_result( ) -> MacroExpansionResult: """Expand macros and retain deterministic resolved dependency facts.""" - facts: _ExpansionFacts = _ExpansionFacts() - state: _ExpansionState = _ExpansionState( + facts: MacroExpansionFacts = MacroExpansionFacts() + state: MacroExpansionState = MacroExpansionState( loaded_macros=loaded_macros, macro_overrides={} if macro_overrides is None else macro_overrides, macro_context=macro_context, @@ -1266,11 +1208,21 @@ def expand_sql_macros_result( facts=facts, consumer=consumer, ) - expanded_sql, spans = _expand_sql_macros( - sql=sql, - consumer_path=file_path, - state=state, - stack=(), + bridge: MacroBridge | None = active_macro_bridge() if not state.macro_overrides else None + expanded_sql, spans = ( + _expand_sql_macros( + sql=sql, + consumer_path=file_path, + state=state, + stack=(), + ) + if bridge is None + else _bridged_sql_macros( + sql=sql, + consumer_path=file_path, + state=state, + bridge=bridge, + ) ) return MacroExpansionResult( sql=expanded_sql, @@ -1281,16 +1233,39 @@ def expand_sql_macros_result( ) +def _bridged_sql_macros( + *, sql: str, consumer_path: Path, state: MacroExpansionState, bridge: MacroBridge +) -> tuple[str, tuple[ExpansionSpan, ...]]: + from sqlbuild.compiler.compile._helpers.macro_bridge.expansion import ( + expand_bridged_sql_macros, + ) + + return expand_bridged_sql_macros( + sql=sql, consumer_path=consumer_path, state=state, bridge=bridge + ) + + def _expand_sql_macros( *, sql: str, consumer_path: Path, - state: _ExpansionState, + state: MacroExpansionState, stack: tuple[DeclarationIdentity, ...], ) -> tuple[str, tuple[ExpansionSpan, ...]]: if MACRO_TOKEN not in sql: return sql, () + return _expand_resolved_sql_macros( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=_expansion_declarations(state=state, consumer_path=consumer_path), + stack=stack, + ) + +def _expansion_declarations( + *, state: MacroExpansionState, consumer_path: Path +) -> DeclarationResolutionContext | None: declarations: DeclarationResolutionContext | None = state.declarations if declarations is None and state.declaration_resolver is not None: from sqlbuild.compiler.compile._helpers.render.declarations import ( @@ -1300,7 +1275,17 @@ def _expand_sql_macros( declarations = resolve_declaration_context( resolver=state.declaration_resolver, file_path=consumer_path ) + return declarations + +def _expand_resolved_sql_macros( + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + stack: tuple[DeclarationIdentity, ...], +) -> tuple[str, tuple[ExpansionSpan, ...]]: rendered_sql_parts: list[str] = [] spans: list[ExpansionSpan] = [] output_length: int = 0 @@ -1399,7 +1384,7 @@ def _evaluate_macro_call( sql: str, call_start_index: int, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], top_level: bool, @@ -1452,29 +1437,13 @@ def _evaluate_macro_call( raise CompileInputError( f"Macro expansion cycle detected: {chain}. Definition paths: {paths}" ) - _ = state.facts.add_dependency(identity) - if stack: - _ = state.facts.add_usage( - UsageRecord(stack[-1], identity, UsageKind.DECLARATION_DEPENDENCY), - ) - elif state.consumer is not None: - visibility: tuple[VisibilityRecord, ...] = ( - declarations.macro_visibility.get(macro_name, ()) if declarations is not None else () - ) - if visibility: - from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility - - for visible in usage_visibility(visibility=visibility, consumer=state.consumer): - _ = state.facts.add_usage( - UsageRecord( - state.consumer, - identity, - UsageKind.RUNTIME, - through=visible.through, - ) - ) - else: - _ = state.facts.add_usage(UsageRecord(state.consumer, identity, UsageKind.RUNTIME)) + _record_macro_use( + macro_name=macro_name, + identity=identity, + state=state, + declarations=declarations, + stack=stack, + ) args_source: str = sql[opening_paren_index + 1 : closing_paren_index] args: tuple[object, ...] kwargs: dict[str, object] @@ -1528,34 +1497,88 @@ def _evaluate_macro_call( _validate_final_macro_sql( macro_name=macro_name, file_path=file_path, macro_result=macro_result ) - report_macro_reference_call_syntax( + _report_generated_references( + macro_name=macro_name, loaded_macro=loaded_macro, macro_result=macro_result, file_path=file_path, - consumer=state.consumer, + state=state, ) - if state.macro_context._enforce_explicit_references: - reject_macro_generated_references( - loaded_macro=loaded_macro, - macro_result=macro_result, - file_path=file_path, - consumer=state.consumer, - ) return macro_result, closing_paren_index + 1 +def _record_macro_use( + *, + macro_name: str, + identity: DeclarationIdentity, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + stack: tuple[DeclarationIdentity, ...], +) -> None: + state.facts.note_event((MACRO_USE_EVENT, macro_name, "")) + _ = state.facts.add_dependency(identity) + if stack: + state.facts.block_memo() + _ = state.facts.add_usage( + UsageRecord(stack[-1], identity, UsageKind.DECLARATION_DEPENDENCY), + ) + elif state.consumer is not None: + visibility: tuple[VisibilityRecord, ...] = ( + declarations.macro_visibility.get(macro_name, ()) if declarations is not None else () + ) + if visibility: + from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility + + for visible in usage_visibility(visibility=visibility, consumer=state.consumer): + _ = state.facts.add_usage( + UsageRecord( + state.consumer, + identity, + UsageKind.RUNTIME, + through=visible.through, + ) + ) + else: + _ = state.facts.add_usage(UsageRecord(state.consumer, identity, UsageKind.RUNTIME)) + + +def _report_generated_references( + *, + macro_name: str, + loaded_macro: LoadedMacro, + macro_result: str, + file_path: Path, + state: MacroExpansionState, +) -> None: + if any(marker in macro_result for marker in GENERATED_REFERENCE_MARKERS): + state.facts.note_event((GENERATED_SQL_EVENT, macro_name, macro_result)) + report_macro_reference_call_syntax( + loaded_macro=loaded_macro, + macro_result=macro_result, + file_path=file_path, + consumer=state.consumer, + ) + if state.macro_context._enforce_explicit_references: + reject_macro_generated_references( + loaded_macro=loaded_macro, + macro_result=macro_result, + file_path=file_path, + consumer=state.consumer, + ) + + def _build_macro_invocation_context( *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, ) -> MacroContext: if declarations is None: return macro_context constant_values: Mapping[str, object] = _build_constant_context_values(declarations.constants) enum_values: Mapping[str, object] = _build_enum_context_values( - declarations=declarations.enums, file_path=file_path + declarations=declarations.enums, file_path=file_path, on_miss=state.facts.block_memo ) constants: Mapping[str, object] = _MacroDeclarationValues( values=constant_values, @@ -1568,6 +1591,7 @@ def _build_macro_invocation_context( declarations=declarations, state=state, ), + on_miss=state.facts.block_memo, ) enum_mapping: Mapping[str, object] = _MacroDeclarationValues( values=enum_values, @@ -1580,6 +1604,7 @@ def _build_macro_invocation_context( declarations=declarations, state=state, ), + on_miss=state.facts.block_memo, ) return replace( macro_context, @@ -1602,6 +1627,7 @@ def _build_enum_context_values( *, declarations: Mapping[str, EnumDeclaration], file_path: Path, + on_miss: Callable[[], None], ) -> Mapping[str, object]: values: dict[str, object] = {} for name, declaration in declarations.items(): @@ -1612,6 +1638,7 @@ def _build_enum_context_values( enum_name=name, values=MappingProxyType(members), file_path=file_path, + on_miss=on_miss, ) return MappingProxyType(values) @@ -1660,10 +1687,11 @@ def _record_macro_declaration_usage( name: str, kind: DeclarationKind, declarations: DeclarationResolutionContext, - state: _ExpansionState, + state: MacroExpansionState, ) -> None: from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility + state.facts.note_event((DECLARATION_READ_EVENT, kind.value, name)) consumer: ResourceIdentity | DeclarationIdentity | None = ( state.consumer or declarations.consumer ) @@ -1738,7 +1766,7 @@ def _parse_macro_arguments( *, args_source: str, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], ) -> tuple[tuple[object, ...], dict[str, object]]: @@ -1796,7 +1824,7 @@ def _rewrite_nested_macro_calls( *, args_source: str, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], ) -> tuple[str, dict[str, object]]: diff --git a/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py b/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py new file mode 100644 index 0000000000..e59c882b93 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py @@ -0,0 +1,115 @@ +"""Facts recorded while expanding the macros of one authored SQL string.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.macro_bridge.constants import ARGUMENT_REFERENCE_EVENT +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent +from sqlbuild.compiler.scopes.models import DeclarationIdentity, UsageRecord +from sqlbuild.python_nodes.models import SqlResourceRef + + +class MacroExpansionFacts: + """Facts one authored string's macro expansion records, in encounter order.""" + + def __init__(self) -> None: + self.dependencies: list[DeclarationIdentity] = [] + self.usages: list[UsageRecord] = [] + self.relations: dict[SqlResourceRef, int] = {} + self.argument_references: dict[SqlResourceRef, None] = {} + self.call_site_refs: list[set[SqlResourceRef]] = [] + self.rendering_approved: list[frozenset[SqlResourceRef]] = [] + self.event_log: list[MacroCallEvent] | None = None + self.memo_blocked: bool = False + + def begin_event_log(self) -> MacroExpansionFacts: + """Start recording the consumer-independent facts of one top-level macro call.""" + + self.event_log = [] + self.memo_blocked = False + return self + + def end_event_log(self) -> tuple[tuple[MacroCallEvent, ...], bool]: + """Stop recording and return the facts and whether the call may be replayed.""" + + events: tuple[MacroCallEvent, ...] = tuple(self.event_log or ()) + self.event_log = None + return events, not self.memo_blocked + + def note_event(self, event: MacroCallEvent) -> None: + """Record one fact of the macro call being executed, when recording.""" + + if self.event_log is not None: + self.event_log.append(event) + + def block_memo(self) -> None: + """Mark the executing call as dependent on its consumer, so it is never replayed.""" + + self.memo_blocked = True + + def add_dependency(self, identity: DeclarationIdentity) -> MacroExpansionFacts: + """Record a resolved dependency in encounter order.""" + + self.dependencies.append(identity) + return self + + def add_usage(self, usage: UsageRecord) -> MacroExpansionFacts: + """Record a resolved macro-to-macro edge in encounter order.""" + + self.usages.append(usage) + return self + + def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None: + """Record references written as arguments of every macro call being expanded.""" + + for ref in refs: + self.note_event((ARGUMENT_REFERENCE_EVENT, ref.kind.value, ref.name)) + self.argument_references.update(dict.fromkeys(refs)) + for approved in self.call_site_refs: + approved.update(refs) + + def open_call_site(self) -> MacroExpansionFacts: + """Start collecting the references written as one macro call's arguments.""" + + self.call_site_refs.append(set()) + return self + + def close_call_site(self) -> frozenset[SqlResourceRef]: + """Stop collecting for the innermost macro call and return its argument references.""" + + return frozenset(self.call_site_refs.pop()) + + def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> MacroExpansionFacts: + """Approve the call-site argument references of the macro about to run.""" + + self.rendering_approved.append(approved) + return self + + def end_rendering(self) -> MacroExpansionFacts: + """Restore the approvals of the enclosing macro after one macro returns.""" + + self.rendering_approved.pop() + return self + + def render_relation(self, ref: object) -> str: + """Render a reference a macro formats; only call-site arguments become placeholders.""" + + from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( + reference_call_text, + relation_placeholder_text, + ) + + if not isinstance(ref, SqlResourceRef): + raise CompileInputError("Only typed resource references render as relations") + if not self.rendering_approved or ref not in self.rendering_approved[-1]: + return reference_call_text(ref) + return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations))) + + def render_relation_placeholders(self, sql: str) -> str: + """Replace typed reference placeholders with the reference call written at the call site.""" + + from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( + render_relation_placeholders, + ) + + return render_relation_placeholders(sql=sql, relations=self.relations) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index ebee19a4b6..2b857fdbdb 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) -from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) @@ -31,6 +30,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( missing_description_diagnostics, ) +from sqlbuild.compiler.compile._helpers.macro_bridge.call_store import ( + target_context_with_macro_call_store, +) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -93,10 +95,12 @@ def build_compile_inputs( effective_target_name: str | None effective_target: TargetConfig | None compile_cache_dir: Path | None - effective_target_name, effective_target, compile_cache_dir = build_compile_target_context( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - no_cache=no_cache, + effective_target_name, effective_target, compile_cache_dir = ( + target_context_with_macro_call_store( + discovered_inputs=discovered_inputs, + selected_target=selected_target, + no_cache=no_cache, + ) ) effective_vars: dict[str, object] = build_effective_vars( diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index c924443f2a..7d54966f76 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -12,6 +12,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( @@ -388,6 +389,19 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: return render_enum_member_value(value=value) +@dataclass(frozen=True) +class MacroExpansionState: + """Inputs and recorded facts of one authored string's macro expansion.""" + + loaded_macros: dict[str, LoadedMacro] + macro_overrides: dict[str, str] + macro_context: MacroContext + declaration_resolver: DeclarationScopeResolver | None + declarations: DeclarationResolutionContext | None + facts: MacroExpansionFacts + consumer: ResourceIdentity | DeclarationIdentity | None = None + + @dataclass(frozen=True) class MacroExpansionResult: """Expanded SQL and resolved macro usage facts from one authored string.""" diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py index 13927c6920..031f90ca47 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -2,7 +2,9 @@ from collections.abc import Callable -from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import CompilerStage, NativeStage +from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge def native_frontier[T]( @@ -13,5 +15,10 @@ def native_frontier[T]( ) -> T: """Produce the frontier object for `until` natively; stages not yet native run in Python.""" - _ = until - return python_stage() if native_stage is None else native_stage() + if native_stage is not None: + return native_stage() + if until is CompilerStage.COMPILE_PROJECT_INPUTS and native_stage_enabled( + NativeStage.MACRO_CALLS + ): + return run_with_macro_bridge(stage=python_stage) + return python_stage() diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 83909b5197..b10c7aab0c 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -22,6 +22,8 @@ NativeStage.REFERENCE_EXTRACTION: NativeStageTier.PREVIEW, NativeStage.DECLARATION_FILES: NativeStageTier.PREVIEW, NativeStage.MODEL_LOOP: NativeStageTier.PREVIEW, + NativeStage.MACRO_CALLS: NativeStageTier.PREVIEW, + NativeStage.MACRO_CALL_STORE: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 82a422b4d2..b1dd2339c0 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -20,6 +20,8 @@ class NativeStage(StrEnum): REFERENCE_EXTRACTION = "reference_extraction" DECLARATION_FILES = "declaration_files" MODEL_LOOP = "model_loop" + MACRO_CALLS = "macro_calls" + MACRO_CALL_STORE = "macro_call_store" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/macro_bridge/__init__.py b/src/sqlbuild/compiler/macro_bridge/__init__.py new file mode 100644 index 0000000000..73c3a31a12 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/__init__.py @@ -0,0 +1 @@ +"""Native macro bridge: call-site scans, the in-compile call memo and splicing.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py new file mode 100644 index 0000000000..a88eb85027 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py @@ -0,0 +1 @@ +"""Macro call store keys and the environment they are valid in.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py new file mode 100644 index 0000000000..67bbca6d4d --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -0,0 +1,216 @@ +"""The environment stored macro call results are valid in: build, interpreter, code and files.""" + +from __future__ import annotations + +import functools +import importlib.machinery +import json +import os +import sys +import sysconfig +from collections.abc import Iterable +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.macro_bridge.constants import ( + INSTALLED_DISTRIBUTION_SUFFIX, + INSTALLED_PACKAGE_DIRECTORY_NAMES, + INSTALLED_RECORD_FILE_NAME, + INTERPRETER_MODULE_ORIGINS, + MACRO_CALL_STORE_ENVIRONMENT_VERSION, + MISSING_SEARCH_PATH_STAMP, + MODULE_DIGEST_ROW_FIELDS, + PROJECT_ROOT_SEARCH_PATH_STAMP, + STORE_TRACKED_ENVIRONMENT_PREFIXES, + STORE_UNTRACKED_ENVIRONMENT_NAMES, +) +from sqlbuild.compiler.macro_bridge.models import ModuleSources + + +def project_fingerprint(*, project_dir: Path, model_paths: list[str]) -> str: + """Digest every project file but model SQL files; OSError if a file is unreadable.""" + + return _native.fingerprint_project_files(str(project_dir), model_paths) + + +def store_environment(*, project_dir: Path, fingerprint: str) -> str: + """Digest the build, interpreter, import path, installed packages, variables and project.""" + + return _native.content_digest( + [ + MACRO_CALL_STORE_ENVIRONMENT_VERSION, + _native.BUILD_IDENTITY, + sys.version, + sys.executable, + sys.platform, + str(sys.implementation.cache_tag), + json.dumps(_search_path_stamps(project_dir=project_dir)), + json.dumps(_tracked_environment()), + json.dumps(_installed_distribution_records()), + fingerprint, + ] + ) + + +def module_sources(modules: Iterable[object]) -> ModuleSources: + """Files backing `modules` outside the interpreter and SQLBuild, and whether all have one.""" + + paths: list[str] = [] + for module in modules: + path: str | None + known: bool + path, known = _backing_file(module) + if not known: + return ModuleSources(paths=(), complete=False) + if path is not None and not _covered_without_content(path): + paths.append(path) + return ModuleSources(paths=tuple(paths), complete=True) + + +def digest_module_files(paths: Iterable[str]) -> dict[str, str] | None: + """Content digest of each module file, or None when any file cannot be read.""" + + ordered: list[str] = list(dict.fromkeys(paths)) + digests: list[str | None] = _native.digest_files(ordered) + if any(digest is None for digest in digests): + return None + return {path: digest for path, digest in zip(ordered, digests, strict=True) if digest} + + +def module_digests_metadata(digests: dict[str, str]) -> bytes: + """Encode module file digests for the store's metadata.""" + + return json.dumps(sorted([path, digest] for path, digest in digests.items())).encode() + + +def unchanged_module_digests(*, metadata: bytes, known: dict[str, str]) -> dict[str, str] | None: + """Return the stored module digests when every module file still has them, else None.""" + + try: + rows: object = json.loads(metadata) + except ValueError: + return None + if not isinstance(rows, list) or not all(map(_is_digest_row, rows)): + return None + stored: dict[str, str] = {row[0]: row[1] for row in rows} + unknown: list[str] = [path for path in stored if path not in known] + current: dict[str, str] | None = digest_module_files(unknown) + if current is None: + return None + current.update((path, known[path]) for path in stored if path in known) + return stored if current == stored else None + + +def _is_digest_row(row: object) -> bool: + return ( + isinstance(row, list) + and len(row) == MODULE_DIGEST_ROW_FIELDS + and all(isinstance(field, str) for field in row) + ) + + +def _backing_file(module: object) -> tuple[str | None, bool]: + spec: object = getattr(module, "__spec__", None) + file: object = getattr(module, "__file__", None) + if spec is None: + return (file, True) if isinstance(file, str) and os.path.isfile(file) else (None, True) + origin: object = getattr(spec, "origin", None) + if getattr(spec, "has_location", False) and isinstance(origin, str): + return origin, True + if origin in INTERPRETER_MODULE_ORIGINS: + return None, True + if ( + origin is None + and file is None + and getattr(spec, "submodule_search_locations", None) is not None + ): + return None, True + return None, False + + +def _covered_without_content(path: str) -> bool: + installed, interpreter, own = _excluded_prefixes() + if path.startswith(own): + return True + if path.startswith(installed): + return path.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)) + return path.startswith(interpreter) + + +@functools.cache +def _excluded_prefixes() -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + return ( + _directory_prefixes(_installed_directories()), + _directory_prefixes(sysconfig.get_paths().get(name) for name in ("stdlib", "platstdlib")), + _directory_prefixes([os.path.dirname(_native.__file__)]), + ) + + +@functools.cache +def _installed_directories() -> tuple[str, ...]: + paths: dict[str, str] = sysconfig.get_paths() + return tuple( + dict.fromkeys( + [ + *(paths[name] for name in ("purelib", "platlib") if name in paths), + *( + entry + for entry in sys.path + if os.path.basename(entry) in INSTALLED_PACKAGE_DIRECTORY_NAMES + ), + ] + ) + ) + + +def _installed_distribution_records() -> list[tuple[str, str | None]]: + records: list[str] = [] + for directory in _installed_directories(): + records.extend(_distribution_records(directory)) + records.sort() + return list(zip(records, _native.digest_files(records), strict=True)) + + +def _distribution_records(directory: str) -> list[str]: + try: + with os.scandir(directory) as entries: + return [ + os.path.join(entry.path, INSTALLED_RECORD_FILE_NAME) + for entry in entries + if entry.name.endswith(INSTALLED_DISTRIBUTION_SUFFIX) + ] + except OSError: + return [] + + +def _directory_prefixes(directories: Iterable[str | None]) -> tuple[str, ...]: + prefixes: list[str] = [] + for directory in filter(None, directories): + prefixes.extend(os.path.join(form, "") for form in (directory, os.path.realpath(directory))) + return tuple(dict.fromkeys(prefixes)) + + +def _search_path_stamps(*, project_dir: Path) -> list[tuple[str, int]]: + project_root: str = os.path.realpath(project_dir) + stamps: list[tuple[str, int]] = [] + for entry in sys.path: + path: str = entry or "." + try: + mtime_ns: int = ( + PROJECT_ROOT_SEARCH_PATH_STAMP + if os.path.realpath(path) == project_root + else os.stat(path).st_mtime_ns + ) + except OSError: + mtime_ns = MISSING_SEARCH_PATH_STAMP + stamps.append((entry, mtime_ns)) + return stamps + + +def _tracked_environment() -> list[tuple[str, str]]: + return sorted( + (name, value) + for name, value in os.environ.items() + if name.startswith(STORE_TRACKED_ENVIRONMENT_PREFIXES) + and name not in STORE_UNTRACKED_ENVIRONMENT_NAMES + ) diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py new file mode 100644 index 0000000000..f507172914 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py @@ -0,0 +1,143 @@ +"""Value-based text of everything a stored macro call result depends on besides its call text.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import json +import types +from pathlib import PurePath + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + LoadedMacro, + MacroContext, +) +from sqlbuild.compiler.macro_bridge.constants import VALUE_RENDERER_FIELD +from sqlbuild.compiler.macro_bridge.exceptions import UncacheableValueError +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.python_nodes.models import SqlResourceRef + + +def macro_store_token(*, loaded: LoadedMacro, identity: DeclarationIdentity | None) -> str: + """Name a macro by its name, file, declaration identity and exact source.""" + + return _key_text( + [ + loaded.name, + loaded.relative_path.as_posix(), + _canonical(identity), + _native.content_digest([loaded.raw_source]), + ] + ) + + +def context_store_token( + *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None +) -> str | None: + """Digest the whole context a macro may read, or None when a value has no stable text.""" + + try: + text: str = _key_text( + [ + _canonical_context(macro_context), + None + if declarations is None + else [ + _canonical(declarations.constants), + _canonical(declarations.enums), + sorted(declarations.inaccessible_constants), + sorted(declarations.inaccessible_enums), + ], + ] + ) + except UncacheableValueError: + return None + return _native.content_digest([text]) + + +def call_class_store_text( + *, + macro_tokens: tuple[str, ...], + context_token: str | None, + prior_relations: tuple[SqlResourceRef, ...] | None, +) -> str: + """Text of one persistent call class: its macros, context and relations passed so far.""" + + return _key_text( + [ + list(macro_tokens), + context_token, + None + if prior_relations is None + else [[relation.kind.value, relation.name] for relation in prior_relations], + ] + ) + + +def _canonical_context(macro_context: MacroContext) -> object: + fields: list[object] = [] + for field in dataclasses.fields(macro_context): + value: object = getattr(macro_context, field.name) + fields.append( + [ + field.name, + _type_name(type(value)) + if field.name == VALUE_RENDERER_FIELD + else _canonical(value), + ] + ) + return [_type_name(type(macro_context)), fields] + + +def _canonical(value: object) -> object: # noqa: PLR0911 + if isinstance(value, enum.Enum): + return ["enum", _type_name(type(value)), _canonical(value.value)] + if value is None or isinstance(value, bool | str): + return value + if isinstance(value, int): + return ["int", str(value)] + if isinstance(value, float): + return ["float", value.hex()] + if isinstance(value, decimal.Decimal): + return ["decimal", str(value)] + if isinstance(value, datetime.date | datetime.time): + return [_type_name(type(value)), value.isoformat()] + if isinstance(value, datetime.timedelta): + return ["timedelta", value.days, value.seconds, value.microseconds] + if isinstance(value, bytes): + return ["bytes", value.hex()] + if isinstance(value, PurePath): + return ["path", value.as_posix()] + if isinstance(value, list | tuple): + return [_type_name(type(value)), [_canonical(item) for item in value]] + if isinstance(value, set | frozenset): + return [ + _type_name(type(value)), + sorted(json.dumps(_canonical(item), sort_keys=True) for item in value), + ] + if isinstance(value, dict | types.MappingProxyType): + return [ + "mapping", + [[_canonical(key), _canonical(item)] for key, item in value.items()], + ] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return [ + _type_name(type(value)), + [ + [field.name, _canonical(getattr(value, field.name))] + for field in dataclasses.fields(value) + ], + ] + raise UncacheableValueError(_type_name(type(value))) + + +def _type_name(kind: type) -> str: + return f"{kind.__module__}.{kind.__qualname__}" + + +def _key_text(value: object) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) diff --git a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py new file mode 100644 index 0000000000..2e8ae1fd31 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py @@ -0,0 +1 @@ +"""Macro bridge runtime classes.""" diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py new file mode 100644 index 0000000000..b073556bc5 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -0,0 +1,333 @@ +"""Per-compile state of the native macro bridge: call-site scans, call classes and the memo.""" + +from __future__ import annotations + +import logging +import sys +from collections.abc import Hashable, Mapping +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + ExpansionSpan, + LoadedMacro, + MacroContext, +) +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + digest_module_files, + module_digests_metadata, + module_sources, + project_fingerprint, + store_environment, + unchanged_module_digests, +) +from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( + call_class_store_text, + context_store_token, + macro_store_token, +) +from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME +from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite, ModuleSources +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind + + +class MacroBridge: + """Scan, memoize and splice macro calls for one compile; Python still runs every macro.""" + + def __init__(self, *, python_version: tuple[int, int], unicode_version: str) -> None: + self._python_version: tuple[int, int] = python_version + self._unicode_version: str = unicode_version + self._memo: _native.MacroCallMemo = _native.MacroCallMemo() + self._class_ids: dict[Hashable, int] = {} + self._retained: dict[int, object] = {} + self._name_tokens: dict[tuple[int, str], tuple[Hashable, str | None] | None] = {} + self._context_tokens: dict[tuple[int, int], tuple[Hashable, str | None]] = {} + self._store_path: Path | None = None + self._store_trusted: bool = False + self._module_digests: dict[str, str] = {} + self._observed_modules: dict[str, object] = {} + self._project_inputs: tuple[Path, list[str], str] = (Path(), [], "") + + def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[str]) -> None: + """Reuse call results stored by earlier compiles of the same code and project files.""" + + if self._store_path is not None: + return + path: Path = cache_dir / MACRO_CALL_STORE_FILE_NAME + metadata: bytes + loaded: int + try: + if not self._observe_modules(): + logging.getLogger(__name__).debug("Macro call store not used: unknown module code") + return + fingerprint: str = project_fingerprint(project_dir=project_dir, model_paths=model_paths) + environment: str = store_environment(project_dir=project_dir, fingerprint=fingerprint) + metadata, loaded = self._memo.attach_store(str(path), environment) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("Macro call store not loaded: %s", error) + return + carried: dict[str, str] | None = ( + unchanged_module_digests(metadata=metadata, known=self._module_digests) + if loaded + else {} + ) + if carried is None: + self._memo.discard_store() + carried = {} + self._store_path = path + self._store_trusted = True + self._module_digests = {**carried, **self._module_digests} + self._project_inputs = (project_dir, model_paths, fingerprint) + self._name_tokens.clear() + self._context_tokens.clear() + + def save_store(self) -> None: + """Save the results this compile used or recorded, unless code or files it read changed.""" + + if self._store_path is None or not self._store_trusted: + return + project_dir, model_paths, fingerprint = self._project_inputs + try: + if ( + not self._observe_modules() + or digest_module_files(self._module_digests) != self._module_digests + or project_fingerprint(project_dir=project_dir, model_paths=model_paths) + != fingerprint + ): + logging.getLogger(__name__).debug( + "Macro call store not saved: code or files changed" + ) + return + _ = self._memo.save_store( + str(self._store_path), module_digests_metadata(self._module_digests) + ) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("Macro call store not saved: %s", error) + + def _observe_modules(self) -> bool: + """Digest the files of modules loaded since the last look; False when one is unknown.""" + + loaded: dict[str, object] = dict(sys.modules) + added: list[object] = [ + module + for name, module in loaded.items() + if self._observed_modules.get(name) is not module + ] + self._observed_modules = loaded + sources: ModuleSources = module_sources(added) + digests: dict[str, str] | None = ( + digest_module_files(sources.paths) if sources.complete else None + ) + if digests is None or any( + self._module_digests.get(path, digest) != digest for path, digest in digests.items() + ): + self._store_trusted = False + return False + self._module_digests.update(digests) + return True + + def scan(self, sql: str) -> tuple[MacroCallSite, ...] | None: + """Return the top-level call sites of `sql`, or None when Python must expand it.""" + + rows: list[tuple[int, int, str, list[str], bool]] | None = _native.scan_macro_call_sites( + sql, self._python_version, self._unicode_version + ) + if rows is None: + return None + return tuple( + MacroCallSite( + start=start, + end=end, + name=name, + tree_names=tuple(tree_names), + typed_reference_text=typed_reference_text, + ) + for start, end, name, tree_names, typed_reference_text in rows + ) + + def call_class( + self, + *, + site: MacroCallSite, + declarations: DeclarationResolutionContext | None, + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + ) -> MacroCallClass | None: + """Return what a call's result may depend on besides its text, or None if unresolvable.""" + + owner: object = loaded_macros if declarations is None else declarations + macros: Mapping[str, LoadedMacro] = ( + loaded_macros if declarations is None else declarations.macros + ) + _ = self._retained.setdefault(id(owner), owner) + _ = self._retained.setdefault(id(macro_context), macro_context) + tokens: list[Hashable] = [] + store_tokens: list[str | None] = [] + for name in site.tree_names: + token: tuple[Hashable, str | None] | None = self._name_token( + owner=owner, macros=macros, declarations=declarations, name=name + ) + if token is None: + return None + tokens.append(token[0]) + store_tokens.append(token[1]) + injects_context: bool = any(macros[name].injects_context for name in site.tree_names) + context: tuple[Hashable, str | None] | None = ( + self._context_token(owner=owner, declarations=declarations, macro_context=macro_context) + if injects_context + else None + ) + persistent_tokens: tuple[str, ...] = tuple( + token for token in store_tokens if token is not None + ) + return MacroCallClass( + key=(tuple(tokens), None if context is None else context[0]), + macro_store_tokens=persistent_tokens, + context_store_token=None if context is None else context[1], + persistent=len(persistent_tokens) == len(store_tokens) + and (context is None or context[1] is not None), + ) + + def lookup( + self, + *, + call_class: MacroCallClass, + call_text: str, + prior_relations: tuple[SqlResourceRef, ...] | None, + ) -> tuple[int, MacroCallRecord | None]: + """Return the class id of a call and its recorded or stored result, if any.""" + + class_key: Hashable = (call_class.key, prior_relations) + class_id: int | None = self._class_ids.get(class_key) + if class_id is None: + class_id = self._class_ids[class_key] = len(self._class_ids) + if self._store_path is not None and call_class.persistent: + self._memo.set_persistent_class( + class_id, + call_class_store_text( + macro_tokens=call_class.macro_store_tokens, + context_token=call_class.context_store_token, + prior_relations=prior_relations, + ), + ) + found: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None = ( + self._memo.lookup(class_id, call_text) + ) + if found is None: + return class_id, None + sql, relations, events = found + return class_id, ( + sql, + tuple( + SqlResourceRef(kind=SqlResourceRefKind(kind), name=name) for kind, name in relations + ), + tuple(events), + ) + + def record( + self, + *, + class_id: int, + call_text: str, + sql: str, + relations: tuple[SqlResourceRef, ...], + events: tuple[MacroCallEvent, ...], + ) -> None: + """Record the result of one executed call for later calls of the same class and text.""" + + if self._store_trusted and len(sys.modules) != len(self._observed_modules): + _ = self._observe_modules() + self._memo.record( + class_id, + call_text, + (sql, [(relation.kind.value, relation.name) for relation in relations], list(events)), + ) + + def splice( + self, *, sql: str, sites: tuple[MacroCallSite, ...], outputs: list[str] + ) -> tuple[str, tuple[ExpansionSpan, ...]]: + """Replace each call site with its output, returning the SQL and substitution spans.""" + + rendered: str + spans: list[tuple[int, int, int, int]] + rendered, spans = _native.splice_macro_calls( + sql, [(site.start, site.end) for site in sites], outputs + ) + return rendered, tuple( + ExpansionSpan( + source_start=source_start, + source_end=source_end, + output_start=output_start, + output_end=output_end, + ) + for source_start, source_end, output_start, output_end in spans + ) + + def stats(self) -> tuple[int, int, int]: + """Memo hits, misses and recorded results so far.""" + + return self._memo.stats() + + def store_stats(self) -> tuple[int, int]: + """Calls found in the store and calls recorded into it.""" + + return self._memo.store_stats() + + def _name_token( + self, + *, + owner: object, + macros: Mapping[str, LoadedMacro], + declarations: DeclarationResolutionContext | None, + name: str, + ) -> tuple[Hashable, str | None] | None: + key: tuple[int, str] = (id(owner), name) + if key in self._name_tokens: + return self._name_tokens[key] + loaded: LoadedMacro | None = macros.get(name) + token: tuple[Hashable, str | None] | None = None + if loaded is not None and ( + declarations is None or name not in declarations.inaccessible_macros + ): + identity: DeclarationIdentity | None = ( + declarations.macro_records[name].identity if declarations is not None else None + ) + token = ( + (name, id(loaded), identity), + macro_store_token(loaded=loaded, identity=identity) + if self._store_path is not None + else None, + ) + self._name_tokens[key] = token + return token + + def _context_token( + self, + *, + owner: object, + declarations: DeclarationResolutionContext | None, + macro_context: MacroContext, + ) -> tuple[Hashable, str | None]: + key: tuple[int, int] = (id(owner), id(macro_context)) + token: tuple[Hashable, str | None] | None = self._context_tokens.get(key) + if token is None: + token = ( + ( + id(macro_context), + None + if declarations is None + else ( + tuple((name, id(value)) for name, value in declarations.constants.items()), + tuple((name, id(value)) for name, value in declarations.enums.items()), + ), + ), + context_store_token(macro_context=macro_context, declarations=declarations) + if self._store_path is not None + else None, + ) + self._context_tokens[key] = token + return token diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py new file mode 100644 index 0000000000..50b84b4929 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -0,0 +1,35 @@ +"""Macro call event tags shared with the native memo, and the active bridge of a render.""" + +from __future__ import annotations + +import itertools +from contextvars import ContextVar + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild.compiler.references.types import SqlReferenceKind + +MACRO_USE_EVENT: int = 0 +DECLARATION_READ_EVENT: int = 1 +GENERATED_SQL_EVENT: int = 2 +ARGUMENT_REFERENCE_EVENT: int = 3 +GENERATED_REFERENCE_MARKERS: tuple[str, ...] = tuple( + f"{kind.function_name}(" for kind in SqlReferenceKind +) +ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( + "sqlbuild_active_macro_bridge", default=None +) +COMPILES_STARTED: itertools.count[int] = itertools.count() +MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" +MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v2" +VALUE_RENDERER_FIELD: str = "_value_renderer" +PROJECT_ROOT_SEARCH_PATH_STAMP: int = -2 +MISSING_SEARCH_PATH_STAMP: int = -1 +MODULE_DIGEST_ROW_FIELDS: int = 2 +INSTALLED_PACKAGE_DIRECTORY_NAMES: frozenset[str] = frozenset({"site-packages", "dist-packages"}) +INSTALLED_DISTRIBUTION_SUFFIX: str = ".dist-info" +INSTALLED_RECORD_FILE_NAME: str = "RECORD" +INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) +STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") +STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( + {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} +) diff --git a/src/sqlbuild/compiler/macro_bridge/exceptions.py b/src/sqlbuild/compiler/macro_bridge/exceptions.py new file mode 100644 index 0000000000..e3e4181d26 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/exceptions.py @@ -0,0 +1,7 @@ +"""Macro bridge exceptions.""" + +from __future__ import annotations + + +class UncacheableValueError(Exception): + """A value a macro may read has no stable text, so calls that may read it are not stored.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/__init__.py b/src/sqlbuild/compiler/macro_bridge/main/__init__.py new file mode 100644 index 0000000000..52c83b9999 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/__init__.py @@ -0,0 +1 @@ +"""Public macro bridge entry points.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py new file mode 100644 index 0000000000..ef5d3668c6 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py @@ -0,0 +1,11 @@ +"""Count the compiles this process has started, so only its first one uses the macro call store.""" + +from __future__ import annotations + +from sqlbuild.compiler.macro_bridge.constants import COMPILES_STARTED + + +def claim_first_compile() -> bool: + """Record that a compile started; True only for the first compile of this process.""" + + return next(COMPILES_STARTED) == 0 diff --git a/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py new file mode 100644 index 0000000000..9196fb60e1 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py @@ -0,0 +1,13 @@ +"""The macro bridge of the running native render stage.""" + +from __future__ import annotations + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE + + +def active_macro_bridge() -> MacroBridge | None: + """Return the macro bridge of the running native render stage, if any.""" + + bridge: object | None = ACTIVE_MACRO_BRIDGE.get() + return bridge if isinstance(bridge, MacroBridge) else None diff --git a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py new file mode 100644 index 0000000000..70b7ee99e6 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py @@ -0,0 +1,52 @@ +"""Run one render stage through the native macro bridge, re-running Python on any failure.""" + +from __future__ import annotations + +import logging +import sys +import unicodedata +from collections.abc import Callable +from contextvars import Token + +import sqlbuild._native as _native +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE + + +def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: + """Run `stage` with the bridge, storing calls on success; a failure re-runs Python.""" + + python_version: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) + if not _native.native_text_supported(python_version, unicodedata.unidata_version): + return stage() + bridge: MacroBridge = MacroBridge( + python_version=python_version, unicode_version=unicodedata.unidata_version + ) + token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) + native_error: Exception | None = None + try: + result: T = stage() + except Exception as error: + native_error = error + else: + bridge.save_store() + hits, misses, recorded = bridge.stats() + store_hits, store_records = bridge.store_stats() + logging.getLogger(__name__).debug( + "Native macro bridge: %d memo hits, %d store hits, %d misses, %d recorded calls, " + "%d stored calls", + hits, + store_hits, + misses, + recorded, + store_records, + ) + return result + finally: + ACTIVE_MACRO_BRIDGE.reset(token) + _ = stage() + raise NativeStageMismatchError( + "The native macro bridge failed where the Python render succeeded; " + "run with SQLBUILD_COMPILER_ENGINE=python" + ) from native_error diff --git a/src/sqlbuild/compiler/macro_bridge/models.py b/src/sqlbuild/compiler/macro_bridge/models.py new file mode 100644 index 0000000000..91ece77cc5 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/models.py @@ -0,0 +1,35 @@ +"""Macro call sites found by the native scan and the classes of their results.""" + +from __future__ import annotations + +from collections.abc import Hashable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroCallSite: + """One top-level macro call in an authored string, in Python string offsets.""" + + start: int + end: int + name: str + tree_names: tuple[str, ...] + typed_reference_text: bool + + +@dataclass(frozen=True) +class MacroCallClass: + """A call's inputs besides its text: by object identity for the memo, by value for the store.""" + + key: Hashable + macro_store_tokens: tuple[str, ...] + context_store_token: str | None + persistent: bool + + +@dataclass(frozen=True) +class ModuleSources: + """Files backing loaded modules, and whether every module's backing could be identified.""" + + paths: tuple[str, ...] + complete: bool diff --git a/src/sqlbuild/compiler/macro_bridge/types.py b/src/sqlbuild/compiler/macro_bridge/types.py new file mode 100644 index 0000000000..767a96c96c --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/types.py @@ -0,0 +1,8 @@ +"""Macro bridge type aliases.""" + +from __future__ import annotations + +from sqlbuild.python_nodes.models import SqlResourceRef + +type MacroCallEvent = tuple[int, str, str] +type MacroCallRecord = tuple[str, tuple[SqlResourceRef, ...], tuple[MacroCallEvent, ...]] diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 3faf8d875d..42665b68ca 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -117,7 +117,7 @@ def test_given_perturbed_native_engine_when_comparing_then_every_artifact_is_loc expected_lines=("DIFF project/waffle_shop",), expected_patterns=( r"- stage capture 0-compile/001-discovered_project_inputs\.json at /model_files/0/", - r"- stage capture 2-plan/001-discovered_project_inputs\.json at /model_files/0/", + r"- stage capture 3-plan/001-discovered_project_inputs\.json at /model_files/0/", ), expected_absent=(), ) @@ -151,7 +151,9 @@ def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_ expected_patterns=( r"- stage capture 0-compile/002-compile_project_inputs\.json at " r"/model_inputs/0/macro_deps/", - r"- stage capture 2-plan/002-compile_project_inputs\.json at " + r"- stage capture 2-compile-store-warm/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + r"- stage capture 3-plan/002-compile_project_inputs\.json at " r"/model_inputs/0/macro_deps/", ), expected_absent=("001-discovered_project_inputs.json",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7539197dfc..90da17fbf2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -5,11 +5,15 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple import pytest from tests.integration.src.sqlbuild.compiler.pipeline._test_types import PreparedCompile +if TYPE_CHECKING: + from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import CompileReuseRun + @dataclass(frozen=True) class SourceSemanticBindingCase: @@ -808,6 +812,16 @@ class CompilerEngineParityTestCase: expected_exit_codes: tuple[int, int] +@dataclass(frozen=True) +class CompilerEngineMacroParityTestCase: + """Two engines compiling separate copies of one macro-heavy project.""" + + description: str + files: dict[str, str] + expected_exit_codes: tuple[int, int] + expected_compiled_fragments: tuple[str, ...] + + @dataclass(frozen=True) class CompilerEngineStoreTestCase: """A sequence of engine compiles in one project and which of them reuse a stored compile.""" @@ -884,3 +898,79 @@ class NativeModelLoopParityTestCase: engines: tuple[str, str] expected_exit_codes: tuple[int, int] expected_report_text: str + + +@dataclass(frozen=True) +class MacroReferenceCallStoreTestCase: + """Repeated compiles of models sharing a macro that returns a rejected reference call.""" + + description: str + engine: str + expected_logged_calls: tuple[int, ...] + + +@dataclass(frozen=True) +class MacroCallStoreEditStep: + """One edit compiled with the macro call store, and the executions it expects.""" + + description: str + edit: Callable[[Path], object] + expected_logged_calls: int + args: tuple[str, ...] = () + env: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True) +class MacroCallStoreEditSequenceTestCase: + """Edits applied in order, each compiled with the macro call store and with --no-cache.""" + + description: str + project_reuse: bool + steps: tuple[MacroCallStoreEditStep, ...] + expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class BrokenMacroCallStoreKeyTestCase: + """An edit the store key no longer covers, which the --no-cache oracle must catch.""" + + description: str + edit: Callable[[Path], None] + expected_matches_uncached: bool + + +@dataclass(frozen=True) +class EngineMacroCallGateTestCase: + """Repeated full compiles under one engine and whether the bridge and store took part.""" + + description: str + engine: str + expected_logged_calls: tuple[int, ...] + expected_store_files: tuple[str, ...] + + +class StaleStoreArrangement(NamedTuple): + """What an arrangement step may use to compile and edit before the fresh-process check.""" + + project_dir: Path + extlib: Path + monkeypatch: pytest.MonkeyPatch + capsys: pytest.CaptureFixture[str] + + +@dataclass(frozen=True) +class StaleMacroModuleStoreTestCase: + """An outside module a macro imported, edited after this process imported it.""" + + description: str + arrange: Callable[[StaleStoreArrangement], None] + compile_afresh: Callable[[StaleStoreArrangement], CompileReuseRun] + expected_flavor: str + + +@dataclass(frozen=True) +class SecondCompileStoreTestCase: + """Two compiles in one process; the second must run its macros without the store.""" + + description: str + expected_second_matches_first: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 7fb69439ac..59ba8d67f5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -14,6 +14,7 @@ import subprocess import sys import time +import zipfile from bisect import bisect_left from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout @@ -34,9 +35,11 @@ import sqlbuild.cli.compile_reuse._helpers.store as reuse_store import sqlbuild.compiler.compile._helpers.assembly.binding_waves as binding_waves import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses +import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, @@ -56,13 +59,19 @@ from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, IncrementalEditStep, SemanticCorpusCase, SetOperationModel, + StaleStoreArrangement, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + MACRO_CALL_LOG_ENV_VAR, +) _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( (0.50, 1_800), @@ -4263,3 +4272,439 @@ def prepare_reference_call_project(*, tmp_path: Path, staging_from: str, mart_fr ) assert seeded.returncode == 0, seeded.stdout + seeded.stderr return project_dir + + +_MACRO_REFERENCE_CALL_MODELS: tuple[str, ...] = ("customer_orders", "customer_returns") +_DIAGNOSTIC_FIELDS: tuple[str, ...] = ( + "code", + "message", + "resource_name", + "path", + "line", + "column", + "help", +) + + +class MacroReferenceCallRuns(NamedTuple): + """Repeated compiles: exit codes, macro executions, and the diagnostics of each run.""" + + returncodes: tuple[int, ...] + logged_calls: tuple[int, ...] + diagnostics: tuple[tuple[tuple[object, ...], ...], ...] + + +def prepare_macro_reference_call_project(*, tmp_path: Path) -> Path: + """Write a project whose models share a macro returning a reference call compile rejects.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _REFERENCE_CALL_PROJECT_TOML, + "sources/raw.yml": _REFERENCE_CALL_SOURCES_YML, + "models/staging/customers.sql": ( + 'MODEL (description "Customers.", materialized view);\n\n' + 'SELECT order_id AS customer_id FROM __source("raw_orders")\n' + ), + "models/marts/_sqlbuild/_macros/customer_relation.py": ( + '"""A macro that returns an unquoted reference call."""\n\n' + "import os\n\n\n" + "def customer_relation() -> str:\n" + ' """Return the customers relation, logging each execution."""\n' + f' path = os.environ.get("{MACRO_CALL_LOG_ENV_VAR}")\n' + " if path:\n" + ' with open(path, "a", encoding="utf-8") as log:\n' + ' log.write("customer_relation\\n")\n' + ' return "__ref(customers)"\n' + ), + **{ + f"models/marts/{name}.sql": ( + f'MODEL (description "{name}.", materialized view);\n\n' + "SELECT customer_id FROM @customer_relation()\n" + ) + for name in _MACRO_REFERENCE_CALL_MODELS + }, + }, + ) + + +def _diagnostic_fields(item: dict[str, object]) -> tuple[object, ...]: + return tuple(item.get(field) for field in _DIAGNOSTIC_FIELDS) + + +def macro_reference_call_runs( + *, project_dir: Path, log_path: Path, engine: str, runs: int +) -> MacroReferenceCallRuns: + """Compile `runs` times in fresh processes, keeping the macro call store between runs.""" + + env: dict[str, str] = { + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + MACRO_CALL_LOG_ENV_VAR: str(log_path), + } + returncodes: list[int] = [] + logged_calls: list[int] = [] + diagnostics: list[tuple[tuple[object, ...], ...]] = [] + for _ in range(runs): + _ = log_path.write_text("", encoding="utf-8") + compiled: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json"), env=env + ) + returncodes.append(compiled.returncode) + logged_calls.append(len(log_path.read_text(encoding="utf-8").splitlines())) + items: list[dict[str, object]] = cast( + list[dict[str, object]], json.loads(compiled.stdout)["diagnostics"] + ) + diagnostics.append(tuple(_diagnostic_fields(item) for item in items)) + return MacroReferenceCallRuns( + returncodes=tuple(returncodes), + logged_calls=tuple(logged_calls), + diagnostics=tuple(diagnostics), + ) + + +MACRO_CALL_STORE_ENGINE: str = "native-preview" +STORE_ENVIRONMENT_REGION_VAR: str = "SQB_STORE_TEST_REGION" +STORE_ARGUMENT_ENV_VAR: str = "STORE_TEST_ARGUMENT" +_REUSE_DISABLED_VALUES: dict[bool, str] = {True: "0", False: "1"} +STALE_FLAVOR_EDIT: str = "salty" +_SOUTH_MODEL: str = "models/south/orders_south.sql" +_STORE_EXTERNAL_MODULE: str = "store_flavor_values" +_ZIPPED_FLAVOR_MODULE: str = "zipped_flavor_values" +_STORE_IMPORTED_MODULES: tuple[str, ...] = ( + _STORE_EXTERNAL_MODULE, + _ZIPPED_FLAVOR_MODULE, + "macros", + "macros._label_values", +) +_STORE_PROJECT_EXTRA_FILES: dict[str, str] = { + "macros/context_labels.py": ( + '"""Macros that read the target, the environment and an outside module."""\n\n' + "import os\n\n\n" + "def env_region() -> str:\n" + ' """Return the region from the environment."""\n' + f' return repr(os.environ.get("{STORE_ENVIRONMENT_REGION_VAR}", "none"))\n\n\n' + "def target_label(ctx) -> str:\n" + ' """Name the compile target."""\n' + " return repr(ctx.target_name)\n\n\n" + "def flavor() -> str:\n" + ' """Return a value from a module outside the project, imported while rendering."""\n' + f" import {_STORE_EXTERNAL_MODULE}\n\n" + f" return repr({_STORE_EXTERNAL_MODULE}.VALUE)\n" + ), + "macros/_label_values.py": ( + 'def label_value() -> str:\n """Return the label literal."""\n return "\'first\'"\n' + ), + "macros/labels.py": ( + "from macros._label_values import label_value\n\n\n" + 'def label() -> str:\n """Return the label from a helper module."""\n' + " return label_value()\n" + ), +} + + +class MacroCallStoreRun(NamedTuple): + """A compile with the macro call store, its --no-cache reference, and macro executions.""" + + incremental: CompileReuseRun + reference: CompileReuseRun + logged_calls: int + + @property + def matches(self) -> bool: + """Return whether exit code, report, and every compiled artifact are byte-identical.""" + + return _edit_outcome(self.incremental) == _edit_outcome(self.reference) + + +def prepare_macro_call_store_project(*, project_dir: Path, extlib: Path) -> None: + """Write a macro-heavy project whose macros read vars, target, constants and enums.""" + + for relative_path, content in MACRO_BRIDGE_PROJECT_FILES.items(): + write_project_file( + project_dir, relative_path, content.replace("WHERE id IN @generated_join()\n", "") + ) + for relative_path, content in _STORE_PROJECT_EXTRA_FILES.items(): + write_project_file(project_dir, relative_path, content) + replace_project_text( + project_dir, + "sqlbuild_project.toml", + '[targets.dev]\nschema = "analytics"\n', + '[targets.dev]\nschema = "analytics"\n\n[targets.prod]\nschema = "analytics"\n', + ) + replace_project_text( + project_dir, + _SOUTH_MODEL, + ' description "South orders",\n', + ' description "South orders",\n constants (_south_bonus 3),\n', + ) + replace_project_text( + project_dir, + _SOUTH_MODEL, + " 'quoted @cents(1)' AS quoted", + " @target_label() AS target_label,\n @flavor() AS flavor,\n @label() AS label,\n" + " @env_region() AS env_region,\n" + " 'quoted @cents(1)' AS quoted", + ) + write_store_flavor(extlib=extlib, value="sweet") + + +def write_store_flavor(*, extlib: Path, value: str) -> None: + """Write, or rewrite in place, the outside module the flavor macro imports.""" + + extlib.mkdir(parents=True, exist_ok=True) + (extlib / f"{_STORE_EXTERNAL_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") + + +def edit_label_helper(project_dir: Path, value: str) -> None: + """Change the literal the helper module behind the label macro returns.""" + + replace_project_text(project_dir, "macros/_label_values.py", "'first'", f"'{value}'") + + +def edit_south_model(project_dir: Path, old: str, new: str) -> None: + """Replace text in the south orders model.""" + + replace_project_text(project_dir, _SOUTH_MODEL, old, new) + + +def macro_call_store_compile( + *, + project_dir: Path, + extlib: Path, + log_path: Path, + project_reuse: bool, + args: tuple[str, ...] = (), + extra_env: tuple[tuple[str, str], ...] = (), +) -> MacroCallStoreRun: + """Compile with the macro call store, count macro executions, then compile with --no-cache.""" + + env: dict[str, str] = { + REUSE_DISABLE_ENV_VAR: _REUSE_DISABLED_VALUES[project_reuse], + MACRO_CALL_LOG_ENV_VAR: str(log_path), + "PYTHONPATH": str(extlib), + "PYTHONDONTWRITEBYTECODE": "1", + STORE_ARGUMENT_ENV_VAR: "first", + **dict(extra_env), + } + global_args: tuple[str, ...] = ("--compiler-engine", MACRO_CALL_STORE_ENGINE) + _ = log_path.write_text("", encoding="utf-8") + incremental: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=env, args=args, global_args=global_args + ) + logged_calls: int = len(log_path.read_text(encoding="utf-8").splitlines()) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=env, args=(*args, "--no-cache"), global_args=global_args + ) + return MacroCallStoreRun( + incremental=incremental, reference=reference, logged_calls=logged_calls + ) + + +def freeze_macro_call_store_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """Break the store key: every project and module state looks like the first one.""" + + monkeypatch.setattr(macro_bridge_class, "store_environment", lambda **_kwargs: "frozen") + monkeypatch.setattr(macro_bridge_class, "unchanged_module_digests", lambda **_kwargs: {}) + monkeypatch.setattr(macro_bridge_class.MacroBridge, "_observe_modules", lambda _bridge: True) + monkeypatch.setattr(call_store_module, "claim_first_compile", lambda: True) + + +def pretend_fresh_process(monkeypatch: pytest.MonkeyPatch) -> None: + """Treat this test process as one no compile has run in yet.""" + + compiles: Iterator[int] = itertools.count() + monkeypatch.setattr(call_store_module, "claim_first_compile", lambda: next(compiles) == 0) + + +def logged_in_process_compile( + *, project_dir: Path, log_path: Path, capsys: pytest.CaptureFixture[str] +) -> int: + """Compile in this process and return how many macro calls ran.""" + + _ = log_path.write_text("", encoding="utf-8") + _ = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + return len(log_path.read_text(encoding="utf-8").splitlines()) + + +def _save_after(monkeypatch: pytest.MonkeyPatch, change: Callable[[], object]) -> None: + original: Callable[[macro_bridge_class.MacroBridge], None] = ( + macro_bridge_class.MacroBridge.save_store + ) + pending: Iterator[Callable[[], object]] = iter([change]) + + def change_then_save(bridge: macro_bridge_class.MacroBridge) -> None: + _ = [next_change() for next_change in itertools.islice(pending, 1)] + original(bridge) + + monkeypatch.setattr(macro_bridge_class.MacroBridge, "save_store", change_then_save) + + +def recompile_in_process_after_edit(arrangement: StaleStoreArrangement) -> None: + """Compile twice in one process, editing the flavor module between the compiles.""" + + pretend_fresh_process(arrangement.monkeypatch) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def edit_flavor_while_saving(arrangement: StaleStoreArrangement) -> None: + """Compile in a fresh process that edits the flavor module just before saving the store.""" + + pretend_fresh_process(arrangement.monkeypatch) + _save_after( + arrangement.monkeypatch, + lambda: write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT), + ) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def move_flavor_while_saving(arrangement: StaleStoreArrangement) -> None: + """Move the loaded flavor away before saving, then write a new one, keeping folder times.""" + + module_path: Path = arrangement.extlib / f"{_STORE_EXTERNAL_MODULE}.py" + folder: os.stat_result = arrangement.extlib.stat() + pretend_fresh_process(arrangement.monkeypatch) + _save_after( + arrangement.monkeypatch, lambda: module_path.rename(module_path.with_suffix(".moved")) + ) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + os.utime(arrangement.extlib, ns=(folder.st_atime_ns, folder.st_mtime_ns)) + + +def rezip_flavor_between_processes(arrangement: StaleStoreArrangement) -> None: + """Serve the flavor from a zip archive, compile in a process, then rebuild the archive.""" + + write_zipped_flavor(extlib=arrangement.extlib, value="sweet") + _ = fresh_process_compile(project_dir=arrangement.project_dir, extlib=arrangement.extlib) + write_zipped_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + + +def backdate_flavor_between_processes(arrangement: StaleStoreArrangement) -> None: + """Compile in a process, then replace the flavor keeping its size and modification time.""" + + module_path: Path = arrangement.extlib / f"{_STORE_EXTERNAL_MODULE}.py" + _ = fresh_process_compile(project_dir=arrangement.project_dir, extlib=arrangement.extlib) + status: os.stat_result = module_path.stat() + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + os.utime(module_path, ns=(status.st_atime_ns, status.st_mtime_ns)) + + +def write_zipped_flavor(*, extlib: Path, value: str) -> None: + """Make the flavor module re-export a value from a module inside a zip archive.""" + + archive: Path = extlib / "flavors.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(f"{_ZIPPED_FLAVOR_MODULE}.py", f"VALUE = {value!r}\n") + _ = (extlib / f"{_STORE_EXTERNAL_MODULE}.py").write_text( + f"import sys\n\nsys.path.insert(0, {str(archive)!r})\n" + f"from {_ZIPPED_FLAVOR_MODULE} import VALUE\n", + encoding="utf-8", + ) + + +def fresh_process_compile(*, project_dir: Path, extlib: Path) -> MacroCallStoreRun: + """Compile with the macro call store in a new process, then with --no-cache.""" + + return macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=project_dir.parent / "fresh-calls.log", + project_reuse=False, + ) + + +def compile_in_new_process(arrangement: StaleStoreArrangement) -> CompileReuseRun: + """Compile with the macro call store in a new process.""" + + return fresh_process_compile( + project_dir=arrangement.project_dir, extlib=arrangement.extlib + ).incremental + + +def compile_as_new_process_here(arrangement: StaleStoreArrangement) -> CompileReuseRun: + """Compile in this process as if it had just started, re-importing the flavor modules.""" + + forget_store_flavor_module() + pretend_fresh_process(arrangement.monkeypatch) + return in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def uncached_reference_compile(*, project_dir: Path, extlib: Path) -> CompileReuseRun: + """Compile with --no-cache in a new process, the oracle for stored results.""" + + return run_reuse_compile( + project_dir=project_dir, + env={"PYTHONPATH": str(extlib), COMPILER_ENGINE_ENV_VAR: MACRO_CALL_STORE_ENGINE}, + args=("--no-cache",), + ) + + +def block_proc_reads(extlib: Path) -> None: + """Make Python file reads under /proc fail in processes importing from `extlib`.""" + + _ = (extlib / "sitecustomize.py").write_text( + "import builtins\nimport io\n\n_open = io.open\n\n\n" + "def _guarded_open(file, *args, **kwargs):\n" + " if str(file).startswith('/proc'):\n" + " raise FileNotFoundError(file)\n" + " return _open(file, *args, **kwargs)\n\n\n" + "builtins.open = io.open = _guarded_open\n", + encoding="utf-8", + ) + + +def forget_store_flavor_module() -> None: + """Drop the flavor and project macro helper modules in-process compiles imported.""" + + _ = [sys.modules.pop(name, None) for name in _STORE_IMPORTED_MODULES] + + +def store_files(project_dir: Path) -> dict[str, bytes]: + """Every macro call store file under the project target folder, by relative path.""" + + return { + path.relative_to(project_dir).as_posix(): path.read_bytes() + for path in sorted((project_dir / "target").rglob(MACRO_CALL_STORE_FILE_NAME)) + } + + +class EngineMacroCallRuns(NamedTuple): + """Full compiles under one engine: exit codes, macro executions, and macro call stores.""" + + returncodes: tuple[int, ...] + logged_calls: tuple[int, ...] + store_files: tuple[str, ...] + + +def engine_macro_call_runs( + *, project_dir: Path, extlib: Path, log_path: Path, engine: str, runs: int +) -> EngineMacroCallRuns: + """Compile fully `runs` times under an engine selected by environment; empty is the default.""" + + env: dict[str, str] = { + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + MACRO_CALL_LOG_ENV_VAR: str(log_path), + "PYTHONPATH": str(extlib), + "PYTHONDONTWRITEBYTECODE": "1", + } + returncodes: list[int] = [] + logged_calls: list[int] = [] + for _ in range(runs): + _ = log_path.write_text("", encoding="utf-8") + returncodes.append(run_reuse_compile(project_dir=project_dir, env=env).returncode) + logged_calls.append(len(log_path.read_text(encoding="utf-8").splitlines())) + return EngineMacroCallRuns( + returncodes=tuple(returncodes), + logged_calls=tuple(logged_calls), + store_files=tuple( + sorted( + path.relative_to(project_dir).as_posix() + for path in (project_dir / "target").rglob(MACRO_CALL_STORE_FILE_NAME) + ) + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py new file mode 100644 index 0000000000..5e1f98e710 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -0,0 +1,495 @@ +"""Compiles that reuse stored macro call results must match an uncached compile after every edit.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + BrokenMacroCallStoreKeyTestCase, + EngineMacroCallGateTestCase, + MacroCallStoreEditSequenceTestCase, + MacroCallStoreEditStep, + MacroReferenceCallStoreTestCase, + SecondCompileStoreTestCase, + StaleMacroModuleStoreTestCase, + StaleStoreArrangement, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + MACRO_CALL_STORE_ENGINE, + STORE_ARGUMENT_ENV_VAR, + STORE_ENVIRONMENT_REGION_VAR, + CompileReuseRun, + EngineMacroCallRuns, + MacroCallStoreRun, + MacroReferenceCallRuns, + backdate_flavor_between_processes, + block_proc_reads, + compile_as_new_process_here, + compile_in_new_process, + edit_flavor_while_saving, + edit_label_helper, + edit_south_model, + engine_macro_call_runs, + forget_store_flavor_module, + freeze_macro_call_store_environment, + in_process_reuse_run, + logged_in_process_compile, + macro_call_store_compile, + macro_reference_call_runs, + move_flavor_while_saving, + prepare_macro_call_store_project, + prepare_macro_reference_call_project, + pretend_fresh_process, + recompile_in_process_after_edit, + replace_project_text, + rezip_flavor_between_processes, + run_reuse_compile, + store_files, + uncached_reference_compile, + write_project_file, + write_store_flavor, +) +from tests.integration.src.sqlbuild.compiler.compile.helpers import MACRO_CALL_LOG_ENV_VAR + +_MACRO_REFERENCE_CALL_RUNS: int = 2 +_MACRO_REFERENCE_CALL_MESSAGE: str = ( + "__ref(customers) is not a valid __ref() call, returned by macro customer_relation()" +) +_MACRO_REFERENCE_CALL_PATH: str = "models/marts/_sqlbuild/_macros/customer_relation.py" +_EDIT_STEPS: tuple[MacroCallStoreEditStep, ...] = ( + MacroCallStoreEditStep( + description="no_change", + edit=lambda _root: None, + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="model_comment", + edit=lambda root: edit_south_model(root, "FROM (", "-- reviewed\nFROM ("), + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="model_adds_a_call", + edit=lambda root: edit_south_model( + root, " @target_label()", ' @logged("tax") AS logged_tax,\n @target_label()' + ), + expected_logged_calls=1, + ), + MacroCallStoreEditStep( + description="model_private_constant_read_by_context_macro", + edit=lambda root: edit_south_model(root, "_south_bonus 3", "_south_bonus 4"), + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="cli_vars_read_by_context_macro", + edit=lambda _root: None, + expected_logged_calls=0, + args=("--vars", '{"region": "south"}'), + ), + MacroCallStoreEditStep( + description="target_read_by_context_macro", + edit=lambda _root: None, + expected_logged_calls=0, + args=("--target", "prod"), + ), + MacroCallStoreEditStep( + description="model_adds_env_interpolated_argument", + edit=lambda root: edit_south_model( + root, + " @target_label()", + f' @logged("@@ENV:{STORE_ARGUMENT_ENV_VAR}") AS logged_env,\n @target_label()', + ), + expected_logged_calls=1, + ), + MacroCallStoreEditStep( + description="env_interpolated_argument_changes_call_text", + edit=lambda _root: None, + expected_logged_calls=1, + env=((STORE_ARGUMENT_ENV_VAR, "second"),), + ), + MacroCallStoreEditStep( + description="environment_variable_read_by_macro", + edit=lambda _root: None, + expected_logged_calls=3, + env=((STORE_ENVIRONMENT_REGION_VAR, "south"),), + ), + MacroCallStoreEditStep( + description="project_var_edit", + edit=lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'region = "north"', 'region = "east"' + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="constant_file_edit", + edit=lambda root: replace_project_text( + root, "constants/base_rate.sql", "value 1)", "value 5)" + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="enum_edit", + edit=lambda root: replace_project_text( + root, "enums/order_status.sql", 'PLACED "placed"', 'PLACED "open"' + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="macro_body_edit", + edit=lambda root: replace_project_text( + root, "macros/common.py", 'return f"{column} * 100"', 'return f"{column} * 1000"' + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="helper_module_edit", + edit=lambda root: edit_label_helper(root, "other"), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="macro_file_added", + edit=lambda root: ( + write_project_file( + root, + "macros/badges.py", + 'def badge() -> str:\n """Return a badge literal."""\n return "\'gold\'"\n', + ), + edit_south_model(root, " @label() AS label,\n", " @badge() AS badge,\n"), + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="macro_file_removed", + edit=lambda root: ( + edit_south_model(root, " @badge() AS badge,\n", ""), + (root / "macros/badges.py").unlink(), + ), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="outside_module_rewritten_in_place", + edit=lambda root: write_store_flavor(extlib=root.parent / "extlib", value="salty"), + expected_logged_calls=3, + ), + MacroCallStoreEditStep( + description="model_comment_after_every_edit", + edit=lambda root: edit_south_model(root, "-- reviewed\n", "-- reviewed twice\n"), + expected_logged_calls=0, + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroCallStoreEditSequenceTestCase( + description="full_render_each_step", project_reuse=False, steps=_EDIT_STEPS + ), + MacroCallStoreEditSequenceTestCase( + description="with_project_and_render_reuse", project_reuse=True, steps=_EDIT_STEPS + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_with_macro_call_store_then_each_step_matches_uncached( + tmp_path: Path, test_case: MacroCallStoreEditSequenceTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "macro-calls.log" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + cold: MacroCallStoreRun = macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=log_path, + project_reuse=test_case.project_reuse, + ) + assert (cold.incremental.returncode, cold.matches) == (0, True), cold.incremental.stderr + + for step in test_case.steps: + _ = step.edit(project_dir) + run: MacroCallStoreRun = macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=log_path, + project_reuse=test_case.project_reuse, + args=step.args, + extra_env=step.env, + ) + + assert run.incremental.returncode == 0, (step.description, run.incremental.stderr) + assert run.matches is test_case.expected_matches_uncached, step.description + assert run.logged_calls == step.expected_logged_calls, step.description + + +@pytest.mark.parametrize( + "test_case", + [ + BrokenMacroCallStoreKeyTestCase( + description="helper_module_edit", + edit=lambda root: edit_label_helper(root, "other"), + expected_matches_uncached=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_store_key_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( + tmp_path: Path, + test_case: BrokenMacroCallStoreKeyTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(tmp_path / "macro-calls.log")) + freeze_macro_call_store_environment(monkeypatch) + cold: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + assert cold.returncode == 0, cold.stderr + test_case.edit(project_dir) + + broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, + env={"PYTHONPATH": str(extlib), "SQLBUILD_COMPILER_ENGINE": MACRO_CALL_STORE_ENGINE}, + args=("--no-cache",), + ) + + assert (broken.returncode, reference.returncode) == (0, 0) + assert (broken.compiled == reference.compiled) is test_case.expected_matches_uncached + + +@pytest.mark.parametrize( + "test_case", + [ + EngineMacroCallGateTestCase( + description="default_engine", + engine="", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="python", + engine="python", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="native", + engine="native", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="native_preview", + engine="native-preview", + expected_logged_calls=(1, 0), + expected_store_files=("target/cache/compiler-native-preview-v1/macro-calls.bin",), + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_compiling_repeatedly_then_only_preview_batches_and_stores_macro_calls( + tmp_path: Path, test_case: EngineMacroCallGateTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + + runs: EngineMacroCallRuns = engine_macro_call_runs( + project_dir=project_dir, + extlib=extlib, + log_path=tmp_path / "macro-calls.log", + engine=test_case.engine, + runs=2, + ) + + assert runs.returncodes == (0, 0) + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.store_files == test_case.expected_store_files + + +@pytest.mark.parametrize( + "test_case", + [ + MacroReferenceCallStoreTestCase( + description="python", + engine="python", + expected_logged_calls=(2, 2), + ), + MacroReferenceCallStoreTestCase( + description="native_preview_memo_then_store", + engine="native-preview", + expected_logged_calls=(1, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_returning_rejected_reference_call_when_replaying_then_reports_match_python( + tmp_path: Path, test_case: MacroReferenceCallStoreTestCase +) -> None: + oracle: MacroReferenceCallRuns = macro_reference_call_runs( + project_dir=prepare_macro_reference_call_project(tmp_path=tmp_path / "python"), + log_path=tmp_path / "python.log", + engine="python", + runs=1, + ) + + runs: MacroReferenceCallRuns = macro_reference_call_runs( + project_dir=prepare_macro_reference_call_project(tmp_path=tmp_path / "engine"), + log_path=tmp_path / "engine.log", + engine=test_case.engine, + runs=_MACRO_REFERENCE_CALL_RUNS, + ) + + assert [diagnostic[:4] for diagnostic in oracle.diagnostics[0]] == [ + ("P012", _MACRO_REFERENCE_CALL_MESSAGE, name, _MACRO_REFERENCE_CALL_PATH) + for name in ("customer_orders", "customer_returns") + ] + assert runs.returncodes == (1,) * _MACRO_REFERENCE_CALL_RUNS + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.diagnostics == oracle.diagnostics * _MACRO_REFERENCE_CALL_RUNS + + +@pytest.mark.parametrize( + "test_case", + [ + StaleMacroModuleStoreTestCase( + description="edited_between_two_compiles_of_one_process", + arrange=recompile_in_process_after_edit, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="edited_between_store_attach_and_save", + arrange=edit_flavor_while_saving, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="moved_away_while_loaded", + arrange=move_flavor_while_saving, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="served_from_a_rebuilt_zip_archive", + arrange=rezip_flavor_between_processes, + compile_afresh=compile_in_new_process, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="replaced_keeping_size_and_modification_time", + arrange=backdate_flavor_between_processes, + compile_afresh=compile_in_new_process, + expected_flavor="'salty'", + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_module_changed_after_use_when_compiling_in_a_new_process_then_no_stale_replay( + tmp_path: Path, + test_case: StaleMacroModuleStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(tmp_path / "macro-calls.log")) + monkeypatch.setattr(sys, "dont_write_bytecode", True) + forget_store_flavor_module() + arrangement: StaleStoreArrangement = StaleStoreArrangement( + project_dir=project_dir, extlib=extlib, monkeypatch=monkeypatch, capsys=capsys + ) + test_case.arrange(arrangement) + + fresh: CompileReuseRun = test_case.compile_afresh(arrangement) + forget_store_flavor_module() + reference: CompileReuseRun = uncached_reference_compile(project_dir=project_dir, extlib=extlib) + + assert (fresh.returncode, reference.returncode) == (0, 0), fresh.stderr + assert fresh.compiled == reference.compiled + assert test_case.expected_flavor in b"".join(fresh.compiled.values()).decode() + + +@pytest.mark.parametrize( + "test_case", + [ + SecondCompileStoreTestCase( + description="second_compile_of_a_process", expected_second_matches_first=True + ) + ], + ids=lambda case: case.description, +) +def test_given_second_compile_in_one_process_when_compiling_then_the_store_is_not_touched( + tmp_path: Path, + test_case: SecondCompileStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "macro-calls.log" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(log_path)) + forget_store_flavor_module() + pretend_fresh_process(monkeypatch) + first: int = logged_in_process_compile( + project_dir=project_dir, log_path=log_path, capsys=capsys + ) + stored: dict[str, bytes] = store_files(project_dir) + + second: int = logged_in_process_compile( + project_dir=project_dir, log_path=log_path, capsys=capsys + ) + forget_store_flavor_module() + + assert first > 0 + assert stored != {} + assert (second == first) is test_case.expected_second_matches_first + assert store_files(project_dir) == stored + + +@pytest.mark.parametrize( + "test_case", + [ + EngineMacroCallGateTestCase( + description="native_preview_without_proc", + engine="native-preview", + expected_logged_calls=(1, 0), + expected_store_files=("target/cache/compiler-native-preview-v1/macro-calls.bin",), + ) + ], + ids=lambda case: case.description, +) +def test_given_no_proc_filesystem_when_compiling_repeatedly_then_the_store_still_serves_calls( + tmp_path: Path, test_case: EngineMacroCallGateTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + block_proc_reads(extlib) + + runs: EngineMacroCallRuns = engine_macro_call_runs( + project_dir=project_dir, + extlib=extlib, + log_path=tmp_path / "macro-calls.log", + engine=test_case.engine, + runs=2, + ) + + assert runs.returncodes == (0, 0) + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.store_files == test_case.expected_store_files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py index 12439f824d..18eef94924 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -7,6 +7,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompilerEngineMacroParityTestCase, CompilerEngineParityTestCase, CompilerEngineRulesStoreTestCase, CompilerEngineStoreTestCase, @@ -22,6 +23,10 @@ report_without_engine, store_digests, ) +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + write_project, +) @pytest.mark.parametrize( @@ -92,6 +97,40 @@ def test_given_no_engine_selection_when_compiling_then_native_runs_and_matches_p assert oracle.compiled +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineMacroParityTestCase( + description="macros_in_models_hooks_tests_audits_sources_functions", + files={ + path: content.replace("WHERE id IN @generated_join()\n", "") + for path, content in MACRO_BRIDGE_PROJECT_FILES.items() + }, + expected_exit_codes=(0, 0), + expected_compiled_fragments=("amount * 21", "'__SQLBUILD_RELATION_1__'"), + ) + ], + ids=lambda case: case.description, +) +def test_given_macro_heavy_project_when_compiling_with_each_engine_then_outputs_are_identical( + test_case: CompilerEngineMacroParityTestCase, tmp_path: Path +) -> None: + python_run: CompileReuseRun = engine_reuse_compile( + project_dir=write_project(root=tmp_path / "python", files=test_case.files), + engine="python", + ) + native_run: CompileReuseRun = engine_reuse_compile( + project_dir=write_project(root=tmp_path / "native", files=test_case.files), + engine="native-preview", + ) + compiled_text: str = b"".join(python_run.compiled.values()).decode() + + assert (python_run.returncode, native_run.returncode) == test_case.expected_exit_codes + assert report_without_engine(native_run) == report_without_engine(python_run) + assert native_run.compiled == python_run.compiled + assert all(map(compiled_text.__contains__, test_case.expected_compiled_fragments)) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compile/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py new file mode 100644 index 0000000000..2b8f407f13 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroBridgeParityTestCase: + """A macro-heavy project both engines must render to identical compile inputs.""" + + description: str + files: dict[str, str] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class MacroBridgeMemoTestCase: + """A project whose repeated macro calls run once per call class under the native engine.""" + + description: str + files: dict[str, str] + expected_python_executions: int + expected_native_executions: int + + +@dataclass(frozen=True) +class MacroBridgeFailureTestCase: + """A project whose rendering fails identically under both engines.""" + + description: str + files: dict[str, str] + expected_message_fragment: str + + +@dataclass(frozen=True) +class MacroExpansionDifferentialTestCase: + """Seeded random SQL text that must expand identically with and without the bridge.""" + + description: str + seed: int + samples: int + expected_minimum_successes: int + expected_mismatches: list[tuple[object, object, object]] diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py new file mode 100644 index 0000000000..fe10c48a49 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -0,0 +1,365 @@ +"""Macro-heavy projects, engine-switched render runs and bridged-expansion outcomes.""" + +from __future__ import annotations + +import random +import re +from collections.abc import Callable +from contextvars import Token +from functools import partial +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, + CompileProjectInputs, + LoadedMacro, + MacroContext, + MacroExpansionResult, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_INVOCATION_ID_MASK, + STAGE_CAPTURE_INVOCATION_ID_PATTERN, +) +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE +from sqlbuild.compiler.scopes.models import ResourceIdentity +from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( + resolve_effective_collection_rendering, +) + +MACRO_CALL_LOG_ENV_VAR: str = "SQLBUILD_TEST_MACRO_CALL_LOG" +_RUN_ID_PATTERN: re.Pattern[str] = re.compile(r'"run_id": "[^"]*"') +_DIFFERENTIAL_FRAGMENTS: tuple[str, ...] = ( + "@m('a')", "@m(1)", "@m()", "@mm(2)", "@mm (2)", "@m (1)", "@n(@m('b'))", "@n(@m(@n()))", + "@n(@mm(3), '@m(4)')", "@tag(__ref('orders'))", "@tag(__seed('rates'))", + "@tag(__ref('customers'))", "@cols(__ref('orders'))", "@gen()", "@nope(1)", "@m(x)", + "@ab c(1)", "@m('é')", "'@m(1)'", "-- @m(1)\n", "/* @n() */", "$$@m()$$", "`@m()`", "@@v", + "@", " ", "\n", "\u3000", "é", "'", "(", ")", "SELECT ", ", ", "\"q\"", +) # fmt: skip +_DIFFERENTIAL_FILE: Path = Path("models/orders_summary.sql") + +MACRO_BRIDGE_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "bridge_demo"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "bridge.duckdb"\n\n' + "[vars]\n" + 'region = "north"\n\n' + "[targets.dev]\n" + 'schema = "analytics"\n\n' + "[settings]\n" + "sql_analysis = false\n\n" + "[scopes]\n" + "enforce_placement = false\n" + ), + "macros/common.py": ( + '"""Shared macros for the bridge demo."""\n\n' + "import os\n\n\n" + "def cents(column: str) -> str:\n" + ' """Convert an amount to cents."""\n' + ' return f"{column} * 100"\n\n\n' + "def wrap(expression: str) -> str:\n" + ' """Parenthesise an expression."""\n' + ' return f"({expression})"\n\n\n' + "def region_filter(ctx) -> str:\n" + ' """Compare the region column with the configured region."""\n' + " return f\"region = '{ctx.vars['region']}'\"\n\n\n" + "def scale(ctx, column: str) -> str:\n" + ' """Scale a column by every visible rate."""\n' + ' return f"{column} * {sum(ctx.constants.values())}"\n\n\n' + "def visible_constants(ctx) -> str:\n" + ' """List the visible constants."""\n' + " return repr(', '.join(ctx.constants))\n\n\n" + "def status_code(ctx) -> str:\n" + ' """Render the placed status."""\n' + " return ctx.render_enum_member(enum_name='order_status', member_name='PLACED')\n\n\n" + "def optional_label(ctx) -> str:\n" + ' """Read an optional constant."""\n' + " return repr(ctx.constants.get('missing_label', 'none'))\n\n\n" + "def columns_of(relation) -> str:\n" + ' """Select every column of a relation."""\n' + ' return f"SELECT * FROM {relation}"\n\n\n' + "def ref_tag(relation) -> str:\n" + ' """Expose the in-expansion relation placeholder text."""\n' + " return repr(str(relation).upper())\n\n\n" + "def generated_join() -> str:\n" + ' """Emit a reference instead of receiving it."""\n' + " return '(SELECT id FROM __ref(\"customers\"))'\n\n\n" + "def logged(column: str) -> str:\n" + ' """Record each execution in the test log."""\n' + f' path = os.environ.get("{MACRO_CALL_LOG_ENV_VAR}")\n' + " if path:\n" + ' with open(path, "a", encoding="utf-8") as log:\n' + ' log.write(column + "\\n")\n' + " return column\n" + ), + "enums/order_status.sql": ( + 'ENUM (\n name order_status,\n members (PLACED "placed", SHIPPED "shipped"),\n);\n' + ), + "models/north/_sqlbuild/_macros/fmt.py": ( + 'def fmt(value: str) -> str:\n """Label a value."""\n return f"\'north:{value}\'"\n' + ), + "constants/base_rate.sql": "CONSTANT (name base_rate, value 1);\n", + "models/north/_sqlbuild/_constants/north_rate.sql": "CONSTANT (name north_rate, value 10);\n", + "models/south/_sqlbuild/_constants/south_rate.sql": "CONSTANT (name south_rate, value 20);\n", + "models/shared/customers.sql": ( + "MODEL (description \"Customers\");\n\nSELECT 1 AS id, 2 AS amount, 'north' AS region\n" + ), + "models/shared/orders_source.sql": ( + 'MODEL (description "Orders source");\n\nSELECT 1 AS id, 3 AS amount\n' + ), + "hooks/sql/stamp.sql": ( + "HOOK (description \"Stamp a value\");\n\nSELECT @cents('2') AS stamped\n" + ), + "functions/sql/udf__cents_of.sql": ( + "FUNCTION (\n" + ' description "Convert a value to cents",\n' + " arguments (v INTEGER),\n" + " returns INTEGER,\n" + ");\n\n" + '@cents("v")\n' + ), + "audits/generic/positive_cents.sql": ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @cents("amount") < 0\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw__payments\n" + " description: Payments\n" + " expression: |\n" + " SELECT @cents('5') AS amount_cents, @wrap(@cents('6')) AS wrapped\n" + ), + "tests/unit/test_cents_macro.sql": ( + 'TEST (mode macro, name "converts_to_cents");\n\n' + "WITH\n" + "__macro_actual__ AS (\n" + ' SELECT @cents("3") AS cents\n' + "),\n" + "__macro_expected__ AS (\n" + " SELECT 300 AS cents\n" + ")\n" + "SELECT 1\n" + ), +} + +_REGION_MODEL_BODY: str = ( + "SELECT\n" + ' @cents("amount") AS amount_cents,\n' + ' @wrap(@cents("amount")) AS wrapped,\n' + ' @scale("amount") AS scaled,\n' + " @visible_constants() AS visible,\n" + " {label} AS label,\n" + " @region_filter() AS region_check,\n" + " @status_code() AS status_value,\n" + " @optional_label() AS optional_label,\n" + " {ref_tags},\n" + ' @logged("amount") AS logged_amount,\n' + " 'quoted @cents(1)' AS quoted -- @cents(2)\n" + 'FROM (@columns_of(__ref("customers"))) AS c\n' + "WHERE id IN @generated_join()\n" +) +_NORTH_REF_TAGS: str = '@ref_tag(__ref("orders_source")) AS first_tag' +_SOUTH_REF_TAGS: str = ( + '@ref_tag(__ref("customers")) AS first_tag, @ref_tag(__ref("orders_source")) AS second_tag' +) + + +def _region_model(*, description: str, hooks: str, ref_tags: str, label: str) -> str: + return ( + f'MODEL (\n description "{description}",\n{hooks}' + " audits [positive_cents ()],\n);\n\n" + + _REGION_MODEL_BODY.format(ref_tags=ref_tags, label=label) + ) + + +_NORTH_HOOKS: str = ( + " pre_hooks [\n" + " inline_sql(\"SELECT @cents('1') AS hook_value\"),\n" + ' sql("stamp"),\n' + " ],\n" +) +MACRO_BRIDGE_PROJECT_FILES.update( + { + "models/north/orders_north.sql": _region_model( + description="North orders", + hooks=_NORTH_HOOKS, + ref_tags=_NORTH_REF_TAGS, + label='@fmt("x")', + ), + "models/north/orders_north_copy.sql": _region_model( + description="North orders copy", + hooks=_NORTH_HOOKS, + ref_tags=_NORTH_REF_TAGS, + label='@fmt("x")', + ), + "models/south/orders_south.sql": _region_model( + description="South orders", hooks="", ref_tags=_SOUTH_REF_TAGS, label="'south'" + ), + } +) + + +def write_project(*, root: Path, files: dict[str, str]) -> Path: + """Write project files below `root` and return it.""" + + for relative_path, content in files.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + return root + + +def render_compile_inputs( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> CompileProjectInputs: + """Render the project's compile inputs through the frontier under one compiler engine.""" + + with monkeypatch.context() as patch: + patch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + adapter: DuckDbAdapter = DuckDbAdapter() + return compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + resolved_connection={}, + defer_model_sql_validation=True, + no_cache=True, + ), + ) + + +def comparable_capture(inputs: CompileProjectInputs) -> str: + """Render compile inputs as canonical JSON without the per-run identifier.""" + + return STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub( + STAGE_CAPTURE_INVOCATION_ID_MASK, + _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)), + ) + + +def render_error( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[type[BaseException], str, type[object]]: + """Render the project under one engine and describe the error it raises.""" + + with pytest.raises(CompileInputError) as raised: + _ = render_compile_inputs(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) + return type(raised.value), str(raised.value), type(raised.value.__cause__) + + +def _wrap_m(value: object = "") -> str: + return f"m<{value}>" + + +def _wrap_n(value: object = "", label: object = "") -> str: + return f"n[{value}{label}]" + + +def _tag(relation: object) -> str: + return repr(str(relation).upper()) + + +def _cols(relation: object) -> str: + return f"SELECT * FROM {relation}" + + +def _generated() -> str: + return '(SELECT id FROM __ref("orders"))' + + +def _loaded_macro(*, name: str, function: Callable[..., object]) -> LoadedMacro: + path: Path = Path(f"macros/{name}.py") + return LoadedMacro( + name=name, file_path=path, relative_path=path, raw_source="", function=function + ) + + +DIFFERENTIAL_MACROS: dict[str, LoadedMacro] = { + name: _loaded_macro(name=name, function=function) + for name, function in ( + ("m", _wrap_m), + ("mm", _wrap_m), + ("n", _wrap_n), + ("tag", _tag), + ("cols", _cols), + ("gen", _generated), + ) +} +_DIFFERENTIAL_CONTEXT: MacroContext = MacroContext( + adapter_name="duckdb", sql_analysis_enabled=False, target_name=None +) + + +class ExpansionFailureCapture: + """Swallow and keep the compile input error a block raises.""" + + def __init__(self) -> None: + self.failure: BaseException | None = None + + def __enter__(self) -> ExpansionFailureCapture: + return self + + def __exit__(self, error_type: object, error: BaseException | None, traceback: object) -> bool: + self.failure = error + return isinstance(error, CompileInputError) + + +def random_macro_sql(*, rng: random.Random) -> str: + """Return SQL text mixing valid and malformed macro calls, quotes, comments and Unicode.""" + + return "".join(rng.choice(_DIFFERENTIAL_FRAGMENTS) for _ in range(rng.randint(0, 12))) + + +def expansion_outcome(*, sql: str, bridge: MacroBridge | None) -> tuple[object, ...]: + """Expand `sql` with or without the bridge; return the result facts or the error.""" + + token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) + capture: ExpansionFailureCapture = ExpansionFailureCapture() + rendered: list[tuple[object, ...]] = [()] + with capture: + result: MacroExpansionResult = expand_sql_macros_result( + sql=sql, + file_path=_DIFFERENTIAL_FILE, + loaded_macros=DIFFERENTIAL_MACROS, + macro_context=_DIFFERENTIAL_CONTEXT, + consumer=ResourceIdentity(ResourceKind.MODEL, "orders_summary"), + ) + rendered[0] = ( + result.sql, + result.spans, + result.dependencies, + result.usages, + result.argument_references, + ) + ACTIVE_MACRO_BRIDGE.reset(token) + return { + True: rendered[0], + False: (type(capture.failure).__name__, str(capture.failure)), + }[capture.failure is None] diff --git a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py new file mode 100644 index 0000000000..2650ab34a3 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py @@ -0,0 +1,228 @@ +"""The native macro bridge renders compile inputs byte-identically to the Python engine.""" + +from __future__ import annotations + +import random +import sys +import unicodedata +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from tests.integration.src.sqlbuild.compiler.compile._test_types import ( + MacroBridgeFailureTestCase, + MacroBridgeMemoTestCase, + MacroBridgeParityTestCase, + MacroExpansionDifferentialTestCase, +) +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + MACRO_CALL_LOG_ENV_VAR, + comparable_capture, + expansion_outcome, + random_macro_sql, + render_compile_inputs, + render_error, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_FAILURE_BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": MACRO_BRIDGE_PROJECT_FILES["sqlbuild_project.toml"], + "macros/common.py": ( + "def cents(column: str) -> str:\n" + ' """Convert an amount to cents."""\n' + ' return f"{column} * 100"\n\n\n' + "def explode(column: str) -> str:\n" + ' """Fail while rendering."""\n' + ' raise ValueError(f"cannot render {column}")\n\n\n' + "def number(column: str) -> int:\n" + ' """Return a value that is not SQL."""\n' + " return 1\n" + ), + "models/a_first.sql": ( + 'MODEL (description "First");\n\nSELECT @cents("amount") AS amount_cents\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeParityTestCase( + description="nested, context, scoped, typed-reference, hook, test, audit, source " + "and function macros", + files=MACRO_BRIDGE_PROJECT_FILES, + expected_fragments=( + "'north:x'", + "amount * 11", + "amount * 21", + "'base_rate, north_rate'", + "'base_rate, south_rate'", + "'__SQLBUILD_RELATION_0__'", + "'__SQLBUILD_RELATION_1__'", + "P006", + "SELECT 2 * 100 AS stamped", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_heavy_project_when_rendering_with_each_engine_then_inputs_are_identical( + test_case: MacroBridgeParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + + python_capture: str = comparable_capture( + render_compile_inputs(project_dir=project_dir, engine="python", monkeypatch=monkeypatch) + ) + native_capture: str = comparable_capture( + render_compile_inputs( + project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch + ) + ) + + assert native_capture == python_capture + assert all(map(python_capture.__contains__, test_case.expected_fragments)) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeMemoTestCase( + description="one global call text in three models", + files=MACRO_BRIDGE_PROJECT_FILES, + expected_python_executions=3, + expected_native_executions=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_repeated_macro_calls_when_rendering_natively_then_each_call_class_runs_once( + test_case: MacroBridgeMemoTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + executions: dict[str, int] = {} + for engine in ("python", "native-preview"): + log_path: Path = tmp_path / f"{engine}.log" + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(log_path)) + _ = render_compile_inputs(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) + executions[engine] = len(log_path.read_text(encoding="utf-8").splitlines()) + + assert executions == { + "python": test_case.expected_python_executions, + "native-preview": test_case.expected_native_executions, + } + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeFailureTestCase( + description="macro raises in a later model", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\nSELECT @explode("amount") AS bogus\n' + ), + }, + expected_message_fragment="failed: cannot render amount", + ), + MacroBridgeFailureTestCase( + description="unknown macro after a successful call", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\n' + 'SELECT @cents("amount") AS a, @nope("amount") AS b\n' + ), + }, + expected_message_fragment="Unknown macro '@nope'", + ), + MacroBridgeFailureTestCase( + description="macro returns a value that is not SQL", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\nSELECT @number("amount") AS b\n' + ), + }, + expected_message_fragment="must return a SQL string", + ), + MacroBridgeFailureTestCase( + description="malformed call after a successful call", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\n' + 'SELECT @cents("amount") AS a, @cents x("amount") AS b\n' + ), + }, + expected_message_fragment="expected opening parenthesis", + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_render_when_rendering_natively_then_raises_the_python_error( + test_case: MacroBridgeFailureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + + python_error: tuple[type[BaseException], str, type[object]] = render_error( + project_dir=project_dir, engine="python", monkeypatch=monkeypatch + ) + native_error: tuple[type[BaseException], str, type[object]] = render_error( + project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch + ) + + assert native_error == python_error + assert test_case.expected_message_fragment in python_error[1] + + +@pytest.mark.parametrize( + "test_case", + [ + MacroExpansionDifferentialTestCase( + description="seeded fragments", + seed=540, + samples=3000, + expected_minimum_successes=600, + expected_mismatches=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_random_macro_sql_when_expanding_through_bridge_then_matches_python( + test_case: MacroExpansionDifferentialTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + texts: list[str] = [random_macro_sql(rng=rng) for _ in range(test_case.samples)] + bridge: MacroBridge = MacroBridge( + python_version=(sys.version_info[0], sys.version_info[1]), + unicode_version=unicodedata.unidata_version, + ) + + python_outcomes: list[tuple[object, ...]] = [ + expansion_outcome(sql=sql, bridge=None) for sql in texts + ] + bridged_outcomes: list[tuple[object, ...]] = [ + expansion_outcome(sql=sql, bridge=bridge) for sql in texts + ] + + assert ( + mismatches( + inputs=list[object](texts), + expected=list[object](python_outcomes), + actual=list[object](bridged_outcomes), + ) + == test_case.expected_mismatches + ) + assert sum(len(outcome) == 5 for outcome in python_outcomes) >= ( + test_case.expected_minimum_successes + ) + assert bridge.stats()[0] > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 3f004a9f97..02134d10f3 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -103,6 +103,42 @@ stage=NativeStage.MODEL_LOOP, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_macro_calls", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MACRO_CALLS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="python_macro_call_store", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_macro_calls", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MACRO_CALLS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_macro_call_store", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_macro_calls", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MACRO_CALLS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="native_preview_macro_call_store", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=True, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py new file mode 100644 index 0000000000..d8d73b71f8 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from sqlbuild.compiler.compile.models import DeclarationResolutionContext, MacroContext +from sqlbuild.python_nodes.models import SqlResourceRef + + +@dataclass(frozen=True) +class ContextStoreTokenTestCase: + """Two macro contexts and whether their stored results may be shared.""" + + description: str + left: MacroContext + right: MacroContext + expected_equal: bool + left_declarations: DeclarationResolutionContext | None = None + right_declarations: DeclarationResolutionContext | None = None + + +@dataclass(frozen=True) +class ContextValueTestCase: + """A macro context value and whether calls that may read it can be stored.""" + + description: str + context: MacroContext + expected_stored: bool + + +@dataclass(frozen=True) +class CallClassStoreTextTestCase: + """A call class's context and relations, compared with a plain class of the same macros.""" + + description: str + context_token: str | None + prior_relations: tuple[SqlResourceRef, ...] | None + expected_equal: bool + + +@dataclass(frozen=True) +class MacroStoreTokenTestCase: + """A change to a loaded macro and whether its stored results stay valid.""" + + description: str + relative_path: Path + raw_source: str + expected_equal: bool + + +@dataclass(frozen=True) +class ModuleDigestsRoundTripTestCase: + """Module digests encoded as store metadata and validated back.""" + + description: str + module_text: str + expected_valid: bool + + +@dataclass(frozen=True) +class StoreEnvironmentTestCase: + """A project edit and whether stored macro calls stay valid after it.""" + + description: str + edit: Callable[[Path], object] + expected_unchanged: bool + + +@dataclass(frozen=True) +class StoreEnvironmentVariableTestCase: + """An environment variable change and whether stored macro calls stay valid after it.""" + + description: str + name: str + value: str + expected_unchanged: bool + + +@dataclass(frozen=True) +class ModuleDigestsTestCase: + """Stored module digests, a change to the module file, and whether they still validate.""" + + description: str + change: Callable[[Path], bytes] + expected_valid: bool + + +@dataclass(frozen=True) +class ModuleSourcesTestCase: + """A loaded module and the file the store digests for it, if its code can be identified.""" + + description: str + module: Callable[[Path], object] + expected_paths: Callable[[Path], tuple[str, ...]] + expected_complete: bool + expected_digestible: bool diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py new file mode 100644 index 0000000000..34953aa1bc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import datetime +import decimal +import json +import os +import sys +import types +import zipfile +from dataclasses import replace +from importlib.machinery import ModuleSpec +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext +from sqlbuild.compiler.discovery.models import ConstantDeclaration +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + project_fingerprint, + store_environment, +) +from sqlbuild.sql_values.models import SqlLogicalType, SqlValue +from sqlbuild.sql_values.types import SqlValueKind + +BASE_CONTEXT: MacroContext = MacroContext( + adapter_name="duckdb", + sql_analysis_enabled=False, + target_name="dev", + vars={ + "region": "north", + "limits": [1, 2.5, decimal.Decimal("3.10")], + "since": datetime.date(2026, 1, 1), + }, +) + + +def with_vars(**values: object) -> MacroContext: + """The base context with some vars replaced.""" + + return replace(BASE_CONTEXT, vars={**BASE_CONTEXT.vars, **values}) + + +def constant_declarations(value: int) -> DeclarationResolutionContext: + """Declarations holding one integer constant.""" + + return DeclarationResolutionContext( + constants={ + "base_rate": ConstantDeclaration( + name="base_rate", + value=SqlValue(logical_type=SqlLogicalType(kind=SqlValueKind.INTEGER), value=value), + relative_path=Path("constants/base_rate.sql"), + ) + } + ) + + +def loaded_macro(*, relative_path: Path, raw_source: str) -> LoadedMacro: + """A loaded `cents` macro from the given file.""" + + return LoadedMacro( + name="cents", + file_path=Path("/project") / relative_path, + relative_path=relative_path, + raw_source=raw_source, + function=lambda column: column, + ) + + +def write_module(path: Path, text: str) -> bytes: + """Write a module file and return metadata recording its current content digest.""" + + _ = path.write_text(text, encoding="utf-8") + return json.dumps([[str(path), _native.digest_files([str(path)])[0]]]).encode() + + +def backdated_module(path: Path) -> bytes: + """Record a module, then replace it with other bytes of the same size and modification time.""" + + metadata: bytes = write_module(path, "VALUE = 1\n") + status: os.stat_result = path.stat() + _ = path.write_text("VALUE = 2\n", encoding="utf-8") + os.utime(path, ns=(status.st_atime_ns, status.st_mtime_ns)) + return metadata + + +def removed_module(path: Path) -> bytes: + """Record a module, then remove it.""" + + metadata: bytes = write_module(path, "VALUE = 1\n") + path.unlink() + return metadata + + +def file_module(root: Path) -> object: + """A module loaded from a source file.""" + + path: Path = root / "flavor.py" + _ = path.write_text("VALUE = 1\n", encoding="utf-8") + spec: ModuleSpec | None = spec_from_file_location("flavor", path) + assert spec is not None + return module_from_spec(spec) + + +def zip_member_module(root: Path) -> object: + """A module whose location is a member of a zip archive on the import path.""" + + archive: Path = root / "flavors.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("zipped_flavor.py", "VALUE = 1\n") + return _located_module("zipped_flavor", str(archive / "zipped_flavor.py")) + + +def _located_module(name: str, origin: str) -> object: + spec: ModuleSpec = ModuleSpec(name, None, origin=origin) + spec.has_location = True + return module_from_spec(spec) + + +def moved_module(root: Path) -> object: + """A module loaded from a file that has since been moved away.""" + + module: object = file_module(root) + _ = (root / "flavor.py").rename(root / "flavor_moved.py") + return module + + +def custom_loader_module(_root: Path) -> object: + """A module from a loader that reports no file location.""" + + return module_from_spec(ModuleSpec("generated_flavor", None, origin="generated")) + + +def synthetic_module(_root: Path) -> object: + """A module object created in code, without an import spec.""" + + return types.ModuleType("synthetic_flavor") + + +def interpreter_module(_root: Path) -> object: + """A module built into the interpreter.""" + + return sys.modules["sys"] + + +def standard_library_module(_root: Path) -> object: + """A standard library module loaded from source.""" + + return sys.modules["json"] + + +def sqlbuild_module(_root: Path) -> object: + """A module of SQLBuild itself.""" + + return sys.modules["sqlbuild.compiler.macro_bridge.constants"] + + +def write_environment_project(root: Path) -> None: + """Write a model and a macro for store environment checks.""" + + (root / "models").mkdir() + (root / "macros").mkdir() + _ = (root / "models/orders.sql").write_text("SELECT 1\n", encoding="utf-8") + _ = (root / "macros/common.py").write_text("X = 1\n", encoding="utf-8") + + +EXCLUDED_MODEL_PATHS: list[str] = ["models/orders.sql"] + + +def environment_of(root: Path) -> str: + """The store environment of a project, excluding its one model file.""" + + return store_environment( + project_dir=root, + fingerprint=project_fingerprint(project_dir=root, model_paths=EXCLUDED_MODEL_PATHS), + ) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py new file mode 100644 index 0000000000..30e76f55d3 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + digest_module_files, + module_digests_metadata, + module_sources, + unchanged_module_digests, +) +from sqlbuild.compiler.macro_bridge.models import ModuleSources +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( + ModuleDigestsRoundTripTestCase, + ModuleDigestsTestCase, + ModuleSourcesTestCase, + StoreEnvironmentTestCase, + StoreEnvironmentVariableTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers.helpers import ( + backdated_module, + custom_loader_module, + environment_of, + file_module, + interpreter_module, + moved_module, + removed_module, + sqlbuild_module, + standard_library_module, + synthetic_module, + write_environment_project, + write_module, + zip_member_module, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleDigestsTestCase( + description="unchanged module", + change=lambda path: write_module(path, "VALUE = 1\n"), + expected_valid=True, + ), + ModuleDigestsTestCase( + description="module replaced keeping its size and modification time", + change=backdated_module, + expected_valid=False, + ), + ModuleDigestsTestCase( + description="module removed", change=removed_module, expected_valid=False + ), + ModuleDigestsTestCase( + description="unreadable metadata", change=lambda _path: b"{", expected_valid=False + ), + ModuleDigestsTestCase( + description="unexpected metadata shape", + change=lambda _path: b'[["flavor.py", 1]]', + expected_valid=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_module_digests_when_validating_then_only_unchanged_contents_pass( + tmp_path: Path, test_case: ModuleDigestsTestCase +) -> None: + metadata: bytes = test_case.change(tmp_path / "flavor.py") + + digests: dict[str, str] | None = unchanged_module_digests(metadata=metadata, known={}) + + assert (digests is not None) is test_case.expected_valid + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleDigestsRoundTripTestCase( + description="one module", module_text="VALUE = 1\n", expected_valid=True + ), + ], + ids=lambda case: case.description, +) +def test_given_module_digests_when_encoding_then_validation_returns_them( + tmp_path: Path, test_case: ModuleDigestsRoundTripTestCase +) -> None: + path: Path = tmp_path / "flavor.py" + _ = write_module(path, test_case.module_text) + digests: dict[str, str] | None = digest_module_files([str(path)]) + assert digests is not None + + validated: dict[str, str] | None = unchanged_module_digests( + metadata=module_digests_metadata(digests), known={} + ) + + assert (validated == digests) is test_case.expected_valid + + +@pytest.mark.parametrize( + "test_case", + [ + StoreEnvironmentTestCase( + description="no edit", edit=lambda _root: None, expected_unchanged=True + ), + StoreEnvironmentTestCase( + description="model edit", + edit=lambda root: (root / "models/orders.sql").write_text("SELECT 2\n"), + expected_unchanged=True, + ), + StoreEnvironmentTestCase( + description="macro edit", + edit=lambda root: (root / "macros/common.py").write_text("X = 2\n"), + expected_unchanged=False, + ), + StoreEnvironmentTestCase( + description="model the caller does not exclude", + edit=lambda root: (root / "models/customers.sql").write_text("SELECT 1\n"), + expected_unchanged=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_edit_when_digesting_store_environment_then_only_model_edits_keep_it( + tmp_path: Path, test_case: StoreEnvironmentTestCase +) -> None: + write_environment_project(tmp_path) + original: str = environment_of(tmp_path) + _ = test_case.edit(tmp_path) + + edited: str = environment_of(tmp_path) + + assert (edited == original) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + StoreEnvironmentVariableTestCase( + description="variable a macro may read", + name="SQB_STORE_REGION", + value="south", + expected_unchanged=False, + ), + StoreEnvironmentVariableTestCase( + description="compiler engine selection", + name="SQLBUILD_COMPILER_ENGINE", + value="python", + expected_unchanged=True, + ), + StoreEnvironmentVariableTestCase( + description="variable outside the tracked prefixes", + name="STORE_REGION", + value="south", + expected_unchanged=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_environment_variable_change_when_digesting_store_environment_then_tracked_ones_count( + tmp_path: Path, test_case: StoreEnvironmentVariableTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + write_environment_project(tmp_path) + monkeypatch.delenv(test_case.name, raising=False) + original: str = environment_of(tmp_path) + monkeypatch.setenv(test_case.name, test_case.value) + + changed: str = environment_of(tmp_path) + + assert (changed == original) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleSourcesTestCase( + description="source file", + module=file_module, + expected_paths=lambda root: (str(root / "flavor.py"),), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="zip archive member", + module=zip_member_module, + expected_paths=lambda root: (str(root / "flavors.zip" / "zipped_flavor.py"),), + expected_complete=True, + expected_digestible=False, + ), + ModuleSourcesTestCase( + description="file moved after loading", + module=moved_module, + expected_paths=lambda root: (str(root / "flavor.py"),), + expected_complete=True, + expected_digestible=False, + ), + ModuleSourcesTestCase( + description="loader without a file location", + module=custom_loader_module, + expected_paths=lambda _root: (), + expected_complete=False, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="module object created in code", + module=synthetic_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="built into the interpreter", + module=interpreter_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="standard library source", + module=standard_library_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="sqlbuild itself", + module=sqlbuild_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_module_when_identifying_its_code_then_only_readable_files_are_trusted( + tmp_path: Path, test_case: ModuleSourcesTestCase +) -> None: + module: object = test_case.module(tmp_path) + + sources: ModuleSources = module_sources([module]) + + assert sources == ModuleSources( + paths=test_case.expected_paths(tmp_path), complete=test_case.expected_complete + ) + assert (digest_module_files(sources.paths) is not None) is test_case.expected_digestible diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py new file mode 100644 index 0000000000..e32305902d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( + call_class_store_text, + context_store_token, + macro_store_token, +) +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( + CallClassStoreTextTestCase, + ContextStoreTokenTestCase, + ContextValueTestCase, + MacroStoreTokenTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers.helpers import ( + BASE_CONTEXT, + constant_declarations, + loaded_macro, + with_vars, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ContextStoreTokenTestCase( + description="equal values in separate objects", + left=with_vars(), + right=with_vars(), + expected_equal=True, + left_declarations=constant_declarations(1), + right_declarations=constant_declarations(1), + ), + ContextStoreTokenTestCase( + description="var value changed", + left=with_vars(), + right=with_vars(region="south"), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="integer and float var values", + left=with_vars(region=1), + right=with_vars(region=1.0), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="boolean and integer var values", + left=with_vars(region=1), + right=with_vars(region=True), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="var order changed", + left=with_vars(first=1, second=2), + right=replace(BASE_CONTEXT, vars={**BASE_CONTEXT.vars, "second": 2, "first": 1}), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="target changed", + left=with_vars(), + right=replace(BASE_CONTEXT, target_name="prod"), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="visible constant value changed", + left=with_vars(), + right=with_vars(), + expected_equal=False, + left_declarations=constant_declarations(1), + right_declarations=constant_declarations(2), + ), + ContextStoreTokenTestCase( + description="declarations missing on one side", + left=with_vars(), + right=with_vars(), + expected_equal=False, + left_declarations=constant_declarations(1), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_contexts_when_naming_their_store_class_then_only_equal_values_match( + test_case: ContextStoreTokenTestCase, +) -> None: + left: str | None = context_store_token( + macro_context=test_case.left, declarations=test_case.left_declarations + ) + right: str | None = context_store_token( + macro_context=test_case.right, declarations=test_case.right_declarations + ) + + assert left is not None + assert (left == right) is test_case.expected_equal + + +@pytest.mark.parametrize( + "test_case", + [ + ContextValueTestCase( + description="dates, decimals and floats", context=with_vars(), expected_stored=True + ), + ContextValueTestCase( + description="object var", context=with_vars(region=object()), expected_stored=False + ), + ContextValueTestCase( + description="nested object var", + context=with_vars(region={"key": [object()]}), + expected_stored=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_context_value_when_naming_store_class_then_only_stable_values_are_stored( + test_case: ContextValueTestCase, +) -> None: + token: str | None = context_store_token(macro_context=test_case.context, declarations=None) + + assert (token is not None) is test_case.expected_stored + + +@pytest.mark.parametrize( + "test_case", + [ + MacroStoreTokenTestCase( + description="same file and source", + relative_path=Path("macros/common.py"), + raw_source="def cents(column): ...\n", + expected_equal=True, + ), + MacroStoreTokenTestCase( + description="source edited", + relative_path=Path("macros/common.py"), + raw_source="def cents(column): return column\n", + expected_equal=False, + ), + MacroStoreTokenTestCase( + description="same source in a scoped file", + relative_path=Path("models/south/_sqlbuild/_macros/common.py"), + raw_source="def cents(column): ...\n", + expected_equal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_macro_change_when_naming_it_then_only_the_same_file_and_source_match( + test_case: MacroStoreTokenTestCase, +) -> None: + original: str = macro_store_token( + loaded=loaded_macro( + relative_path=Path("macros/common.py"), raw_source="def cents(column): ...\n" + ), + identity=None, + ) + + changed: str = macro_store_token( + loaded=loaded_macro(relative_path=test_case.relative_path, raw_source=test_case.raw_source), + identity=None, + ) + + assert (original == changed) is test_case.expected_equal + + +@pytest.mark.parametrize( + "test_case", + [ + CallClassStoreTextTestCase( + description="same class", context_token=None, prior_relations=None, expected_equal=True + ), + CallClassStoreTextTestCase( + description="no relations passed yet", + context_token=None, + prior_relations=(), + expected_equal=False, + ), + CallClassStoreTextTestCase( + description="one relation passed", + context_token=None, + prior_relations=(SqlResourceRef(kind=SqlResourceRefKind.MODEL, name="orders"),), + expected_equal=False, + ), + CallClassStoreTextTestCase( + description="context macro", + context_token="ctx", + prior_relations=None, + expected_equal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_call_class_inputs_when_naming_it_then_only_identical_inputs_match( + test_case: CallClassStoreTextTestCase, +) -> None: + plain: str = call_class_store_text( + macro_tokens=("cents",), context_token=None, prior_relations=None + ) + + text: str = call_class_store_text( + macro_tokens=("cents",), + context_token=test_case.context_token, + prior_relations=test_case.prior_relations, + ) + + assert (text == plain) is test_case.expected_equal diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py new file mode 100644 index 0000000000..33c8dbcefc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DeepNestingScanTestCase: + """Macro calls nested `depth` levels deep and the scan the bridge must return quickly.""" + + description: str + depth: int + expected_tree_names: tuple[tuple[str, ...], ...] + expected_max_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py new file mode 100644 index 0000000000..758149c91a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sys +import time +import unicodedata + +import pytest + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.models import MacroCallSite +from tests.unit.src.sqlbuild.compiler.macro_bridge.classes._test_types import ( + DeepNestingScanTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeepNestingScanTestCase( + description="one thousand levels", + depth=1_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + DeepNestingScanTestCase( + description="twenty thousand levels", + depth=20_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + DeepNestingScanTestCase( + description="one hundred thousand levels", + depth=100_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_deeply_nested_calls_when_scanning_then_the_bridge_returns_promptly( + test_case: DeepNestingScanTestCase, +) -> None: + bridge: MacroBridge = MacroBridge( + python_version=(sys.version_info[0], sys.version_info[1]), + unicode_version=unicodedata.unidata_version, + ) + sql: str = "@m(" * test_case.depth + "x" + ")" * test_case.depth + started: float = time.perf_counter() + + sites: tuple[MacroCallSite, ...] | None = bridge.scan(sql) + + assert time.perf_counter() - started < test_case.expected_max_seconds + assert tuple(site.tree_names for site in sites or ()) == test_case.expected_tree_names diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py new file mode 100644 index 0000000000..699b51074f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroBridgeFailureTestCase: + """A render stage that fails with the macro bridge active.""" + + description: str + stage_with_bridge: Callable[[], str] + stage_without_bridge: Callable[[], str] + expected_error: type[Exception] + expected_bridged_runs: list[bool] + + +@dataclass(frozen=True) +class MacroBridgeSuccessTestCase: + """A render stage that succeeds with the macro bridge active.""" + + description: str + expected_result: str + expected_bridged_runs: list[bool] + + +@dataclass(frozen=True) +class MacroBridgeErrorContextTestCase: + """A stage whose Python error must not chain the bridge failure as its context.""" + + description: str + stage: Callable[[], str] + expected_context: BaseException | None diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py new file mode 100644 index 0000000000..9914dce8e2 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -0,0 +1,33 @@ +"""Render stages that record whether the macro bridge was active.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge + + +def rendered() -> str: + """Render successfully.""" + + return "rendered" + + +def failed() -> str: + """Fail to render.""" + + raise CompileInputError("render failed") + + +def recording_stage( + *, runs: list[bool], stages: dict[bool, Callable[[], str]] +) -> Callable[[], str]: + """Return a stage recording whether each run had the bridge, then running its variant.""" + + def stage() -> str: + bridged: bool = active_macro_bridge() is not None + runs.append(bridged) + return stages[bridged]() + + return stage diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py new file mode 100644 index 0000000000..5b556aeb0d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge +from tests.unit.src.sqlbuild.compiler.macro_bridge.main._test_types import ( + MacroBridgeErrorContextTestCase, + MacroBridgeFailureTestCase, + MacroBridgeSuccessTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge.main.helpers import ( + failed, + recording_stage, + rendered, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeSuccessTestCase( + description="success runs once with the bridge", + expected_result="rendered", + expected_bridged_runs=[True], + ), + ], + ids=lambda case: case.description, +) +def test_given_succeeding_stage_when_running_with_macro_bridge_then_runs_once_bridged( + test_case: MacroBridgeSuccessTestCase, +) -> None: + runs: list[bool] = [] + + result: str = run_with_macro_bridge( + stage=recording_stage(runs=runs, stages={True: rendered, False: failed}) + ) + + assert result == test_case.expected_result + assert runs == test_case.expected_bridged_runs + assert active_macro_bridge() is None + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeFailureTestCase( + description="failure re-runs Python and raises its error", + stage_with_bridge=failed, + stage_without_bridge=failed, + expected_error=CompileInputError, + expected_bridged_runs=[True, False], + ), + MacroBridgeFailureTestCase( + description="native-only failure is reported as a divergence", + stage_with_bridge=failed, + stage_without_bridge=rendered, + expected_error=NativeStageMismatchError, + expected_bridged_runs=[True, False], + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_stage_when_running_with_macro_bridge_then_python_reruns( + test_case: MacroBridgeFailureTestCase, +) -> None: + runs: list[bool] = [] + stage: Callable[[], str] = recording_stage( + runs=runs, + stages={True: test_case.stage_with_bridge, False: test_case.stage_without_bridge}, + ) + + with pytest.raises(test_case.expected_error): + _ = run_with_macro_bridge(stage=stage) + + assert runs == test_case.expected_bridged_runs + assert active_macro_bridge() is None + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeErrorContextTestCase( + description="python error after a bridge failure", + stage=failed, + expected_context=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_error_when_rerun_after_bridge_failure_then_error_has_no_bridge_context( + test_case: MacroBridgeErrorContextTestCase, +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = run_with_macro_bridge(stage=test_case.stage) + + assert raised.value.__context__ is test_case.expected_context + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 6c24b05777..b07bbd8c1c 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -100,7 +100,9 @@ anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.id files count only by presence. Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs -must produce the same output. A compile whose templates read `run.id` is never reused. +must produce the same output. Macro results are also reused within a compile and may be kept between +compiles; see [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). A +compile whose templates read `run.id` is never reused. Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when `SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index b6f389f70d..8e3fda3c32 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -204,6 +204,31 @@ SELECT 1 AS id See [SQL Hooks](/docs/concepts/models/hooks/sql) for hook lifecycle and context syntax. +## Macros must be deterministic + +A macro must return the same result whenever it is called with the same arguments and sees the same +context. Avoid module-level state such as counters, random values, the current time, and reads of +files or environment variables that the call does not receive. + +SQLBuild relies on this contract to reuse macro results: + +- [Compile reuse](/docs/cli/compile) replays stored renders without running macros again. +- A compile may run each distinct macro call once and reuse its result for identical calls + elsewhere in the project. +- Macro call results may be kept between compiles under `target/cache/`. They are recomputed when + macro or other Python files, configuration, declarations, seeds, sources, variables, the target, + the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps + them, and a changed call is always run again. Only the first compile in a process uses and saves + them: later compiles in the same process, such as in a notebook or a long-running server, run their + macros again. A compile that has loaded Python code it cannot read from a plain file, such as a + module imported from a zip archive, neither uses nor saves them. + +Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and +`@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, +and visible constants and enums. Environment variables a macro reads directly, for example with +`os.environ`, are compared only when their names start with `SQLBUILD_` or `SQB_`, the same set +compile reuse tracks. A macro whose result changes for any other reason may return a stale result. + ## Where macros work Macros are supported in: