From aa1d30fe94db45b9cf1029454cfd5e7a92c6b15f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:11:34 +0100 Subject: [PATCH 01/11] feat(native): add a render crate that scans, splices and memoizes macro calls The sqlbuild-render crate ports the Python macro call scanner byte for byte (quotes, comments, dollar quotes, the identifier/whitespace quirks, Python Unicode classes), splices rendered calls with code-point spans, and holds the in-compile memo of recorded calls and their events. Anything the scanner cannot reproduce exactly defers to Python. --- Cargo.toml | 3 + crates/README.md | 5 +- crates/sqlbuild-render/Cargo.toml | 14 ++ crates/sqlbuild-render/src/lib.rs | 5 + .../src/macro_calls/_helpers/mod.rs | 2 + .../src/macro_calls/_helpers/offsets.rs | 45 ++++ .../src/macro_calls/_helpers/scanner.rs | 238 ++++++++++++++++++ .../src/macro_calls/main/mod.rs | 2 + .../macro_calls/main/scan_macro_call_sites.rs | 42 ++++ .../macro_calls/main/splice_macro_calls.rs | 45 ++++ crates/sqlbuild-render/src/macro_calls/mod.rs | 6 + .../sqlbuild-render/src/macro_calls/models.rs | 87 +++++++ .../sqlbuild-render/src/macro_calls/tests.rs | 10 + .../src/macro_calls/tests/helpers.rs | 29 +++ .../macro_calls/tests/test_macro_call_memo.rs | 47 ++++ .../tests/test_scan_macro_call_sites.rs | 113 +++++++++ .../tests/test_splice_macro_calls.rs | 76 ++++++ .../src/macro_calls/tests/test_types.rs | 26 ++ fensu.toml | 1 + 19 files changed, 794 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-render/Cargo.toml create mode 100644 crates/sqlbuild-render/src/lib.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/main/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/models.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests/helpers.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/tests/test_types.rs diff --git a/Cargo.toml b/Cargo.toml index 9b4efa3107..255a55dcd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", + "crates/sqlbuild-render", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -23,6 +24,7 @@ order = [ "sqlbuild-discovery", "sqlbuild-scopes", "sqlbuild-model-config", + "sqlbuild-render", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -73,6 +75,7 @@ sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } +sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index 6993d60495..81d9613b9b 100644 --- a/crates/README.md +++ b/crates/README.md @@ -12,6 +12,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | +| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -22,8 +23,8 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-scopes - -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-render + -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core ``` `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and diff --git a/crates/sqlbuild-render/Cargo.toml b/crates/sqlbuild-render/Cargo.toml new file mode 100644 index 0000000000..ac3a9cf79b --- /dev/null +++ b/crates/sqlbuild-render/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sqlbuild-render" +version.workspace = true +description = "Native rendering for SQLBuild: macro call-site scanning, splicing and the in-compile macro call memo." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sqlbuild-core.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-render/src/lib.rs b/crates/sqlbuild-render/src/lib.rs new file mode 100644 index 0000000000..d2567e5538 --- /dev/null +++ b/crates/sqlbuild-render/src/lib.rs @@ -0,0 +1,5 @@ +//! Native rendering: macro call sites, splicing of macro results and the in-compile call memo. + +#![forbid(unsafe_code)] + +pub mod macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs new file mode 100644 index 0000000000..e6dea55521 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod offsets; +pub(crate) mod scanner; diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs new file mode 100644 index 0000000000..6dbc61a808 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/offsets.rs @@ -0,0 +1,45 @@ +//! Conversions between UTF-8 byte offsets and Python code-point offsets of one text. + +/// Return the code-point offset of each ascending byte offset in `byte_offsets`. +pub(crate) fn char_offsets(text: &str, byte_offsets: &[usize]) -> Vec { + if text.is_ascii() { + return byte_offsets.to_vec(); + } + let mut converted = Vec::with_capacity(byte_offsets.len()); + let mut byte = 0; + let mut chars = 0; + for offset in byte_offsets { + chars += text[byte..*offset].chars().count(); + byte = *offset; + converted.push(chars); + } + converted +} + +/// Return the byte offset of each ascending code-point offset, or `None` past the end of `text`. +pub(crate) fn byte_offsets(text: &str, char_offsets: &[usize]) -> Option> { + if text.is_ascii() { + return char_offsets + .iter() + .all(|offset| *offset <= text.len()) + .then(|| char_offsets.to_vec()); + } + let mut converted = Vec::with_capacity(char_offsets.len()); + let mut indices = text + .char_indices() + .map(|(byte, _)| byte) + .chain([text.len()]); + let mut position = 0; + let mut current = indices.next()?; + for offset in char_offsets { + if *offset < position { + return None; + } + while position < *offset { + current = indices.next()?; + position += 1; + } + converted.push(current); + } + Some(converted) +} diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs new file mode 100644 index 0000000000..49ed169cb8 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -0,0 +1,238 @@ +//! Byte offset port of the Python macro call scanner; unsure or rejected text defers to Python. + +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; + +use crate::macro_calls::models::ScanDeferral; + +const TYPED_REFERENCE_NAMES: [&str; 3] = ["__ref", "__source", "__seed"]; + +/// One call found by the scanner, in byte offsets of the scanned text. +pub(crate) struct ScannedCall { + pub(crate) start: usize, + pub(crate) open: usize, + pub(crate) close: usize, + pub(crate) name: String, +} + +/// Return every top-level call of `text` in order. +pub(crate) fn top_level_calls( + python: PythonText, + text: &str, +) -> Result, ScanDeferral> { + let mut calls: Vec = Vec::new(); + let mut cursor = 0; + while cursor < text.len() { + let Some(start) = next_macro_start(python, text, cursor)? else { + break; + }; + let call = call_at(python, text, start)?; + cursor = call.close + 1; + calls.push(call); + } + Ok(calls) +} + +/// Return the names of every call nested in `args`, depth first, with repeats. +pub(crate) fn nested_names(python: PythonText, args: &str) -> Result, ScanDeferral> { + let mut names: Vec = Vec::new(); + for call in top_level_calls(python, args)? { + let inner = nested_names(python, &args[call.open + 1..call.close])?; + names.push(call.name); + names.extend(inner); + } + Ok(names) +} + +/// Whether call arguments mention a typed reference function name anywhere. +pub(crate) fn mentions_typed_reference(args: &str) -> bool { + TYPED_REFERENCE_NAMES.iter().any(|name| args.contains(name)) +} + +fn call_at(python: PythonText, text: &str, start: usize) -> Result { + let name_end = identifier_end(python, text, start + 1); + let open = skip_whitespace(text, name_end); + if text.as_bytes().get(open) != Some(&b'(') { + return Err(ScanDeferral); + } + let close = matching_paren(text.as_bytes(), open)?; + Ok(ScannedCall { + start, + open, + close, + name: text[start + 1..name_end].to_owned(), + }) +} + +fn next_macro_start( + python: PythonText, + text: &str, + from: usize, +) -> Result, ScanDeferral> { + let bytes = text.as_bytes(); + let mut index = from; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, + b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), + b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), + b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, + b'@' => { + if macro_call_starts_at(python, text, index)? { + return Ok(Some(index)); + } + index += 1; + } + _ => index += 1, + } + } + Ok(None) +} + +fn macro_call_starts_at(python: PythonText, text: &str, at: usize) -> Result { + let Some(first) = char_at(text, at + 1) else { + return Ok(false); + }; + if !first.is_ascii() { + return Err(ScanDeferral); + } + if !(first.is_ascii_alphabetic() || first == '_') { + return Ok(false); + } + let mut cursor = at + 2; + while let Some(character) = char_at(text, cursor) { + if !is_identifier_continue(python, character) { + break; + } + cursor = skip_whitespace(text, cursor + character.len_utf8()); + } + Ok(text.as_bytes().get(cursor) == Some(&b'(')) +} + +fn identifier_end(python: PythonText, text: &str, from: usize) -> usize { + let mut cursor = from; + while let Some(character) = char_at(text, cursor) { + if !is_identifier_continue(python, character) { + break; + } + cursor += character.len_utf8(); + } + cursor +} + +fn is_identifier_continue(python: PythonText, character: char) -> bool { + character == '_' || is_python_alnum(python, character) +} + +fn skip_whitespace(text: &str, from: usize) -> usize { + let mut cursor = from; + while let Some(character) = char_at(text, cursor) { + if !is_python_space(character) { + break; + } + cursor += character.len_utf8(); + } + cursor +} + +fn char_at(text: &str, index: usize) -> Option { + text.get(index..).and_then(|rest| rest.chars().next()) +} + +fn matching_paren(bytes: &[u8], open: usize) -> Result { + let mut depth = 1usize; + let mut index = open + 1; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => { + index = quote_end(bytes, index)?; + continue; + } + b'$' => { + if let Some(end) = dollar_quote_end(bytes, index)? { + index = end; + continue; + } + } + b'-' if bytes.get(index + 1) == Some(&b'-') => { + index = line_comment_end(bytes, index); + continue; + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index = block_comment_end(bytes, index)?; + continue; + } + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + _ => {} + } + index += 1; + } + Err(ScanDeferral) +} + +fn quote_end(bytes: &[u8], start: usize) -> Result { + let quote = bytes[start]; + let doubled_escapes = quote == b'\'' || quote == b'"'; + let mut index = start + 1; + loop { + let offset = bytes[index.min(bytes.len())..] + .iter() + .position(|byte| *byte == quote) + .ok_or(ScanDeferral)?; + index += offset; + if doubled_escapes && bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + 1); + } +} + +fn dollar_quote_end(bytes: &[u8], start: usize) -> Result, ScanDeferral> { + if start + .checked_sub(1) + .is_some_and(|previous| continues_dollar_word(bytes[previous])) + { + return Ok(None); + } + let tag_end = bytes[start + 1..] + .iter() + .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) + .map_or(bytes.len(), |offset| start + 1 + offset); + let tag = &bytes[start + 1..tag_end]; + if bytes.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { + return Ok(None); + } + let delimiter = &bytes[start..=tag_end]; + bytes[tag_end + 1..] + .windows(delimiter.len()) + .position(|window| window == delimiter) + .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) + .ok_or(ScanDeferral) +} + +fn continues_dollar_word(byte: u8) -> bool { + !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') +} + +fn line_comment_end(bytes: &[u8], start: usize) -> usize { + bytes[start..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(bytes.len(), |offset| start + offset + 1) +} + +fn block_comment_end(bytes: &[u8], start: usize) -> Result { + bytes[start + 2..] + .windows(2) + .position(|pair| pair == b"*/") + .map(|offset| start + 2 + offset + 2) + .ok_or(ScanDeferral) +} diff --git a/crates/sqlbuild-render/src/macro_calls/main/mod.rs b/crates/sqlbuild-render/src/macro_calls/main/mod.rs new file mode 100644 index 0000000000..2f945c96d5 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/mod.rs @@ -0,0 +1,2 @@ +pub mod scan_macro_call_sites; +pub mod splice_macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs new file mode 100644 index 0000000000..47598cc9dd --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs @@ -0,0 +1,42 @@ +//! Top-level macro call sites of one authored SQL string, as Python's expansion loop visits them. + +use sqlbuild_core::text::models::PythonText; + +use crate::macro_calls::_helpers::offsets::char_offsets; +use crate::macro_calls::_helpers::scanner::{ + mentions_typed_reference, nested_names, top_level_calls, +}; +use crate::macro_calls::models::{MacroCallSite, ScanDeferral}; + +/// Every top-level call site in order under `python`'s character classes, or a deferral. +pub fn scan_macro_call_sites( + python: PythonText, + sql: &str, +) -> Result, ScanDeferral> { + let calls = top_level_calls(python, sql)?; + let boundaries: Vec = calls + .iter() + .flat_map(|call| [call.start, call.close + 1]) + .collect(); + let offsets = char_offsets(sql, &boundaries); + calls + .into_iter() + .zip(offsets.as_chunks::<2>().0) + .map(|(call, offsets)| { + let args = &sql[call.open + 1..call.close]; + let mut tree_names: Vec = vec![call.name.clone()]; + for name in nested_names(python, args)? { + if !tree_names.contains(&name) { + tree_names.push(name); + } + } + Ok(MacroCallSite { + start: offsets[0], + end: offsets[1], + name: call.name, + tree_names, + typed_reference_text: mentions_typed_reference(args), + }) + }) + .collect() +} diff --git a/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs b/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs new file mode 100644 index 0000000000..c1a986e5b1 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/splice_macro_calls.rs @@ -0,0 +1,45 @@ +//! Replace macro call sites with their rendered SQL and report each substitution span. + +use crate::macro_calls::_helpers::offsets::byte_offsets; +use crate::macro_calls::models::MacroSpliceSpan; + +/// Splice `outputs` over ascending code-point `sites`; `None` when they do not fit `sql`. +pub fn splice_macro_calls( + sql: &str, + sites: &[(usize, usize)], + outputs: &[String], +) -> Option<(String, Vec)> { + if sites.len() != outputs.len() { + return None; + } + let boundaries: Vec = sites + .iter() + .flat_map(|(start, end)| [*start, *end]) + .collect(); + if boundaries.windows(2).any(|pair| pair[0] > pair[1]) { + return None; + } + let bytes = byte_offsets(sql, &boundaries)?; + let mut rendered = + String::with_capacity(sql.len() + outputs.iter().map(String::len).sum::()); + let mut spans = Vec::with_capacity(sites.len()); + let mut cursor_byte = 0; + let mut output_chars = 0; + for ((site, output), range) in sites.iter().zip(outputs).zip(bytes.as_chunks::<2>().0) { + let literal = &sql[cursor_byte..range[0]]; + rendered.push_str(literal); + output_chars += literal.chars().count(); + rendered.push_str(output); + let output_len = output.chars().count(); + spans.push(MacroSpliceSpan { + source_start: site.0, + source_end: site.1, + output_start: output_chars, + output_end: output_chars + output_len, + }); + output_chars += output_len; + cursor_byte = range[1]; + } + rendered.push_str(&sql[cursor_byte..]); + Some((rendered, spans)) +} diff --git a/crates/sqlbuild-render/src/macro_calls/mod.rs b/crates/sqlbuild-render/src/macro_calls/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-render/src/macro_calls/models.rs b/crates/sqlbuild-render/src/macro_calls/models.rs new file mode 100644 index 0000000000..f125026a05 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/models.rs @@ -0,0 +1,87 @@ +//! Macro call sites, recorded macro call results and the in-compile memo that replays them. + +use std::collections::HashMap; +use std::sync::Arc; + +/// One top-level `@macro(...)` call in an authored SQL string; offsets count code points. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroCallSite { + /// Offset of the `@` that starts the call. + pub start: usize, + /// Offset just past the call's closing parenthesis. + pub end: usize, + /// The called macro's name. + pub name: String, + /// Every macro name in the call, the top-level name first, then nested calls once each. + pub tree_names: Vec, + /// The arguments mention `__ref`, `__source` or `__seed`, so typed references may be passed. + pub typed_reference_text: bool, +} + +/// The scan met text Python reports or classifies differently; Python must expand the string. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScanDeferral; + +/// One consumer-independent fact a macro call produced, replayed for every consumer in order. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum MacroCallEvent { + /// A macro, top-level or nested, was resolved and called. + MacroUse { name: String }, + /// A macro read a constant or enum through its context. + DeclarationRead { kind: String, name: String }, + /// A macro returned SQL that names typed references, a generated-reference candidate. + GeneratedSql { macro_name: String, sql: String }, + /// A typed reference was written as a macro argument. + ArgumentReference { kind: String, name: String }, +} + +/// The recorded result of one top-level macro call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroCallEntry { + /// The macro's SQL before relation placeholders are rendered. + pub sql: String, + /// Typed references the call added to the string's relation table, in order. + pub relations: Vec<(String, String)>, + /// Facts the call produced, in encounter order. + pub events: Vec, +} + +/// Substitution offsets of one spliced macro call, in code points. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MacroSpliceSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + +/// Recorded macro call results of one compile, keyed by call class and exact call text. +#[derive(Debug, Default)] +pub struct MacroCallMemo { + entries: HashMap<(u64, String), Arc>, + hits: usize, + misses: usize, +} + +impl MacroCallMemo { + /// Return the recorded result for the call, counting the lookup as a hit or a miss. + pub fn lookup(&mut self, class_id: u64, call_text: &str) -> Option> { + let found = self.entries.get(&(class_id, call_text.to_owned())).cloned(); + if found.is_some() { + self.hits += 1; + } else { + self.misses += 1; + } + found + } + + /// Record the result of one executed call. + pub fn record(&mut self, class_id: u64, call_text: String, entry: MacroCallEntry) { + let _ = self.entries.insert((class_id, call_text), Arc::new(entry)); + } + + /// Lookups that found a recorded result, lookups that did not, and recorded results. + pub fn stats(&self) -> (usize, usize, usize) { + (self.hits, self.misses, self.entries.len()) + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests.rs b/crates/sqlbuild-render/src/macro_calls/tests.rs new file mode 100644 index 0000000000..6340f8ffc3 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_macro_call_memo.rs"] +mod macro_call_memo; +#[path = "tests/test_scan_macro_call_sites.rs"] +mod scan_macro_call_sites; +#[path = "tests/test_splice_macro_calls.rs"] +mod splice_macro_calls; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs new file mode 100644 index 0000000000..181f047d89 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs @@ -0,0 +1,29 @@ +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; + +use crate::macro_calls::models::{MacroCallEntry, MacroCallEvent, MacroCallMemo}; + +/// A memo holding one recorded `@cents(amount)` call in class 1, and that call's entry. +pub(crate) fn recorded_memo() -> (MacroCallMemo, MacroCallEntry) { + let mut memo = MacroCallMemo::default(); + let entry = MacroCallEntry { + sql: "amount * 100".to_owned(), + relations: vec![("model".to_owned(), "orders".to_owned())], + events: vec![ + MacroCallEvent::MacroUse { + name: "cents".to_owned(), + }, + MacroCallEvent::DeclarationRead { + kind: "constant".to_owned(), + name: "scale".to_owned(), + }, + ], + }; + memo.record(1, "@cents(amount)".to_owned(), entry.clone()); + (memo, entry) +} + +/// The character classes of Python 3.12. +pub(crate) fn python_312() -> PythonText { + python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs new file mode 100644 index 0000000000..5a9866a43b --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs @@ -0,0 +1,47 @@ +use crate::macro_calls::tests::helpers::recorded_memo; +use crate::macro_calls::tests::test_types::MacroCallMemoTestCase; + +#[test] +fn given_recorded_call_when_looking_up_then_only_same_class_and_text_hit() { + let test_cases = [ + MacroCallMemoTestCase { + description: "same class and call text", + class_id: 1, + call_text: "@cents(amount)", + expected_hit: true, + }, + MacroCallMemoTestCase { + description: "another call class", + class_id: 2, + call_text: "@cents(amount)", + expected_hit: false, + }, + MacroCallMemoTestCase { + description: "another call text", + class_id: 1, + call_text: "@cents(total)", + expected_hit: false, + }, + ]; + + for test_case in test_cases { + let (mut memo, entry) = recorded_memo(); + let found = memo.lookup(test_case.class_id, test_case.call_text); + assert_eq!( + found.as_deref() == Some(&entry), + test_case.expected_hit, + "{}", + test_case.description + ); + assert_eq!( + memo.stats(), + ( + usize::from(test_case.expected_hit), + usize::from(!test_case.expected_hit), + 1 + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs new file mode 100644 index 0000000000..b77c6b4632 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -0,0 +1,113 @@ +use crate::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites; +use crate::macro_calls::tests::helpers::python_312; +use crate::macro_calls::tests::test_types::{ExpectedSite, ScanMacroCallSitesTestCase}; + +#[test] +fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansion() { + let test_cases = [ + ScanMacroCallSitesTestCase { + description: "one call", + sql: "SELECT @a(1) FROM t", + expected_sites: Some(&[(7, 12, "a", &["a"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "nested calls belong to the enclosing site", + sql: "@outer(@inner(@leaf()), '@quoted()', @inner(2)) y", + expected_sites: Some(&[(0, 47, "outer", &["outer", "inner", "leaf"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "quoted text and comments hide calls", + sql: "'@a(1)' -- @b(2)\n/* @c(3) */ \"@d()\" `@e()` $$@f()$$ @g()", + expected_sites: Some(&[(52, 56, "g", &["g"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a one-letter name followed by whitespace is not a call", + sql: "@a (1)", + expected_sites: Some(&[]), + }, + ScanMacroCallSitesTestCase { + description: "a longer name may be separated from its parenthesis", + sql: "@ab (1)", + expected_sites: Some(&[(0, 7, "ab", &["ab"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "whitespace inside a name defers to Python's error", + sql: "@ab c(1)", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "an unclosed quote after a call defers", + sql: "@a(1) 'oops", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "an unclosed call defers", + sql: "@a(1", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "offsets count code points", + sql: "SELECT 'é', @a(1)", + expected_sites: Some(&[(12, 17, "a", &["a"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a non-ASCII character after @ defers", + sql: "@é(1)", + expected_sites: None, + }, + ScanMacroCallSitesTestCase { + description: "typed reference text is flagged", + sql: "@m(__ref('x')) @n('__seed') @o(1)", + expected_sites: Some(&[ + (0, 14, "m", &["m"], true), + (15, 27, "n", &["n"], true), + (28, 33, "o", &["o"], false), + ]), + }, + ScanMacroCallSitesTestCase { + description: "a dollar sign continuing a word does not open a quote", + sql: "a$$ @m()", + expected_sites: Some(&[(4, 8, "m", &["m"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "interpolation markers are not calls", + sql: "x @@var @m()", + expected_sites: Some(&[(8, 12, "m", &["m"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "parentheses inside quoted arguments do not close the call", + sql: "@m(')', \"(\", `)`) z", + expected_sites: Some(&[(0, 17, "m", &["m"], false)]), + }, + ]; + + for test_case in test_cases { + let actual: Option> = scan_macro_call_sites(python_312(), test_case.sql) + .ok() + .map(|sites| { + sites + .into_iter() + .map(|site| { + let names: Vec<&'static str> = site + .tree_names + .iter() + .map(|name| &*Box::leak(name.clone().into_boxed_str())) + .collect(); + ( + site.start, + site.end, + &*Box::leak(site.name.into_boxed_str()), + &*Box::leak(names.into_boxed_slice()), + site.typed_reference_text, + ) + }) + .collect() + }); + assert_eq!( + actual, + test_case.expected_sites.map(<[ExpectedSite]>::to_vec), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs new file mode 100644 index 0000000000..fc43f46ee6 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_splice_macro_calls.rs @@ -0,0 +1,76 @@ +use crate::macro_calls::main::splice_macro_calls::splice_macro_calls; +use crate::macro_calls::tests::test_types::SpliceMacroCallsTestCase; + +#[test] +fn given_call_sites_and_outputs_when_splicing_then_sql_and_spans_match_python() { + let test_cases = [ + SpliceMacroCallsTestCase { + description: "one call", + sql: "SELECT @a(1) FROM t", + sites: &[(7, 12)], + outputs: &["x+1"], + expected_splice: Some(("SELECT x+1 FROM t", &[(7, 12, 7, 10)])), + }, + SpliceMacroCallsTestCase { + description: "code-point offsets around non-ASCII text", + sql: "é @a() ü @b()", + sites: &[(2, 6), (9, 13)], + outputs: &["ö", "zz"], + expected_splice: Some(("é ö ü zz", &[(2, 6, 2, 3), (9, 13, 6, 8)])), + }, + SpliceMacroCallsTestCase { + description: "no sites leave the text unchanged", + sql: "a @ b", + sites: &[], + outputs: &[], + expected_splice: Some(("a @ b", &[])), + }, + SpliceMacroCallsTestCase { + description: "descending sites are rejected", + sql: "@a() @b()", + sites: &[(5, 9), (0, 4)], + outputs: &["x", "y"], + expected_splice: None, + }, + SpliceMacroCallsTestCase { + description: "out-of-bounds sites are rejected", + sql: "é @a()", + sites: &[(2, 9)], + outputs: &["x"], + expected_splice: None, + }, + ]; + + for test_case in test_cases { + let outputs: Vec = test_case + .outputs + .iter() + .map(|output| (*output).to_owned()) + .collect(); + let actual = + splice_macro_calls(test_case.sql, test_case.sites, &outputs).map(|(sql, spans)| { + ( + sql, + spans + .into_iter() + .map(|span| { + ( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ) + }) + .collect::>(), + ) + }); + assert_eq!( + actual, + test_case + .expected_splice + .map(|(sql, spans)| (sql.to_owned(), spans.to_vec())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs new file mode 100644 index 0000000000..c2c785e736 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs @@ -0,0 +1,26 @@ +/// One expected call site: start, end, name, tree names and whether typed reference text appears. +pub(crate) type ExpectedSite = (usize, usize, &'static str, &'static [&'static str], bool); + +/// One expected splice: the rendered SQL and each span's source and output offsets. +pub(crate) type ExpectedSplice = (&'static str, &'static [(usize, usize, usize, usize)]); + +pub(crate) struct ScanMacroCallSitesTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sites: Option<&'static [ExpectedSite]>, +} + +pub(crate) struct SpliceMacroCallsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) sites: &'static [(usize, usize)], + pub(crate) outputs: &'static [&'static str], + pub(crate) expected_splice: Option, +} + +pub(crate) struct MacroCallMemoTestCase { + pub(crate) description: &'static str, + pub(crate) class_id: u64, + pub(crate) call_text: &'static str, + pub(crate) expected_hit: bool, +} diff --git a/fensu.toml b/fensu.toml index ac01351948..82f714743b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -310,6 +310,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-python", + "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", From e9720f8b1f85334694b38af48c92706c0812b471 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:11:34 +0100 Subject: [PATCH 02/11] feat(native): expose macro call scanning, splicing and the memo to Python A macro_bridge module in the PyO3 crate registers scan_macro_call_sites and splice_macro_calls (run with the GIL released) and the MacroCallMemo class, whose recorded events cross as plain (tag, text, text) rows. --- Cargo.lock | 8 +++ crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/main/register.rs | 1 + crates/sqlbuild-python/src/lib.rs | 1 + .../src/macro_bridge/_helpers/functions.rs | 53 +++++++++++++++ .../src/macro_bridge/_helpers/memo.rs | 60 +++++++++++++++++ .../src/macro_bridge/_helpers/mod.rs | 3 + .../src/macro_bridge/_helpers/rows.rs | 65 +++++++++++++++++++ .../src/macro_bridge/constants.rs | 6 ++ .../src/macro_bridge/main/mod.rs | 1 + .../src/macro_bridge/main/register.rs | 14 ++++ .../sqlbuild-python/src/macro_bridge/mod.rs | 5 ++ .../src/macro_bridge/models.rs | 11 ++++ .../sqlbuild-python/src/macro_bridge/types.rs | 10 +++ src/sqlbuild/_native.pyi | 19 ++++++ 15 files changed, 258 insertions(+) create mode 100644 crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/constants.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/main/mod.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/main/register.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/mod.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/models.rs create mode 100644 crates/sqlbuild-python/src/macro_bridge/types.rs diff --git a/Cargo.lock b/Cargo.lock index ff615f69eb..e9256fd552 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,11 +705,19 @@ dependencies = [ "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", + "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-render" +version = "0.139.0" +dependencies = [ + "sqlbuild-core", +] + [[package]] name = "sqlbuild-rules" version = "0.139.0" diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 7eab2dc618..55aa620ffa 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -25,6 +25,7 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 6e226b6956..6b882ff58b 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -29,6 +29,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; + crate::macro_bridge::main::register::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/lib.rs b/crates/sqlbuild-python/src/lib.rs index 7c2526111e..ba7c854c34 100644 --- a/crates/sqlbuild-python/src/lib.rs +++ b/crates/sqlbuild-python/src/lib.rs @@ -1,3 +1,4 @@ #![forbid(unsafe_code)] pub mod bindings; +mod macro_bridge; diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs new file mode 100644 index 0000000000..bcfd897d80 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs @@ -0,0 +1,53 @@ +//! Call-site scanning and splicing for the Python macro bridge, with the GIL released. + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; +use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; +use sqlbuild_render::macro_calls::models::ScanDeferral; + +use crate::bindings::types::CompilerDetach; +use crate::macro_bridge::_helpers::rows::{site_row, span_row}; +use crate::macro_bridge::types::{SiteRow, SpanRow}; + +fn compiler_error(message: String) -> pyo3::PyErr { + PyRuntimeError::new_err(message) +} + +/// Every top-level macro call site of `sql`, or `None` when Python must expand the string. +#[pyfunction] +pub(crate) fn scan_macro_call_sites( + py: Python<'_>, + sql: &str, + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult>> { + let Some(python) = python_text(python_version, unicode_version) else { + return Ok(None); + }; + let scanned = py + .compiler_detach(|| Ok(scan_sites(python, sql))) + .map_err(compiler_error)?; + match scanned { + Ok(sites) => Ok(Some(sites.into_iter().map(site_row).collect())), + Err(ScanDeferral) => Ok(None), + } +} + +/// Splice rendered macro outputs over their call sites, returning the SQL and substitution spans. +#[pyfunction] +pub(crate) fn splice_macro_calls( + py: Python<'_>, + sql: &str, + sites: Vec<(usize, usize)>, + outputs: Vec, +) -> PyResult<(String, Vec)> { + let spliced = py + .compiler_detach(|| Ok(splice_calls(sql, &sites, &outputs))) + .map_err(compiler_error)?; + let (rendered, spans) = + spliced.ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; + Ok((rendered, spans.into_iter().map(span_row).collect())) +} diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs new file mode 100644 index 0000000000..ba5776e875 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs @@ -0,0 +1,60 @@ +//! Python methods of the in-compile macro call memo. + +use pyo3::exceptions::PyValueError; +use pyo3::{PyResult, pymethods}; +use sqlbuild_render::macro_calls::models::MacroCallEntry; + +use crate::macro_bridge::_helpers::rows::{event_from_row, event_row}; +use crate::macro_bridge::models::MacroCallMemo; +use crate::macro_bridge::types::EntryRow; + +fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("macro call memo is poisoned") +} + +#[pymethods] +impl MacroCallMemo { + #[new] + fn new() -> Self { + Self::default() + } + + /// The recorded result of a call in `class_id`, or `None` when it must execute. + fn lookup(&self, class_id: u64, call_text: &str) -> PyResult> { + let found = self + .inner + .lock() + .map_err(poisoned)? + .lookup(class_id, call_text); + Ok(found.map(|entry| { + ( + entry.sql.clone(), + entry.relations.clone(), + entry.events.iter().map(event_row).collect(), + ) + })) + } + + /// Record the `(sql, relations, events)` result of one executed call. + fn record(&self, class_id: u64, call_text: String, entry: EntryRow) -> PyResult<()> { + let (sql, relations, events) = entry; + let entry = MacroCallEntry { + sql, + relations, + events: events + .into_iter() + .map(event_from_row) + .collect::>()?, + }; + self.inner + .lock() + .map_err(poisoned)? + .record(class_id, call_text, entry); + Ok(()) + } + + /// Hits, misses and recorded results so far. + fn stats(&self) -> PyResult<(usize, usize, usize)> { + Ok(self.inner.lock().map_err(poisoned)?.stats()) + } +} diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs new file mode 100644 index 0000000000..bbdf7c8b94 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod functions; +pub(crate) mod memo; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs new file mode 100644 index 0000000000..cc12477e93 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs @@ -0,0 +1,65 @@ +//! Conversions between native macro bridge models and their Python rows. + +use pyo3::PyResult; +use pyo3::exceptions::PyValueError; +use sqlbuild_render::macro_calls::models::{MacroCallEvent, MacroCallSite, MacroSpliceSpan}; + +use crate::macro_bridge::constants::{ + ARGUMENT_REFERENCE, DECLARATION_READ, GENERATED_SQL, MACRO_USE, +}; +use crate::macro_bridge::types::{EventRow, SiteRow, SpanRow}; + +pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { + ( + site.start, + site.end, + site.name, + site.tree_names, + site.typed_reference_text, + ) +} + +pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { + ( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ) +} + +pub(crate) fn event_row(event: &MacroCallEvent) -> EventRow { + match event { + MacroCallEvent::MacroUse { name } => (MACRO_USE, name.clone(), String::new()), + MacroCallEvent::DeclarationRead { kind, name } => { + (DECLARATION_READ, kind.clone(), name.clone()) + } + MacroCallEvent::GeneratedSql { macro_name, sql } => { + (GENERATED_SQL, macro_name.clone(), sql.clone()) + } + MacroCallEvent::ArgumentReference { kind, name } => { + (ARGUMENT_REFERENCE, kind.clone(), name.clone()) + } + } +} + +pub(crate) fn event_from_row((tag, first, second): EventRow) -> PyResult { + match tag { + MACRO_USE => Ok(MacroCallEvent::MacroUse { name: first }), + DECLARATION_READ => Ok(MacroCallEvent::DeclarationRead { + kind: first, + name: second, + }), + GENERATED_SQL => Ok(MacroCallEvent::GeneratedSql { + macro_name: first, + sql: second, + }), + ARGUMENT_REFERENCE => Ok(MacroCallEvent::ArgumentReference { + kind: first, + name: second, + }), + _ => Err(PyValueError::new_err(format!( + "unknown macro call event tag {tag}" + ))), + } +} diff --git a/crates/sqlbuild-python/src/macro_bridge/constants.rs b/crates/sqlbuild-python/src/macro_bridge/constants.rs new file mode 100644 index 0000000000..13ba6f0ade --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/constants.rs @@ -0,0 +1,6 @@ +//! Macro call event tags shared with `sqlbuild.compiler.macro_bridge.constants`. + +pub(crate) const MACRO_USE: u8 = 0; +pub(crate) const DECLARATION_READ: u8 = 1; +pub(crate) const GENERATED_SQL: u8 = 2; +pub(crate) const ARGUMENT_REFERENCE: u8 = 3; diff --git a/crates/sqlbuild-python/src/macro_bridge/main/mod.rs b/crates/sqlbuild-python/src/macro_bridge/main/mod.rs new file mode 100644 index 0000000000..51101fd370 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod register; diff --git a/crates/sqlbuild-python/src/macro_bridge/main/register.rs b/crates/sqlbuild-python/src/macro_bridge/main/register.rs new file mode 100644 index 0000000000..06cf3f3316 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/main/register.rs @@ -0,0 +1,14 @@ +//! Register the macro bridge's functions and classes on the extension module. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::macro_bridge::_helpers::functions::{scan_macro_call_sites, splice_macro_calls}; +use crate::macro_bridge::models::MacroCallMemo; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; + module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/macro_bridge/mod.rs b/crates/sqlbuild-python/src/macro_bridge/mod.rs new file mode 100644 index 0000000000..c376860cb1 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; +pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/macro_bridge/models.rs b/crates/sqlbuild-python/src/macro_bridge/models.rs new file mode 100644 index 0000000000..0287377097 --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/models.rs @@ -0,0 +1,11 @@ +//! Python classes of the macro bridge. + +use pyo3::pyclass; +use std::sync::Mutex; + +/// The macro call results one compile records and replays, keyed by call class and call text. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug, Default)] +pub(crate) struct MacroCallMemo { + pub(crate) inner: Mutex, +} diff --git a/crates/sqlbuild-python/src/macro_bridge/types.rs b/crates/sqlbuild-python/src/macro_bridge/types.rs new file mode 100644 index 0000000000..c8339393ae --- /dev/null +++ b/crates/sqlbuild-python/src/macro_bridge/types.rs @@ -0,0 +1,10 @@ +//! Python-facing rows of the macro bridge. + +/// One call site: start, end, name, tree names and whether typed reference text appears. +pub(crate) type SiteRow = (usize, usize, String, Vec, bool); +/// One substitution span: source start and end, output start and end. +pub(crate) type SpanRow = (usize, usize, usize, usize); +/// One recorded event: its tag and two text fields. +pub(crate) type EventRow = (u8, String, String); +/// One recorded call: its SQL, added relations and events. +pub(crate) type EntryRow = (String, Vec<(String, String)>, Vec); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5b64d00c95..8d1c6aa1bc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -184,6 +184,25 @@ def substitute_static_project_vars( def extract_static_sql_references( sql: str, ) -> list[tuple[str, str, str | None, int | None]] | None: ... +def scan_macro_call_sites( + sql: str, python_version: tuple[int, int], unicode_version: str +) -> list[tuple[int, int, str, list[str], bool]] | None: ... +def splice_macro_calls( + sql: str, sites: list[tuple[int, int]], outputs: list[str] +) -> tuple[str, list[tuple[int, int, int, int]]]: ... + +class MacroCallMemo: + def __init__(self) -> None: ... + def lookup( + self, class_id: int, call_text: str + ) -> tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None: ... + def record( + self, + class_id: int, + call_text: str, + entry: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]], + ) -> None: ... + def stats(self) -> tuple[int, int, int]: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... From 4cccd2e1cd304b3ea5f3adc47d5ad61be3f0a011 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:11:35 +0100 Subject: [PATCH 03/11] feat(native): batch macro calls through a native render bridge With the native engine, the compile-input render runs every expand_sql_macros_result call through the macro bridge: Rust scans the top-level call sites, Python resolves their call class and executes memo misses in order with the existing _evaluate_macro_call, recorded calls replay their ordered events (macro uses, declaration reads, generated-reference candidates, typed argument references) for each consumer, and Rust splices the results. Relation placeholders render exactly as in the serial path. Any native render failure re-runs the Python stage, which raises the exact error; a re-run that succeeds is reported as a divergence. The Python engine is unchanged. --- .../_helpers/macro_bridge/expansion.py | 202 ++++++++++ .../compile/_helpers/render/macros.py | 284 +++++++------- .../compile/classes/macro_expansion_facts.py | 115 ++++++ src/sqlbuild/compiler/compile/exceptions.py | 4 + src/sqlbuild/compiler/compile/models.py | 14 + .../frontier/_helpers/native_frontier.py | 8 +- .../compiler/macro_bridge/__init__.py | 1 + .../compiler/macro_bridge/classes/__init__.py | 1 + .../macro_bridge/classes/macro_bridge.py | 195 ++++++++++ .../compiler/macro_bridge/constants.py | 18 + .../compiler/macro_bridge/main/__init__.py | 1 + .../macro_bridge/main/active_macro_bridge.py | 13 + .../main/run_with_macro_bridge.py | 47 +++ src/sqlbuild/compiler/macro_bridge/models.py | 16 + src/sqlbuild/compiler/macro_bridge/types.py | 8 + .../cli/commands/main/compile/_test_types.py | 10 + .../compile/test_compiler_engine_switch.py | 39 ++ .../sqlbuild/compiler/compile/_test_types.py | 42 ++ .../src/sqlbuild/compiler/compile/helpers.py | 358 ++++++++++++++++++ .../compile/test_native_macro_bridge.py | 226 +++++++++++ .../compiler/macro_bridge/__init__.py | 0 .../compiler/macro_bridge/main/__init__.py | 0 .../compiler/macro_bridge/main/_test_types.py | 33 ++ .../compiler/macro_bridge/main/helpers.py | 33 ++ .../main/test_run_with_macro_bridge.py | 104 +++++ 25 files changed, 1642 insertions(+), 130 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py create mode 100644 src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py create mode 100644 src/sqlbuild/compiler/macro_bridge/__init__.py create mode 100644 src/sqlbuild/compiler/macro_bridge/classes/__init__.py create mode 100644 src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py create mode 100644 src/sqlbuild/compiler/macro_bridge/constants.py create mode 100644 src/sqlbuild/compiler/macro_bridge/main/__init__.py create mode 100644 src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py create mode 100644 src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py create mode 100644 src/sqlbuild/compiler/macro_bridge/models.py create mode 100644 src/sqlbuild/compiler/macro_bridge/types.py create mode 100644 tests/integration/src/sqlbuild/compiler/compile/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/compile/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/main/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py new file mode 100644 index 0000000000..4c5f8caad4 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -0,0 +1,202 @@ +"""Expand one authored string through the native macro bridge, executing misses in Python.""" + +from __future__ import annotations + +from collections.abc import Hashable +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.render.macros import ( + _evaluate_macro_call, + _expand_resolved_sql_macros, + _expansion_declarations, + _record_macro_declaration_usage, + _record_macro_use, + _report_generated_references, +) +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts +from sqlbuild.compiler.compile.constants import MACRO_TOKEN +from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeRenderMismatchError +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + ExpansionSpan, + MacroExpansionState, +) +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ( + ARGUMENT_REFERENCE_EVENT, + DECLARATION_READ_EVENT, + GENERATED_SQL_EVENT, + MACRO_USE_EVENT, +) +from sqlbuild.compiler.macro_bridge.models import MacroCallSite +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.compiler.scopes.types import DeclarationKind +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind + + +def expand_bridged_sql_macros( + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + bridge: MacroBridge, +) -> tuple[str, tuple[ExpansionSpan, ...]]: + """Expand one string, replaying memoized calls and running the rest in Python in order.""" + + if MACRO_TOKEN not in sql: + return sql, () + declarations: DeclarationResolutionContext | None = _expansion_declarations( + state=state, consumer_path=consumer_path + ) + sites: tuple[MacroCallSite, ...] | None = bridge.scan(sql) + call_classes: list[Hashable] = [] + for site in sites or (): + call_class: Hashable | None = bridge.call_class( + site=site, + declarations=declarations, + loaded_macros=state.loaded_macros, + macro_context=state.macro_context, + ) + if call_class is None: + sites = None + break + call_classes.append(call_class) + if sites is None: + return _expand_resolved_sql_macros( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=declarations, + stack=(), + ) + outputs: list[str] = [ + _bridged_call_output( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=declarations, + bridge=bridge, + site=site, + call_class=call_class, + ) + for site, call_class in zip(sites, call_classes, strict=True) + ] + return bridge.splice(sql=sql, sites=sites, outputs=outputs) + + +def _bridged_call_output( # noqa: PLR0913 + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + bridge: MacroBridge, + site: MacroCallSite, + call_class: Hashable, +) -> str: + facts: MacroExpansionFacts = state.facts + call_text: str = sql[site.start : site.end] + prior_relations: tuple[SqlResourceRef, ...] | None = ( + tuple(facts.relations) if site.typed_reference_text else None + ) + class_id: int + recorded: MacroCallRecord | None + class_id, recorded = bridge.lookup( + call_class=call_class, call_text=call_text, prior_relations=prior_relations + ) + if recorded is not None: + recorded_sql, relations, events = recorded + for event in events: + _replay_macro_call_event( + event=event, + consumer_path=consumer_path, + state=state, + declarations=declarations, + ) + for relation in relations: + _ = facts.relations.setdefault(relation, len(facts.relations)) + return facts.render_relation_placeholders(recorded_sql) + known_relations: int = len(facts.relations) + _ = facts.begin_event_log() + try: + macro_result: object + next_index: int + macro_result, next_index = _evaluate_macro_call( + sql=sql, + call_start_index=site.start, + file_path=consumer_path, + state=state, + declarations=declarations, + stack=(), + top_level=True, + ) + finally: + events, replayable = facts.end_event_log() + if not isinstance(macro_result, str): + raise CompileInputError( + f"Macro '@{site.name}' in '{consumer_path}' must return a SQL string when used " + "directly in SQL" + ) + if next_index != site.end: + raise NativeRenderMismatchError( + f"Native macro call scan of '{consumer_path}' ended at {site.end}, Python at " + f"{next_index}" + ) + added_relations: tuple[SqlResourceRef, ...] = tuple(facts.relations)[known_relations:] + if replayable and (prior_relations is not None or not added_relations): + bridge.record( + class_id=class_id, + call_text=call_text, + sql=macro_result, + relations=added_relations, + events=events, + ) + return facts.render_relation_placeholders(macro_result) + + +def _replay_macro_call_event( + *, + event: MacroCallEvent, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, +) -> None: + tag, first, second = event + if tag == MACRO_USE_EVENT: + _record_macro_use( + macro_name=first, + identity=( + declarations.macro_records[first].identity + if declarations is not None + else DeclarationIdentity(kind=DeclarationKind.MACRO, name=first) + ), + state=state, + declarations=declarations, + stack=(), + ) + elif tag == DECLARATION_READ_EVENT: + if declarations is None: + raise NativeRenderMismatchError("Replayed a declaration read without declarations") + _record_macro_declaration_usage( + name=second, kind=DeclarationKind(first), declarations=declarations, state=state + ) + elif tag == GENERATED_SQL_EVENT: + _report_generated_references( + macro_name=first, + loaded_macro=( + declarations.macros[first] + if declarations is not None + else state.loaded_macros[first] + ), + macro_result=second, + file_path=consumer_path, + state=state, + ) + elif tag == ARGUMENT_REFERENCE_EVENT: + state.facts.record_call_site_refs( + (SqlResourceRef(kind=SqlResourceRefKind(first), name=second),) + ) + else: + raise NativeRenderMismatchError(f"Unknown macro call event {tag}") diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index b71c5ef5a6..f011f320a0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -11,7 +11,7 @@ import threading from collections.abc import Callable, Iterator, Mapping from contextvars import Token -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, replace from pathlib import Path from types import MappingProxyType, ModuleType from typing import cast @@ -19,12 +19,10 @@ from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( call_site_sql_references, evaluate_typed_reference, - reference_call_text, reject_macro_generated_references, - relation_placeholder_text, - render_relation_placeholders, report_macro_reference_call_syntax, ) +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import ( DECLARATION_REFERENCE_NAMES, MACRO_CONTEXT_PARAMETER_NAME, @@ -41,6 +39,7 @@ LoadedMacro, MacroContext, MacroExpansionResult, + MacroExpansionState, StaticMacroExport, StaticMacroFault, StaticMacroInventory, @@ -50,6 +49,14 @@ DiscoveredMacroFile, EnumDeclaration, ) +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ( + DECLARATION_READ_EVENT, + GENERATED_REFERENCE_MARKERS, + GENERATED_SQL_EVENT, + MACRO_USE_EVENT, +) +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) @@ -104,83 +111,6 @@ class _MacroAnalysisInputs: package_names: frozenset[str] -@dataclass -class _ExpansionFacts: - dependencies: list[DeclarationIdentity] = field(default_factory=list) - usages: list[UsageRecord] = field(default_factory=list) - relations: dict[SqlResourceRef, int] = field(default_factory=dict) - argument_references: dict[SqlResourceRef, None] = field(default_factory=dict) - call_site_refs: list[set[SqlResourceRef]] = field(default_factory=list) - rendering_approved: list[frozenset[SqlResourceRef]] = field(default_factory=list) - - def add_dependency(self, identity: DeclarationIdentity) -> _ExpansionFacts: - """Record a resolved dependency in encounter order.""" - - self.dependencies.append(identity) - return self - - def add_usage(self, usage: UsageRecord) -> _ExpansionFacts: - """Record a resolved macro-to-macro edge in encounter order.""" - - self.usages.append(usage) - return self - - def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None: - """Record references written as arguments of every macro call being expanded.""" - - self.argument_references.update(dict.fromkeys(refs)) - for approved in self.call_site_refs: - approved.update(refs) - - def open_call_site(self) -> _ExpansionFacts: - """Start collecting the references written as one macro call's arguments.""" - - self.call_site_refs.append(set()) - return self - - def close_call_site(self) -> frozenset[SqlResourceRef]: - """Stop collecting for the innermost macro call and return its argument references.""" - - return frozenset(self.call_site_refs.pop()) - - def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> _ExpansionFacts: - """Approve the call-site argument references of the macro about to run.""" - - self.rendering_approved.append(approved) - return self - - def end_rendering(self) -> _ExpansionFacts: - """Restore the approvals of the enclosing macro after one macro returns.""" - - self.rendering_approved.pop() - return self - - def render_relation(self, ref: object) -> str: - """Render a reference a macro formats; only call-site arguments become placeholders.""" - - if not isinstance(ref, SqlResourceRef): - raise CompileInputError("Only typed resource references render as relations") - if not self.rendering_approved or ref not in self.rendering_approved[-1]: - return reference_call_text(ref) - return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations))) - - def render_relation_placeholders(self, sql: str) -> str: - """Replace typed reference placeholders with the reference call written at the call site.""" - - return render_relation_placeholders(sql=sql, relations=self.relations) - - -@dataclass(frozen=True) -class _ExpansionState: - loaded_macros: dict[str, LoadedMacro] - macro_overrides: dict[str, str] - macro_context: MacroContext - declaration_resolver: DeclarationScopeResolver | None - declarations: DeclarationResolutionContext | None - facts: _ExpansionFacts - consumer: ResourceIdentity | DeclarationIdentity | None = None - - class _MacroDeclarationValues(Mapping[str, object]): def __init__( self, @@ -190,17 +120,20 @@ def __init__( declaration_kind: str, file_path: Path, on_access: Callable[[str], None], + on_miss: Callable[[], None], ) -> None: self._values: Mapping[str, object] = values self._inaccessible: Mapping[str, DeclarationRecord] = inaccessible self._declaration_kind: str = declaration_kind self._file_path: Path = file_path self._on_access: Callable[[str], None] = on_access + self._on_miss: Callable[[], None] = on_miss def __getitem__(self, name: str) -> object: if name in self._values: self._on_access(name) return self._values[name] + self._on_miss() inaccessible: DeclarationRecord | None = self._inaccessible.get(name) if inaccessible is not None: owner: str = inaccessible.owning_path or "global" @@ -227,14 +160,23 @@ def __len__(self) -> int: class _MacroEnumMembers(Mapping[str, str | int]): - def __init__(self, *, enum_name: str, values: Mapping[str, str | int], file_path: Path) -> None: + def __init__( + self, + *, + enum_name: str, + values: Mapping[str, str | int], + file_path: Path, + on_miss: Callable[[], None], + ) -> None: self._enum_name: str = enum_name self._values: Mapping[str, str | int] = values self._file_path: Path = file_path + self._on_miss: Callable[[], None] = on_miss def __getitem__(self, name: str) -> str | int: if name in self._values: return self._values[name] + self._on_miss() available: str = ", ".join(sorted(self._values)) or "none" raise MacroDeclarationLookupError( f"Unknown member '{name}' for enum '{self._enum_name}' in '{self._file_path}'. " @@ -1256,8 +1198,8 @@ def expand_sql_macros_result( ) -> MacroExpansionResult: """Expand macros and retain deterministic resolved dependency facts.""" - facts: _ExpansionFacts = _ExpansionFacts() - state: _ExpansionState = _ExpansionState( + facts: MacroExpansionFacts = MacroExpansionFacts() + state: MacroExpansionState = MacroExpansionState( loaded_macros=loaded_macros, macro_overrides={} if macro_overrides is None else macro_overrides, macro_context=macro_context, @@ -1266,11 +1208,21 @@ def expand_sql_macros_result( facts=facts, consumer=consumer, ) - expanded_sql, spans = _expand_sql_macros( - sql=sql, - consumer_path=file_path, - state=state, - stack=(), + bridge: MacroBridge | None = active_macro_bridge() if not state.macro_overrides else None + expanded_sql, spans = ( + _expand_sql_macros( + sql=sql, + consumer_path=file_path, + state=state, + stack=(), + ) + if bridge is None + else _bridged_sql_macros( + sql=sql, + consumer_path=file_path, + state=state, + bridge=bridge, + ) ) return MacroExpansionResult( sql=expanded_sql, @@ -1281,16 +1233,39 @@ def expand_sql_macros_result( ) +def _bridged_sql_macros( + *, sql: str, consumer_path: Path, state: MacroExpansionState, bridge: MacroBridge +) -> tuple[str, tuple[ExpansionSpan, ...]]: + from sqlbuild.compiler.compile._helpers.macro_bridge.expansion import ( + expand_bridged_sql_macros, + ) + + return expand_bridged_sql_macros( + sql=sql, consumer_path=consumer_path, state=state, bridge=bridge + ) + + def _expand_sql_macros( *, sql: str, consumer_path: Path, - state: _ExpansionState, + state: MacroExpansionState, stack: tuple[DeclarationIdentity, ...], ) -> tuple[str, tuple[ExpansionSpan, ...]]: if MACRO_TOKEN not in sql: return sql, () + return _expand_resolved_sql_macros( + sql=sql, + consumer_path=consumer_path, + state=state, + declarations=_expansion_declarations(state=state, consumer_path=consumer_path), + stack=stack, + ) + +def _expansion_declarations( + *, state: MacroExpansionState, consumer_path: Path +) -> DeclarationResolutionContext | None: declarations: DeclarationResolutionContext | None = state.declarations if declarations is None and state.declaration_resolver is not None: from sqlbuild.compiler.compile._helpers.render.declarations import ( @@ -1300,7 +1275,17 @@ def _expand_sql_macros( declarations = resolve_declaration_context( resolver=state.declaration_resolver, file_path=consumer_path ) + return declarations + +def _expand_resolved_sql_macros( + *, + sql: str, + consumer_path: Path, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + stack: tuple[DeclarationIdentity, ...], +) -> tuple[str, tuple[ExpansionSpan, ...]]: rendered_sql_parts: list[str] = [] spans: list[ExpansionSpan] = [] output_length: int = 0 @@ -1399,7 +1384,7 @@ def _evaluate_macro_call( sql: str, call_start_index: int, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], top_level: bool, @@ -1452,29 +1437,13 @@ def _evaluate_macro_call( raise CompileInputError( f"Macro expansion cycle detected: {chain}. Definition paths: {paths}" ) - _ = state.facts.add_dependency(identity) - if stack: - _ = state.facts.add_usage( - UsageRecord(stack[-1], identity, UsageKind.DECLARATION_DEPENDENCY), - ) - elif state.consumer is not None: - visibility: tuple[VisibilityRecord, ...] = ( - declarations.macro_visibility.get(macro_name, ()) if declarations is not None else () - ) - if visibility: - from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility - - for visible in usage_visibility(visibility=visibility, consumer=state.consumer): - _ = state.facts.add_usage( - UsageRecord( - state.consumer, - identity, - UsageKind.RUNTIME, - through=visible.through, - ) - ) - else: - _ = state.facts.add_usage(UsageRecord(state.consumer, identity, UsageKind.RUNTIME)) + _record_macro_use( + macro_name=macro_name, + identity=identity, + state=state, + declarations=declarations, + stack=stack, + ) args_source: str = sql[opening_paren_index + 1 : closing_paren_index] args: tuple[object, ...] kwargs: dict[str, object] @@ -1528,34 +1497,88 @@ def _evaluate_macro_call( _validate_final_macro_sql( macro_name=macro_name, file_path=file_path, macro_result=macro_result ) - report_macro_reference_call_syntax( + _report_generated_references( + macro_name=macro_name, loaded_macro=loaded_macro, macro_result=macro_result, file_path=file_path, - consumer=state.consumer, + state=state, ) - if state.macro_context._enforce_explicit_references: - reject_macro_generated_references( - loaded_macro=loaded_macro, - macro_result=macro_result, - file_path=file_path, - consumer=state.consumer, - ) return macro_result, closing_paren_index + 1 +def _record_macro_use( + *, + macro_name: str, + identity: DeclarationIdentity, + state: MacroExpansionState, + declarations: DeclarationResolutionContext | None, + stack: tuple[DeclarationIdentity, ...], +) -> None: + state.facts.note_event((MACRO_USE_EVENT, macro_name, "")) + _ = state.facts.add_dependency(identity) + if stack: + state.facts.block_memo() + _ = state.facts.add_usage( + UsageRecord(stack[-1], identity, UsageKind.DECLARATION_DEPENDENCY), + ) + elif state.consumer is not None: + visibility: tuple[VisibilityRecord, ...] = ( + declarations.macro_visibility.get(macro_name, ()) if declarations is not None else () + ) + if visibility: + from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility + + for visible in usage_visibility(visibility=visibility, consumer=state.consumer): + _ = state.facts.add_usage( + UsageRecord( + state.consumer, + identity, + UsageKind.RUNTIME, + through=visible.through, + ) + ) + else: + _ = state.facts.add_usage(UsageRecord(state.consumer, identity, UsageKind.RUNTIME)) + + +def _report_generated_references( + *, + macro_name: str, + loaded_macro: LoadedMacro, + macro_result: str, + file_path: Path, + state: MacroExpansionState, +) -> None: + if any(marker in macro_result for marker in GENERATED_REFERENCE_MARKERS): + state.facts.note_event((GENERATED_SQL_EVENT, macro_name, macro_result)) + report_macro_reference_call_syntax( + loaded_macro=loaded_macro, + macro_result=macro_result, + file_path=file_path, + consumer=state.consumer, + ) + if state.macro_context._enforce_explicit_references: + reject_macro_generated_references( + loaded_macro=loaded_macro, + macro_result=macro_result, + file_path=file_path, + consumer=state.consumer, + ) + + def _build_macro_invocation_context( *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, ) -> MacroContext: if declarations is None: return macro_context constant_values: Mapping[str, object] = _build_constant_context_values(declarations.constants) enum_values: Mapping[str, object] = _build_enum_context_values( - declarations=declarations.enums, file_path=file_path + declarations=declarations.enums, file_path=file_path, on_miss=state.facts.block_memo ) constants: Mapping[str, object] = _MacroDeclarationValues( values=constant_values, @@ -1568,6 +1591,7 @@ def _build_macro_invocation_context( declarations=declarations, state=state, ), + on_miss=state.facts.block_memo, ) enum_mapping: Mapping[str, object] = _MacroDeclarationValues( values=enum_values, @@ -1580,6 +1604,7 @@ def _build_macro_invocation_context( declarations=declarations, state=state, ), + on_miss=state.facts.block_memo, ) return replace( macro_context, @@ -1602,6 +1627,7 @@ def _build_enum_context_values( *, declarations: Mapping[str, EnumDeclaration], file_path: Path, + on_miss: Callable[[], None], ) -> Mapping[str, object]: values: dict[str, object] = {} for name, declaration in declarations.items(): @@ -1612,6 +1638,7 @@ def _build_enum_context_values( enum_name=name, values=MappingProxyType(members), file_path=file_path, + on_miss=on_miss, ) return MappingProxyType(values) @@ -1660,10 +1687,11 @@ def _record_macro_declaration_usage( name: str, kind: DeclarationKind, declarations: DeclarationResolutionContext, - state: _ExpansionState, + state: MacroExpansionState, ) -> None: from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility + state.facts.note_event((DECLARATION_READ_EVENT, kind.value, name)) consumer: ResourceIdentity | DeclarationIdentity | None = ( state.consumer or declarations.consumer ) @@ -1738,7 +1766,7 @@ def _parse_macro_arguments( *, args_source: str, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], ) -> tuple[tuple[object, ...], dict[str, object]]: @@ -1796,7 +1824,7 @@ def _rewrite_nested_macro_calls( *, args_source: str, file_path: Path, - state: _ExpansionState, + state: MacroExpansionState, declarations: DeclarationResolutionContext | None, stack: tuple[DeclarationIdentity, ...], ) -> tuple[str, dict[str, object]]: diff --git a/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py b/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py new file mode 100644 index 0000000000..e59c882b93 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py @@ -0,0 +1,115 @@ +"""Facts recorded while expanding the macros of one authored SQL string.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.macro_bridge.constants import ARGUMENT_REFERENCE_EVENT +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent +from sqlbuild.compiler.scopes.models import DeclarationIdentity, UsageRecord +from sqlbuild.python_nodes.models import SqlResourceRef + + +class MacroExpansionFacts: + """Facts one authored string's macro expansion records, in encounter order.""" + + def __init__(self) -> None: + self.dependencies: list[DeclarationIdentity] = [] + self.usages: list[UsageRecord] = [] + self.relations: dict[SqlResourceRef, int] = {} + self.argument_references: dict[SqlResourceRef, None] = {} + self.call_site_refs: list[set[SqlResourceRef]] = [] + self.rendering_approved: list[frozenset[SqlResourceRef]] = [] + self.event_log: list[MacroCallEvent] | None = None + self.memo_blocked: bool = False + + def begin_event_log(self) -> MacroExpansionFacts: + """Start recording the consumer-independent facts of one top-level macro call.""" + + self.event_log = [] + self.memo_blocked = False + return self + + def end_event_log(self) -> tuple[tuple[MacroCallEvent, ...], bool]: + """Stop recording and return the facts and whether the call may be replayed.""" + + events: tuple[MacroCallEvent, ...] = tuple(self.event_log or ()) + self.event_log = None + return events, not self.memo_blocked + + def note_event(self, event: MacroCallEvent) -> None: + """Record one fact of the macro call being executed, when recording.""" + + if self.event_log is not None: + self.event_log.append(event) + + def block_memo(self) -> None: + """Mark the executing call as dependent on its consumer, so it is never replayed.""" + + self.memo_blocked = True + + def add_dependency(self, identity: DeclarationIdentity) -> MacroExpansionFacts: + """Record a resolved dependency in encounter order.""" + + self.dependencies.append(identity) + return self + + def add_usage(self, usage: UsageRecord) -> MacroExpansionFacts: + """Record a resolved macro-to-macro edge in encounter order.""" + + self.usages.append(usage) + return self + + def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None: + """Record references written as arguments of every macro call being expanded.""" + + for ref in refs: + self.note_event((ARGUMENT_REFERENCE_EVENT, ref.kind.value, ref.name)) + self.argument_references.update(dict.fromkeys(refs)) + for approved in self.call_site_refs: + approved.update(refs) + + def open_call_site(self) -> MacroExpansionFacts: + """Start collecting the references written as one macro call's arguments.""" + + self.call_site_refs.append(set()) + return self + + def close_call_site(self) -> frozenset[SqlResourceRef]: + """Stop collecting for the innermost macro call and return its argument references.""" + + return frozenset(self.call_site_refs.pop()) + + def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> MacroExpansionFacts: + """Approve the call-site argument references of the macro about to run.""" + + self.rendering_approved.append(approved) + return self + + def end_rendering(self) -> MacroExpansionFacts: + """Restore the approvals of the enclosing macro after one macro returns.""" + + self.rendering_approved.pop() + return self + + def render_relation(self, ref: object) -> str: + """Render a reference a macro formats; only call-site arguments become placeholders.""" + + from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( + reference_call_text, + relation_placeholder_text, + ) + + if not isinstance(ref, SqlResourceRef): + raise CompileInputError("Only typed resource references render as relations") + if not self.rendering_approved or ref not in self.rendering_approved[-1]: + return reference_call_text(ref) + return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations))) + + def render_relation_placeholders(self, sql: str) -> str: + """Replace typed reference placeholders with the reference call written at the call site.""" + + from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( + render_relation_placeholders, + ) + + return render_relation_placeholders(sql=sql, relations=self.relations) diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 5d849a9b26..2dbd139c49 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -19,6 +19,10 @@ class MacroDeclarationLookupError(CompileInputError, KeyError): """Retain authored diagnostics while honoring the Mapping lookup contract.""" +class NativeRenderMismatchError(RuntimeError): + """Raised when native rendering disagrees with Python, so the Python stage must re-run.""" + + class NativeSqlTestResponseError(CompileInputError): """Raised when native SQL-test extraction returns malformed data.""" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index c924443f2a..7d54966f76 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -12,6 +12,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( @@ -388,6 +389,19 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: return render_enum_member_value(value=value) +@dataclass(frozen=True) +class MacroExpansionState: + """Inputs and recorded facts of one authored string's macro expansion.""" + + loaded_macros: dict[str, LoadedMacro] + macro_overrides: dict[str, str] + macro_context: MacroContext + declaration_resolver: DeclarationScopeResolver | None + declarations: DeclarationResolutionContext | None + facts: MacroExpansionFacts + consumer: ResourceIdentity | DeclarationIdentity | None = None + + @dataclass(frozen=True) class MacroExpansionResult: """Expanded SQL and resolved macro usage facts from one authored string.""" diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py index 13927c6920..1505be9c60 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -3,6 +3,7 @@ from collections.abc import Callable from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge def native_frontier[T]( @@ -13,5 +14,8 @@ def native_frontier[T]( ) -> T: """Produce the frontier object for `until` natively; stages not yet native run in Python.""" - _ = until - return python_stage() if native_stage is None else native_stage() + if native_stage is not None: + return native_stage() + if until is CompilerStage.COMPILE_PROJECT_INPUTS: + return run_with_macro_bridge(stage=python_stage) + return python_stage() diff --git a/src/sqlbuild/compiler/macro_bridge/__init__.py b/src/sqlbuild/compiler/macro_bridge/__init__.py new file mode 100644 index 0000000000..73c3a31a12 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/__init__.py @@ -0,0 +1 @@ +"""Native macro bridge: call-site scans, the in-compile call memo and splicing.""" diff --git a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py new file mode 100644 index 0000000000..2e8ae1fd31 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py @@ -0,0 +1 @@ +"""Macro bridge runtime classes.""" diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py new file mode 100644 index 0000000000..ca378276c5 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -0,0 +1,195 @@ +"""Per-compile state of the native macro bridge: call-site scans, call classes and the memo.""" + +from __future__ import annotations + +from collections.abc import Hashable, Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + ExpansionSpan, + LoadedMacro, + MacroContext, +) +from sqlbuild.compiler.macro_bridge.models import MacroCallSite +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind + + +class MacroBridge: + """Scan, memoize and splice macro calls for one compile; Python still runs every macro.""" + + def __init__(self, *, python_version: tuple[int, int], unicode_version: str) -> None: + self._python_version: tuple[int, int] = python_version + self._unicode_version: str = unicode_version + self._memo: _native.MacroCallMemo = _native.MacroCallMemo() + self._class_ids: dict[Hashable, int] = {} + self._retained: dict[int, object] = {} + self._name_tokens: dict[tuple[int, str], Hashable | None] = {} + self._context_tokens: dict[tuple[int, int], Hashable] = {} + + def scan(self, sql: str) -> tuple[MacroCallSite, ...] | None: + """Return the top-level call sites of `sql`, or None when Python must expand it.""" + + rows: list[tuple[int, int, str, list[str], bool]] | None = _native.scan_macro_call_sites( + sql, self._python_version, self._unicode_version + ) + if rows is None: + return None + return tuple( + MacroCallSite( + start=start, + end=end, + name=name, + tree_names=tuple(tree_names), + typed_reference_text=typed_reference_text, + ) + for start, end, name, tree_names, typed_reference_text in rows + ) + + def call_class( + self, + *, + site: MacroCallSite, + declarations: DeclarationResolutionContext | None, + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + ) -> Hashable | None: + """Return what a call's result may depend on besides its text, or None if unresolvable.""" + + owner: object = loaded_macros if declarations is None else declarations + macros: Mapping[str, LoadedMacro] = ( + loaded_macros if declarations is None else declarations.macros + ) + _ = self._retained.setdefault(id(owner), owner) + _ = self._retained.setdefault(id(macro_context), macro_context) + tokens: list[Hashable] = [] + for name in site.tree_names: + token: Hashable | None = self._name_token( + owner=owner, macros=macros, declarations=declarations, name=name + ) + if token is None: + return None + tokens.append(token) + injects_context: bool = any(macros[name].injects_context for name in site.tree_names) + context: Hashable | None = ( + self._context_token(owner=owner, declarations=declarations, macro_context=macro_context) + if injects_context + else None + ) + return (tuple(tokens), context) + + def lookup( + self, + *, + call_class: Hashable, + call_text: str, + prior_relations: tuple[SqlResourceRef, ...] | None, + ) -> tuple[int, MacroCallRecord | None]: + """Return the class id of a call and its recorded result, if one was recorded.""" + + class_key: Hashable = (call_class, prior_relations) + class_id: int = self._class_ids.setdefault(class_key, len(self._class_ids)) + found: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None = ( + self._memo.lookup(class_id, call_text) + ) + if found is None: + return class_id, None + sql, relations, events = found + return class_id, ( + sql, + tuple( + SqlResourceRef(kind=SqlResourceRefKind(kind), name=name) for kind, name in relations + ), + tuple(events), + ) + + def record( + self, + *, + class_id: int, + call_text: str, + sql: str, + relations: tuple[SqlResourceRef, ...], + events: tuple[MacroCallEvent, ...], + ) -> None: + """Record the result of one executed call for later calls of the same class and text.""" + + self._memo.record( + class_id, + call_text, + (sql, [(relation.kind.value, relation.name) for relation in relations], list(events)), + ) + + def splice( + self, *, sql: str, sites: tuple[MacroCallSite, ...], outputs: list[str] + ) -> tuple[str, tuple[ExpansionSpan, ...]]: + """Replace each call site with its output, returning the SQL and substitution spans.""" + + rendered: str + spans: list[tuple[int, int, int, int]] + rendered, spans = _native.splice_macro_calls( + sql, [(site.start, site.end) for site in sites], outputs + ) + return rendered, tuple( + ExpansionSpan( + source_start=source_start, + source_end=source_end, + output_start=output_start, + output_end=output_end, + ) + for source_start, source_end, output_start, output_end in spans + ) + + def stats(self) -> tuple[int, int, int]: + """Memo hits, misses and recorded results so far.""" + + return self._memo.stats() + + def _name_token( + self, + *, + owner: object, + macros: Mapping[str, LoadedMacro], + declarations: DeclarationResolutionContext | None, + name: str, + ) -> Hashable | None: + key: tuple[int, str] = (id(owner), name) + if key in self._name_tokens: + return self._name_tokens[key] + loaded: LoadedMacro | None = macros.get(name) + token: Hashable | None = ( + None + if loaded is None + or (declarations is not None and name in declarations.inaccessible_macros) + else ( + name, + id(loaded), + declarations.macro_records[name].identity if declarations is not None else None, + ) + ) + self._name_tokens[key] = token + return token + + def _context_token( + self, + *, + owner: object, + declarations: DeclarationResolutionContext | None, + macro_context: MacroContext, + ) -> Hashable: + key: tuple[int, int] = (id(owner), id(macro_context)) + token: Hashable | None = self._context_tokens.get(key) + if token is None: + token = ( + id(macro_context), + None + if declarations is None + else ( + tuple((name, id(value)) for name, value in declarations.constants.items()), + tuple((name, id(value)) for name, value in declarations.enums.items()), + ), + ) + self._context_tokens[key] = token + return token diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py new file mode 100644 index 0000000000..aee9b7ede8 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -0,0 +1,18 @@ +"""Macro call event tags shared with the native memo, and the active bridge of a render.""" + +from __future__ import annotations + +from contextvars import ContextVar + +from sqlbuild.compiler.references.types import SqlReferenceKind + +MACRO_USE_EVENT: int = 0 +DECLARATION_READ_EVENT: int = 1 +GENERATED_SQL_EVENT: int = 2 +ARGUMENT_REFERENCE_EVENT: int = 3 +GENERATED_REFERENCE_MARKERS: tuple[str, ...] = tuple( + f"{kind.function_name}(" for kind in SqlReferenceKind +) +ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( + "sqlbuild_active_macro_bridge", default=None +) diff --git a/src/sqlbuild/compiler/macro_bridge/main/__init__.py b/src/sqlbuild/compiler/macro_bridge/main/__init__.py new file mode 100644 index 0000000000..52c83b9999 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/__init__.py @@ -0,0 +1 @@ +"""Public macro bridge entry points.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py new file mode 100644 index 0000000000..9196fb60e1 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/active_macro_bridge.py @@ -0,0 +1,13 @@ +"""The macro bridge of the running native render stage.""" + +from __future__ import annotations + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE + + +def active_macro_bridge() -> MacroBridge | None: + """Return the macro bridge of the running native render stage, if any.""" + + bridge: object | None = ACTIVE_MACRO_BRIDGE.get() + return bridge if isinstance(bridge, MacroBridge) else None diff --git a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py new file mode 100644 index 0000000000..981f174d6e --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py @@ -0,0 +1,47 @@ +"""Run one render stage through the native macro bridge, re-running Python on any failure.""" + +from __future__ import annotations + +import logging +import sys +import unicodedata +from collections.abc import Callable +from contextvars import Token + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.exceptions import NativeRenderMismatchError +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE + + +def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: + """Run `stage` with the bridge; a failure re-runs Python, which raises the exact error.""" + + python_version: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) + if not _native.native_text_supported(python_version, unicodedata.unidata_version): + return stage() + bridge: MacroBridge = MacroBridge( + python_version=python_version, unicode_version=unicodedata.unidata_version + ) + token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) + native_error: Exception | None = None + try: + result: T = stage() + except Exception as error: + native_error = error + else: + hits, misses, recorded = bridge.stats() + logging.getLogger(__name__).debug( + "Native macro bridge: %d memo hits, %d misses, %d recorded calls", + hits, + misses, + recorded, + ) + return result + finally: + ACTIVE_MACRO_BRIDGE.reset(token) + _ = stage() + raise NativeRenderMismatchError( + "The native macro bridge failed where the Python render succeeded; " + "run with SQLBUILD_COMPILER_ENGINE=python" + ) from native_error diff --git a/src/sqlbuild/compiler/macro_bridge/models.py b/src/sqlbuild/compiler/macro_bridge/models.py new file mode 100644 index 0000000000..3f520b5f92 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/models.py @@ -0,0 +1,16 @@ +"""Macro call sites found by the native scan.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroCallSite: + """One top-level macro call in an authored string, in Python string offsets.""" + + start: int + end: int + name: str + tree_names: tuple[str, ...] + typed_reference_text: bool diff --git a/src/sqlbuild/compiler/macro_bridge/types.py b/src/sqlbuild/compiler/macro_bridge/types.py new file mode 100644 index 0000000000..767a96c96c --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/types.py @@ -0,0 +1,8 @@ +"""Macro bridge type aliases.""" + +from __future__ import annotations + +from sqlbuild.python_nodes.models import SqlResourceRef + +type MacroCallEvent = tuple[int, str, str] +type MacroCallRecord = tuple[str, tuple[SqlResourceRef, ...], tuple[MacroCallEvent, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7539197dfc..9dbdf12564 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -808,6 +808,16 @@ class CompilerEngineParityTestCase: expected_exit_codes: tuple[int, int] +@dataclass(frozen=True) +class CompilerEngineMacroParityTestCase: + """Two engines compiling separate copies of one macro-heavy project.""" + + description: str + files: dict[str, str] + expected_exit_codes: tuple[int, int] + expected_compiled_fragments: tuple[str, ...] + + @dataclass(frozen=True) class CompilerEngineStoreTestCase: """A sequence of engine compiles in one project and which of them reuse a stored compile.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py index 12439f824d..dcabbd8a75 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -7,6 +7,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompilerEngineMacroParityTestCase, CompilerEngineParityTestCase, CompilerEngineRulesStoreTestCase, CompilerEngineStoreTestCase, @@ -22,6 +23,10 @@ report_without_engine, store_digests, ) +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + write_project, +) @pytest.mark.parametrize( @@ -92,6 +97,40 @@ def test_given_no_engine_selection_when_compiling_then_native_runs_and_matches_p assert oracle.compiled +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineMacroParityTestCase( + description="macros_in_models_hooks_tests_audits_sources_functions", + files={ + path: content.replace("WHERE id IN @generated_join()\n", "") + for path, content in MACRO_BRIDGE_PROJECT_FILES.items() + }, + expected_exit_codes=(0, 0), + expected_compiled_fragments=("amount * 21", "'__SQLBUILD_RELATION_1__'"), + ) + ], + ids=lambda case: case.description, +) +def test_given_macro_heavy_project_when_compiling_with_each_engine_then_outputs_are_identical( + test_case: CompilerEngineMacroParityTestCase, tmp_path: Path +) -> None: + python_run: CompileReuseRun = engine_reuse_compile( + project_dir=write_project(root=tmp_path / "python", files=test_case.files), + engine="python", + ) + native_run: CompileReuseRun = engine_reuse_compile( + project_dir=write_project(root=tmp_path / "native", files=test_case.files), + engine="native", + ) + compiled_text: str = b"".join(python_run.compiled.values()).decode() + + assert (python_run.returncode, native_run.returncode) == test_case.expected_exit_codes + assert report_without_engine(native_run) == report_without_engine(python_run) + assert native_run.compiled == python_run.compiled + assert all(map(compiled_text.__contains__, test_case.expected_compiled_fragments)) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compile/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py new file mode 100644 index 0000000000..2b8f407f13 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroBridgeParityTestCase: + """A macro-heavy project both engines must render to identical compile inputs.""" + + description: str + files: dict[str, str] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class MacroBridgeMemoTestCase: + """A project whose repeated macro calls run once per call class under the native engine.""" + + description: str + files: dict[str, str] + expected_python_executions: int + expected_native_executions: int + + +@dataclass(frozen=True) +class MacroBridgeFailureTestCase: + """A project whose rendering fails identically under both engines.""" + + description: str + files: dict[str, str] + expected_message_fragment: str + + +@dataclass(frozen=True) +class MacroExpansionDifferentialTestCase: + """Seeded random SQL text that must expand identically with and without the bridge.""" + + description: str + seed: int + samples: int + expected_minimum_successes: int + expected_mismatches: list[tuple[object, object, object]] diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py new file mode 100644 index 0000000000..36e6be4a4a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -0,0 +1,358 @@ +"""Macro-heavy projects, engine-switched render runs and bridged-expansion outcomes.""" + +from __future__ import annotations + +import random +import re +from collections.abc import Callable +from contextvars import Token +from functools import partial +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, + CompileProjectInputs, + LoadedMacro, + MacroContext, + MacroExpansionResult, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE +from sqlbuild.compiler.scopes.models import ResourceIdentity +from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( + resolve_effective_collection_rendering, +) + +MACRO_CALL_LOG_ENV_VAR: str = "SQLBUILD_TEST_MACRO_CALL_LOG" +_RUN_ID_PATTERN: re.Pattern[str] = re.compile(r'"run_id": "[^"]*"') +_DIFFERENTIAL_FRAGMENTS: tuple[str, ...] = ( + "@m('a')", "@m(1)", "@m()", "@mm(2)", "@mm (2)", "@m (1)", "@n(@m('b'))", "@n(@m(@n()))", + "@n(@mm(3), '@m(4)')", "@tag(__ref('orders'))", "@tag(__seed('rates'))", + "@tag(__ref('customers'))", "@cols(__ref('orders'))", "@gen()", "@nope(1)", "@m(x)", + "@ab c(1)", "@m('é')", "'@m(1)'", "-- @m(1)\n", "/* @n() */", "$$@m()$$", "`@m()`", "@@v", + "@", " ", "\n", "\u3000", "é", "'", "(", ")", "SELECT ", ", ", "\"q\"", +) # fmt: skip +_DIFFERENTIAL_FILE: Path = Path("models/orders_summary.sql") + +MACRO_BRIDGE_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "bridge_demo"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "bridge.duckdb"\n\n' + "[vars]\n" + 'region = "north"\n\n' + "[targets.dev]\n" + 'schema = "analytics"\n\n' + "[settings]\n" + "sql_analysis = false\n\n" + "[scopes]\n" + "enforce_placement = false\n" + ), + "macros/common.py": ( + '"""Shared macros for the bridge demo."""\n\n' + "import os\n\n\n" + "def cents(column: str) -> str:\n" + ' """Convert an amount to cents."""\n' + ' return f"{column} * 100"\n\n\n' + "def wrap(expression: str) -> str:\n" + ' """Parenthesise an expression."""\n' + ' return f"({expression})"\n\n\n' + "def region_filter(ctx) -> str:\n" + ' """Compare the region column with the configured region."""\n' + " return f\"region = '{ctx.vars['region']}'\"\n\n\n" + "def scale(ctx, column: str) -> str:\n" + ' """Scale a column by every visible rate."""\n' + ' return f"{column} * {sum(ctx.constants.values())}"\n\n\n' + "def visible_constants(ctx) -> str:\n" + ' """List the visible constants."""\n' + " return repr(', '.join(ctx.constants))\n\n\n" + "def status_code(ctx) -> str:\n" + ' """Render the placed status."""\n' + " return ctx.render_enum_member(enum_name='order_status', member_name='PLACED')\n\n\n" + "def optional_label(ctx) -> str:\n" + ' """Read an optional constant."""\n' + " return repr(ctx.constants.get('missing_label', 'none'))\n\n\n" + "def columns_of(relation) -> str:\n" + ' """Select every column of a relation."""\n' + ' return f"SELECT * FROM {relation}"\n\n\n' + "def ref_tag(relation) -> str:\n" + ' """Expose the in-expansion relation placeholder text."""\n' + " return repr(str(relation).upper())\n\n\n" + "def generated_join() -> str:\n" + ' """Emit a reference instead of receiving it."""\n' + " return '(SELECT id FROM __ref(\"customers\"))'\n\n\n" + "def logged(column: str) -> str:\n" + ' """Record each execution in the test log."""\n' + f' path = os.environ.get("{MACRO_CALL_LOG_ENV_VAR}")\n' + " if path:\n" + ' with open(path, "a", encoding="utf-8") as log:\n' + ' log.write(column + "\\n")\n' + " return column\n" + ), + "enums/order_status.sql": ( + 'ENUM (\n name order_status,\n members (PLACED "placed", SHIPPED "shipped"),\n);\n' + ), + "models/north/_sqlbuild/_macros/fmt.py": ( + 'def fmt(value: str) -> str:\n """Label a value."""\n return f"\'north:{value}\'"\n' + ), + "constants/base_rate.sql": "CONSTANT (name base_rate, value 1);\n", + "models/north/_sqlbuild/_constants/north_rate.sql": "CONSTANT (name north_rate, value 10);\n", + "models/south/_sqlbuild/_constants/south_rate.sql": "CONSTANT (name south_rate, value 20);\n", + "models/shared/customers.sql": ( + "MODEL (description \"Customers\");\n\nSELECT 1 AS id, 2 AS amount, 'north' AS region\n" + ), + "models/shared/orders_source.sql": ( + 'MODEL (description "Orders source");\n\nSELECT 1 AS id, 3 AS amount\n' + ), + "hooks/sql/stamp.sql": ( + "HOOK (description \"Stamp a value\");\n\nSELECT @cents('2') AS stamped\n" + ), + "functions/sql/udf__cents_of.sql": ( + "FUNCTION (\n" + ' description "Convert a value to cents",\n' + " arguments (v INTEGER),\n" + " returns INTEGER,\n" + ");\n\n" + '@cents("v")\n' + ), + "audits/generic/positive_cents.sql": ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @cents("amount") < 0\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw__payments\n" + " description: Payments\n" + " expression: |\n" + " SELECT @cents('5') AS amount_cents, @wrap(@cents('6')) AS wrapped\n" + ), + "tests/unit/test_cents_macro.sql": ( + 'TEST (mode macro, name "converts_to_cents");\n\n' + "WITH\n" + "__macro_actual__ AS (\n" + ' SELECT @cents("3") AS cents\n' + "),\n" + "__macro_expected__ AS (\n" + " SELECT 300 AS cents\n" + ")\n" + "SELECT 1\n" + ), +} + +_REGION_MODEL_BODY: str = ( + "SELECT\n" + ' @cents("amount") AS amount_cents,\n' + ' @wrap(@cents("amount")) AS wrapped,\n' + ' @scale("amount") AS scaled,\n' + " @visible_constants() AS visible,\n" + " {label} AS label,\n" + " @region_filter() AS region_check,\n" + " @status_code() AS status_value,\n" + " @optional_label() AS optional_label,\n" + " {ref_tags},\n" + ' @logged("amount") AS logged_amount,\n' + " 'quoted @cents(1)' AS quoted -- @cents(2)\n" + 'FROM (@columns_of(__ref("customers"))) AS c\n' + "WHERE id IN @generated_join()\n" +) +_NORTH_REF_TAGS: str = '@ref_tag(__ref("orders_source")) AS first_tag' +_SOUTH_REF_TAGS: str = ( + '@ref_tag(__ref("customers")) AS first_tag, @ref_tag(__ref("orders_source")) AS second_tag' +) + + +def _region_model(*, description: str, hooks: str, ref_tags: str, label: str) -> str: + return ( + f'MODEL (\n description "{description}",\n{hooks}' + " audits [positive_cents ()],\n);\n\n" + + _REGION_MODEL_BODY.format(ref_tags=ref_tags, label=label) + ) + + +_NORTH_HOOKS: str = ( + " pre_hooks [\n" + " inline_sql(\"SELECT @cents('1') AS hook_value\"),\n" + ' sql("stamp"),\n' + " ],\n" +) +MACRO_BRIDGE_PROJECT_FILES.update( + { + "models/north/orders_north.sql": _region_model( + description="North orders", + hooks=_NORTH_HOOKS, + ref_tags=_NORTH_REF_TAGS, + label='@fmt("x")', + ), + "models/north/orders_north_copy.sql": _region_model( + description="North orders copy", + hooks=_NORTH_HOOKS, + ref_tags=_NORTH_REF_TAGS, + label='@fmt("x")', + ), + "models/south/orders_south.sql": _region_model( + description="South orders", hooks="", ref_tags=_SOUTH_REF_TAGS, label="'south'" + ), + } +) + + +def write_project(*, root: Path, files: dict[str, str]) -> Path: + """Write project files below `root` and return it.""" + + for relative_path, content in files.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + return root + + +def render_compile_inputs( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> CompileProjectInputs: + """Render the project's compile inputs through the frontier under one compiler engine.""" + + with monkeypatch.context() as patch: + patch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + adapter: DuckDbAdapter = DuckDbAdapter() + return compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + resolved_connection={}, + defer_model_sql_validation=True, + no_cache=True, + ), + ) + + +def comparable_capture(inputs: CompileProjectInputs) -> str: + """Render compile inputs as canonical JSON without the per-run identifier.""" + + return _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)) + + +def render_error( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[type[BaseException], str, type[object]]: + """Render the project under one engine and describe the error it raises.""" + + with pytest.raises(CompileInputError) as raised: + _ = render_compile_inputs(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) + return type(raised.value), str(raised.value), type(raised.value.__cause__) + + +def _wrap_m(value: object = "") -> str: + return f"m<{value}>" + + +def _wrap_n(value: object = "", label: object = "") -> str: + return f"n[{value}{label}]" + + +def _tag(relation: object) -> str: + return repr(str(relation).upper()) + + +def _cols(relation: object) -> str: + return f"SELECT * FROM {relation}" + + +def _generated() -> str: + return '(SELECT id FROM __ref("orders"))' + + +def _loaded_macro(*, name: str, function: Callable[..., object]) -> LoadedMacro: + path: Path = Path(f"macros/{name}.py") + return LoadedMacro( + name=name, file_path=path, relative_path=path, raw_source="", function=function + ) + + +DIFFERENTIAL_MACROS: dict[str, LoadedMacro] = { + name: _loaded_macro(name=name, function=function) + for name, function in ( + ("m", _wrap_m), + ("mm", _wrap_m), + ("n", _wrap_n), + ("tag", _tag), + ("cols", _cols), + ("gen", _generated), + ) +} +_DIFFERENTIAL_CONTEXT: MacroContext = MacroContext( + adapter_name="duckdb", sql_analysis_enabled=False, target_name=None +) + + +class ExpansionFailureCapture: + """Swallow and keep the compile input error a block raises.""" + + def __init__(self) -> None: + self.failure: BaseException | None = None + + def __enter__(self) -> ExpansionFailureCapture: + return self + + def __exit__(self, error_type: object, error: BaseException | None, traceback: object) -> bool: + self.failure = error + return isinstance(error, CompileInputError) + + +def random_macro_sql(*, rng: random.Random) -> str: + """Return SQL text mixing valid and malformed macro calls, quotes, comments and Unicode.""" + + return "".join(rng.choice(_DIFFERENTIAL_FRAGMENTS) for _ in range(rng.randint(0, 12))) + + +def expansion_outcome(*, sql: str, bridge: MacroBridge | None) -> tuple[object, ...]: + """Expand `sql` with or without the bridge; return the result facts or the error.""" + + token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) + capture: ExpansionFailureCapture = ExpansionFailureCapture() + rendered: list[tuple[object, ...]] = [()] + with capture: + result: MacroExpansionResult = expand_sql_macros_result( + sql=sql, + file_path=_DIFFERENTIAL_FILE, + loaded_macros=DIFFERENTIAL_MACROS, + macro_context=_DIFFERENTIAL_CONTEXT, + consumer=ResourceIdentity(ResourceKind.MODEL, "orders_summary"), + ) + rendered[0] = ( + result.sql, + result.spans, + result.dependencies, + result.usages, + result.argument_references, + ) + ACTIVE_MACRO_BRIDGE.reset(token) + return { + True: rendered[0], + False: (type(capture.failure).__name__, str(capture.failure)), + }[capture.failure is None] diff --git a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py new file mode 100644 index 0000000000..fa60262c11 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py @@ -0,0 +1,226 @@ +"""The native macro bridge renders compile inputs byte-identically to the Python engine.""" + +from __future__ import annotations + +import random +import sys +import unicodedata +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from tests.integration.src.sqlbuild.compiler.compile._test_types import ( + MacroBridgeFailureTestCase, + MacroBridgeMemoTestCase, + MacroBridgeParityTestCase, + MacroExpansionDifferentialTestCase, +) +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + MACRO_CALL_LOG_ENV_VAR, + comparable_capture, + expansion_outcome, + random_macro_sql, + render_compile_inputs, + render_error, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_FAILURE_BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": MACRO_BRIDGE_PROJECT_FILES["sqlbuild_project.toml"], + "macros/common.py": ( + "def cents(column: str) -> str:\n" + ' """Convert an amount to cents."""\n' + ' return f"{column} * 100"\n\n\n' + "def explode(column: str) -> str:\n" + ' """Fail while rendering."""\n' + ' raise ValueError(f"cannot render {column}")\n\n\n' + "def number(column: str) -> int:\n" + ' """Return a value that is not SQL."""\n' + " return 1\n" + ), + "models/a_first.sql": ( + 'MODEL (description "First");\n\nSELECT @cents("amount") AS amount_cents\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeParityTestCase( + description="nested, context, scoped, typed-reference, hook, test, audit, source " + "and function macros", + files=MACRO_BRIDGE_PROJECT_FILES, + expected_fragments=( + "'north:x'", + "amount * 11", + "amount * 21", + "'base_rate, north_rate'", + "'base_rate, south_rate'", + "'__SQLBUILD_RELATION_0__'", + "'__SQLBUILD_RELATION_1__'", + "P006", + "SELECT 2 * 100 AS stamped", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_heavy_project_when_rendering_with_each_engine_then_inputs_are_identical( + test_case: MacroBridgeParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + + python_capture: str = comparable_capture( + render_compile_inputs(project_dir=project_dir, engine="python", monkeypatch=monkeypatch) + ) + native_capture: str = comparable_capture( + render_compile_inputs(project_dir=project_dir, engine="native", monkeypatch=monkeypatch) + ) + + assert native_capture == python_capture + assert all(map(python_capture.__contains__, test_case.expected_fragments)) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeMemoTestCase( + description="one global call text in three models", + files=MACRO_BRIDGE_PROJECT_FILES, + expected_python_executions=3, + expected_native_executions=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_repeated_macro_calls_when_rendering_natively_then_each_call_class_runs_once( + test_case: MacroBridgeMemoTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + executions: dict[str, int] = {} + for engine in ("python", "native"): + log_path: Path = tmp_path / f"{engine}.log" + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(log_path)) + _ = render_compile_inputs(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) + executions[engine] = len(log_path.read_text(encoding="utf-8").splitlines()) + + assert executions == { + "python": test_case.expected_python_executions, + "native": test_case.expected_native_executions, + } + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeFailureTestCase( + description="macro raises in a later model", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\nSELECT @explode("amount") AS bogus\n' + ), + }, + expected_message_fragment="failed: cannot render amount", + ), + MacroBridgeFailureTestCase( + description="unknown macro after a successful call", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\n' + 'SELECT @cents("amount") AS a, @nope("amount") AS b\n' + ), + }, + expected_message_fragment="Unknown macro '@nope'", + ), + MacroBridgeFailureTestCase( + description="macro returns a value that is not SQL", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\nSELECT @number("amount") AS b\n' + ), + }, + expected_message_fragment="must return a SQL string", + ), + MacroBridgeFailureTestCase( + description="malformed call after a successful call", + files={ + **_FAILURE_BASE_FILES, + "models/b_second.sql": ( + 'MODEL (description "Second");\n\n' + 'SELECT @cents("amount") AS a, @cents x("amount") AS b\n' + ), + }, + expected_message_fragment="expected opening parenthesis", + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_render_when_rendering_natively_then_raises_the_python_error( + test_case: MacroBridgeFailureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) + + python_error: tuple[type[BaseException], str, type[object]] = render_error( + project_dir=project_dir, engine="python", monkeypatch=monkeypatch + ) + native_error: tuple[type[BaseException], str, type[object]] = render_error( + project_dir=project_dir, engine="native", monkeypatch=monkeypatch + ) + + assert native_error == python_error + assert test_case.expected_message_fragment in python_error[1] + + +@pytest.mark.parametrize( + "test_case", + [ + MacroExpansionDifferentialTestCase( + description="seeded fragments", + seed=540, + samples=3000, + expected_minimum_successes=600, + expected_mismatches=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_random_macro_sql_when_expanding_through_bridge_then_matches_python( + test_case: MacroExpansionDifferentialTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + texts: list[str] = [random_macro_sql(rng=rng) for _ in range(test_case.samples)] + bridge: MacroBridge = MacroBridge( + python_version=(sys.version_info[0], sys.version_info[1]), + unicode_version=unicodedata.unidata_version, + ) + + python_outcomes: list[tuple[object, ...]] = [ + expansion_outcome(sql=sql, bridge=None) for sql in texts + ] + bridged_outcomes: list[tuple[object, ...]] = [ + expansion_outcome(sql=sql, bridge=bridge) for sql in texts + ] + + assert ( + mismatches( + inputs=list[object](texts), + expected=list[object](python_outcomes), + actual=list[object](bridged_outcomes), + ) + == test_case.expected_mismatches + ) + assert sum(len(outcome) == 5 for outcome in python_outcomes) >= ( + test_case.expected_minimum_successes + ) + assert bridge.stats()[0] > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py new file mode 100644 index 0000000000..699b51074f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/_test_types.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MacroBridgeFailureTestCase: + """A render stage that fails with the macro bridge active.""" + + description: str + stage_with_bridge: Callable[[], str] + stage_without_bridge: Callable[[], str] + expected_error: type[Exception] + expected_bridged_runs: list[bool] + + +@dataclass(frozen=True) +class MacroBridgeSuccessTestCase: + """A render stage that succeeds with the macro bridge active.""" + + description: str + expected_result: str + expected_bridged_runs: list[bool] + + +@dataclass(frozen=True) +class MacroBridgeErrorContextTestCase: + """A stage whose Python error must not chain the bridge failure as its context.""" + + description: str + stage: Callable[[], str] + expected_context: BaseException | None diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py new file mode 100644 index 0000000000..9914dce8e2 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -0,0 +1,33 @@ +"""Render stages that record whether the macro bridge was active.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge + + +def rendered() -> str: + """Render successfully.""" + + return "rendered" + + +def failed() -> str: + """Fail to render.""" + + raise CompileInputError("render failed") + + +def recording_stage( + *, runs: list[bool], stages: dict[bool, Callable[[], str]] +) -> Callable[[], str]: + """Return a stage recording whether each run had the bridge, then running its variant.""" + + def stage() -> str: + bridged: bool = active_macro_bridge() is not None + runs.append(bridged) + return stages[bridged]() + + return stage diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py new file mode 100644 index 0000000000..d529893f7e --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeRenderMismatchError +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge +from tests.unit.src.sqlbuild.compiler.macro_bridge.main._test_types import ( + MacroBridgeErrorContextTestCase, + MacroBridgeFailureTestCase, + MacroBridgeSuccessTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge.main.helpers import ( + failed, + recording_stage, + rendered, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeSuccessTestCase( + description="success runs once with the bridge", + expected_result="rendered", + expected_bridged_runs=[True], + ), + ], + ids=lambda case: case.description, +) +def test_given_succeeding_stage_when_running_with_macro_bridge_then_runs_once_bridged( + test_case: MacroBridgeSuccessTestCase, +) -> None: + runs: list[bool] = [] + + result: str = run_with_macro_bridge( + stage=recording_stage(runs=runs, stages={True: rendered, False: failed}) + ) + + assert result == test_case.expected_result + assert runs == test_case.expected_bridged_runs + assert active_macro_bridge() is None + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeFailureTestCase( + description="failure re-runs Python and raises its error", + stage_with_bridge=failed, + stage_without_bridge=failed, + expected_error=CompileInputError, + expected_bridged_runs=[True, False], + ), + MacroBridgeFailureTestCase( + description="native-only failure is reported as a divergence", + stage_with_bridge=failed, + stage_without_bridge=rendered, + expected_error=NativeRenderMismatchError, + expected_bridged_runs=[True, False], + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_stage_when_running_with_macro_bridge_then_python_reruns( + test_case: MacroBridgeFailureTestCase, +) -> None: + runs: list[bool] = [] + stage: Callable[[], str] = recording_stage( + runs=runs, + stages={True: test_case.stage_with_bridge, False: test_case.stage_without_bridge}, + ) + + with pytest.raises(test_case.expected_error): + _ = run_with_macro_bridge(stage=stage) + + assert runs == test_case.expected_bridged_runs + assert active_macro_bridge() is None + + +@pytest.mark.parametrize( + "test_case", + [ + MacroBridgeErrorContextTestCase( + description="python error after a bridge failure", + stage=failed, + expected_context=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_error_when_rerun_after_bridge_failure_then_error_has_no_bridge_context( + test_case: MacroBridgeErrorContextTestCase, +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = run_with_macro_bridge(stage=test_case.stage) + + assert raised.value.__context__ is test_case.expected_context + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 069f2104cf287a0c0803b5735f1cf0d8592bf341 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:49:30 +0100 Subject: [PATCH 04/11] feat(native): add a shared native store and carry macro calls in it The sqlbuild-cache crate is the one store and fingerprinting scheme for native caches: a content digest, a project file fingerprint that skips the folders project reuse skips, and one checksummed, atomically replaced file per cache kind whose entries are immutable and dropped after 64 saves unused. A damaged, foreign or older file opens empty. The macro call memo can attach a store: calls of a persistent class are looked up by a digest of their class text and call text, and recorded results are stored. The Python extension exposes the digest, the project fingerprint and the memo's store methods. --- Cargo.lock | 41 ++++ Cargo.toml | 3 + crates/README.md | 6 +- crates/sqlbuild-cache/Cargo.toml | 20 ++ .../sqlbuild-cache/src/digest/_helpers/mod.rs | 1 + .../src/digest/_helpers/walk.rs | 72 +++++++ crates/sqlbuild-cache/src/digest/constants.rs | 32 +++ crates/sqlbuild-cache/src/digest/errors.rs | 25 +++ .../src/digest/main/content_digest.rs | 16 ++ .../digest/main/fingerprint_project_files.rs | 38 ++++ .../src/digest/main/hex_digest.rs | 8 + crates/sqlbuild-cache/src/digest/main/mod.rs | 3 + crates/sqlbuild-cache/src/digest/mod.rs | 8 + crates/sqlbuild-cache/src/digest/tests.rs | 8 + .../src/digest/tests/helpers.rs | 29 +++ .../src/digest/tests/test_content_digest.rs | 69 ++++++ .../tests/test_fingerprint_project_files.rs | 124 +++++++++++ .../src/digest/tests/test_types.rs | 29 +++ crates/sqlbuild-cache/src/digest/types.rs | 4 + crates/sqlbuild-cache/src/lib.rs | 6 + .../src/store/_helpers/frame.rs | 74 +++++++ .../sqlbuild-cache/src/store/_helpers/mod.rs | 1 + crates/sqlbuild-cache/src/store/constants.rs | 8 + crates/sqlbuild-cache/src/store/errors.rs | 5 + crates/sqlbuild-cache/src/store/main/mod.rs | 1 + .../src/store/main/open_native_store.rs | 46 ++++ crates/sqlbuild-cache/src/store/mod.rs | 8 + crates/sqlbuild-cache/src/store/models.rs | 175 ++++++++++++++++ crates/sqlbuild-cache/src/store/tests.rs | 6 + .../sqlbuild-cache/src/store/tests/helpers.rs | 52 +++++ .../src/store/tests/test_native_store.rs | 197 ++++++++++++++++++ .../src/store/tests/test_types.rs | 34 +++ crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/main/register.rs | 1 + crates/sqlbuild-python/src/lib.rs | 1 + .../src/macro_bridge/_helpers/memo.rs | 65 +++++- .../src/macro_bridge/constants.rs | 3 + .../src/native_store/_helpers/functions.rs | 34 +++ .../src/native_store/_helpers/mod.rs | 1 + .../src/native_store/main/mod.rs | 1 + .../src/native_store/main/register.rs | 12 ++ .../sqlbuild-python/src/native_store/mod.rs | 2 + crates/sqlbuild-render/Cargo.toml | 4 + .../src/macro_calls/_helpers/entry_codec.rs | 80 +++++++ .../src/macro_calls/_helpers/mod.rs | 1 + .../sqlbuild-render/src/macro_calls/models.rs | 81 ++++++- .../src/macro_calls/tests/helpers.rs | 47 +++++ .../macro_calls/tests/test_macro_call_memo.rs | 79 ++++++- .../src/macro_calls/tests/test_types.rs | 8 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 8 + 51 files changed, 1566 insertions(+), 13 deletions(-) create mode 100644 crates/sqlbuild-cache/Cargo.toml create mode 100644 crates/sqlbuild-cache/src/digest/_helpers/mod.rs create mode 100644 crates/sqlbuild-cache/src/digest/_helpers/walk.rs create mode 100644 crates/sqlbuild-cache/src/digest/constants.rs create mode 100644 crates/sqlbuild-cache/src/digest/errors.rs create mode 100644 crates/sqlbuild-cache/src/digest/main/content_digest.rs create mode 100644 crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs create mode 100644 crates/sqlbuild-cache/src/digest/main/hex_digest.rs create mode 100644 crates/sqlbuild-cache/src/digest/main/mod.rs create mode 100644 crates/sqlbuild-cache/src/digest/mod.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests/helpers.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests/test_types.rs create mode 100644 crates/sqlbuild-cache/src/digest/types.rs create mode 100644 crates/sqlbuild-cache/src/lib.rs create mode 100644 crates/sqlbuild-cache/src/store/_helpers/frame.rs create mode 100644 crates/sqlbuild-cache/src/store/_helpers/mod.rs create mode 100644 crates/sqlbuild-cache/src/store/constants.rs create mode 100644 crates/sqlbuild-cache/src/store/errors.rs create mode 100644 crates/sqlbuild-cache/src/store/main/mod.rs create mode 100644 crates/sqlbuild-cache/src/store/main/open_native_store.rs create mode 100644 crates/sqlbuild-cache/src/store/mod.rs create mode 100644 crates/sqlbuild-cache/src/store/models.rs create mode 100644 crates/sqlbuild-cache/src/store/tests.rs create mode 100644 crates/sqlbuild-cache/src/store/tests/helpers.rs create mode 100644 crates/sqlbuild-cache/src/store/tests/test_native_store.rs create mode 100644 crates/sqlbuild-cache/src/store/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/native_store/_helpers/functions.rs create mode 100644 crates/sqlbuild-python/src/native_store/_helpers/mod.rs create mode 100644 crates/sqlbuild-python/src/native_store/main/mod.rs create mode 100644 crates/sqlbuild-python/src/native_store/main/register.rs create mode 100644 crates/sqlbuild-python/src/native_store/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs diff --git a/Cargo.lock b/Cargo.lock index e9256fd552..6f292fc245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,6 +561,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "saphyr-parser" version = "0.1.0" @@ -662,6 +671,16 @@ dependencies = [ "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-cache" +version = "0.139.0" +dependencies = [ + "rayon", + "sha2", + "tempfile", + "walkdir", +] + [[package]] name = "sqlbuild-config" version = "0.139.0" @@ -701,6 +720,7 @@ dependencies = [ "pyo3", "serde_json", "sqlbuild-analysis", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", @@ -715,7 +735,9 @@ dependencies = [ name = "sqlbuild-render" version = "0.139.0" dependencies = [ + "sqlbuild-cache", "sqlbuild-core", + "tempfile", ] [[package]] @@ -962,6 +984,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "winapi" version = "0.3.9" @@ -978,6 +1010,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 255a55dcd3..8dd0df853d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/sqlbuild-core", + "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", "crates/sqlbuild-discovery", @@ -19,6 +20,7 @@ resolver = "2" [workspace.metadata.native-layers] order = [ "sqlbuild-core", + "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", "sqlbuild-discovery", @@ -72,6 +74,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-cache = { path = "crates/sqlbuild-cache" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } diff --git a/crates/README.md b/crates/README.md index 81d9613b9b..40d973f153 100644 --- a/crates/README.md +++ b/crates/README.md @@ -7,12 +7,13 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | -| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events. Anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -24,7 +25,8 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-render - -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache + -> sqlbuild-core ``` `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and diff --git a/crates/sqlbuild-cache/Cargo.toml b/crates/sqlbuild-cache/Cargo.toml new file mode 100644 index 0000000000..6ffbf21d5c --- /dev/null +++ b/crates/sqlbuild-cache/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sqlbuild-cache" +version.workspace = true +description = "The shared native store and fingerprinting scheme for SQLBuild's compiler caches." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +rayon.workspace = true +sha2.workspace = true +tempfile.workspace = true +walkdir.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-cache/src/digest/_helpers/mod.rs b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs new file mode 100644 index 0000000000..208de6bb67 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod walk; diff --git a/crates/sqlbuild-cache/src/digest/_helpers/walk.rs b/crates/sqlbuild-cache/src/digest/_helpers/walk.rs new file mode 100644 index 0000000000..af89bd61eb --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/_helpers/walk.rs @@ -0,0 +1,72 @@ +//! List the project files a fingerprint covers, in a stable order. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use walkdir::{DirEntry, WalkDir}; + +use crate::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use crate::digest::errors::FingerprintError; + +/// One covered file: its project-relative path, location and whether only presence counts. +pub(crate) struct CoveredFile { + pub(crate) relative_path: String, + pub(crate) path: PathBuf, + pub(crate) presence_only: bool, +} + +/// Every file below `root` outside the excluded folders and `excluded_files`, links followed. +pub(crate) fn covered_files( + root: &Path, + excluded_files: &HashSet, +) -> Result, FingerprintError> { + let mut files: Vec = Vec::new(); + let walk = WalkDir::new(root) + .follow_links(true) + .sort_by_file_name() + .into_iter() + .filter_entry(|entry| !excluded_directory(entry)); + for entry in walk { + let entry: DirEntry = entry.map_err(FingerprintError::Walk)?; + if !entry.file_type().is_file() { + continue; + } + let relative_path: String = relative_text(root, entry.path())?; + if excluded_files.contains(&relative_path) { + continue; + } + let presence_only: bool = PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .any(|suffix| relative_path.ends_with(suffix)); + files.push(CoveredFile { + relative_path, + path: entry.into_path(), + presence_only, + }); + } + Ok(files) +} + +fn excluded_directory(entry: &DirEntry) -> bool { + if !entry.file_type().is_dir() || entry.depth() == 0 { + return false; + } + let Some(name) = entry.file_name().to_str() else { + return false; + }; + EXCLUDED_DIRECTORIES.contains(&name) + || (entry.depth() == 1 && EXCLUDED_ROOT_DIRECTORIES.contains(&name)) +} + +fn relative_text(root: &Path, path: &Path) -> Result { + let relative: &Path = path + .strip_prefix(root) + .map_err(|_| FingerprintError::OutsideRoot(path.to_path_buf()))?; + let non_utf8 = || FingerprintError::NonUtf8Path(path.to_path_buf()); + let parts: Vec<&str> = relative + .components() + .map(|component| component.as_os_str().to_str().ok_or_else(non_utf8)) + .collect::, FingerprintError>>()?; + Ok(parts.join("/")) +} diff --git a/crates/sqlbuild-cache/src/digest/constants.rs b/crates/sqlbuild-cache/src/digest/constants.rs new file mode 100644 index 0000000000..d0a53f5c3a --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/constants.rs @@ -0,0 +1,32 @@ +//! What the project fingerprint walk skips; the same folders project reuse leaves out. + +/// Folders skipped only directly below the project root. +pub const EXCLUDED_ROOT_DIRECTORIES: &[&str] = &[ + "logs", + "target", + "venv", + ".cache", + ".fensu", + ".hg", + ".idea", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".sqlbuild", + ".svn", + ".tox", + ".venv", + ".vscode", +]; + +/// Folders skipped at any depth. +pub const EXCLUDED_DIRECTORIES: &[&str] = &[".git", "__pycache__"]; + +/// Files that count only by presence because every build rewrites them. +pub const PRESENCE_ONLY_FILE_SUFFIXES: &[&str] = &[".duckdb", ".duckdb.wal"]; + +/// Digest tag of a file whose bytes count. +pub const CONTENT_TAG: &[u8] = b"content"; +/// Digest tag of a file that counts only by presence. +pub const PRESENCE_TAG: &[u8] = b"presence"; diff --git a/crates/sqlbuild-cache/src/digest/errors.rs b/crates/sqlbuild-cache/src/digest/errors.rs new file mode 100644 index 0000000000..87a9b1757b --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/errors.rs @@ -0,0 +1,25 @@ +//! Why a project fingerprint could not cover every file. + +use std::fmt; + +/// A file the fingerprint must cover could not be listed or read. +#[derive(Debug)] +pub enum FingerprintError { + Walk(walkdir::Error), + NonUtf8Path(std::path::PathBuf), + OutsideRoot(std::path::PathBuf), + Read(std::path::PathBuf, std::io::Error), +} + +impl fmt::Display for FingerprintError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Walk(error) => write!(formatter, "cannot list project files: {error}"), + Self::NonUtf8Path(path) => write!(formatter, "path is not UTF-8: {}", path.display()), + Self::OutsideRoot(path) => { + write!(formatter, "path is outside the project: {}", path.display()) + } + Self::Read(path, error) => write!(formatter, "cannot read {}: {error}", path.display()), + } + } +} diff --git a/crates/sqlbuild-cache/src/digest/main/content_digest.rs b/crates/sqlbuild-cache/src/digest/main/content_digest.rs new file mode 100644 index 0000000000..d7076d9eba --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/content_digest.rs @@ -0,0 +1,16 @@ +//! The one digest every native cache key is built from. + +use sha2::{Digest, Sha256}; + +use crate::digest::types::ContentDigest; + +/// Digest `parts` with each part length-prefixed, so part boundaries cannot shift. +pub fn content_digest>(parts: &[T]) -> ContentDigest { + let mut hasher: Sha256 = Sha256::new(); + for part in parts { + let bytes: &[u8] = part.as_ref(); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + hasher.finalize().into() +} diff --git a/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs b/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs new file mode 100644 index 0000000000..9c544f71cf --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/fingerprint_project_files.rs @@ -0,0 +1,38 @@ +//! Fingerprint the content of a project's files, leaving out the files a cache key covers itself. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::path::Path; + +use crate::digest::_helpers::walk::{CoveredFile, covered_files}; +use crate::digest::constants::{CONTENT_TAG, PRESENCE_TAG}; +use crate::digest::errors::FingerprintError; +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; + +/// Digest every covered file's project-relative path and bytes; fails if any file is unreadable. +pub fn fingerprint_project_files( + root: &Path, + excluded_files: &HashSet, +) -> Result { + let files: Vec = covered_files(root, excluded_files)?; + let digests: Vec = files + .par_iter() + .map(file_digest) + .collect::, FingerprintError>>()?; + let mut hasher: Sha256 = Sha256::new(); + for (file, digest) in files.iter().zip(digests) { + hasher.update(content_digest(&[file.relative_path.as_bytes(), &digest])); + } + Ok(hasher.finalize().into()) +} + +fn file_digest(file: &CoveredFile) -> Result { + if file.presence_only { + return Ok(content_digest(&[PRESENCE_TAG])); + } + let bytes: Vec = std::fs::read(&file.path) + .map_err(|error| FingerprintError::Read(file.path.clone(), error))?; + Ok(content_digest(&[CONTENT_TAG, &bytes])) +} diff --git a/crates/sqlbuild-cache/src/digest/main/hex_digest.rs b/crates/sqlbuild-cache/src/digest/main/hex_digest.rs new file mode 100644 index 0000000000..1ef1c12e40 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/hex_digest.rs @@ -0,0 +1,8 @@ +//! Text form of a content digest. + +use crate::digest::types::ContentDigest; + +/// Lowercase hexadecimal text of a digest. +pub fn hex_digest(digest: &ContentDigest) -> String { + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs new file mode 100644 index 0000000000..d9c3e7727b --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -0,0 +1,3 @@ +pub mod content_digest; +pub mod fingerprint_project_files; +pub mod hex_digest; diff --git a/crates/sqlbuild-cache/src/digest/mod.rs b/crates/sqlbuild-cache/src/digest/mod.rs new file mode 100644 index 0000000000..49b58ce941 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/digest/tests.rs b/crates/sqlbuild-cache/src/digest/tests.rs new file mode 100644 index 0000000000..712cba961e --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_content_digest.rs"] +mod content_digest; +#[path = "tests/test_fingerprint_project_files.rs"] +mod fingerprint_project_files; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/digest/tests/helpers.rs b/crates/sqlbuild-cache/src/digest/tests/helpers.rs new file mode 100644 index 0000000000..179519088f --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/helpers.rs @@ -0,0 +1,29 @@ +use std::collections::HashSet; +use std::path::Path; + +/// The model file the fingerprint test project leaves to the cache key. +pub(crate) const EXCLUDED_MODEL: &str = "models/orders.sql"; + +/// Write a small project: config, a macro, a model and files in skipped folders. +pub(crate) fn write_project(root: &Path) { + for (relative_path, contents) in [ + ("sqlbuild_project.toml", "name = \"orders\"\n"), + ("macros/cents.py", "def cents(amount):\n return amount\n"), + (EXCLUDED_MODEL, "SELECT 1 AS id\n"), + ("target/cache/store.bin", "old"), + ("macros/__pycache__/cents.pyc", "bytecode"), + ("warehouse.duckdb", "pages"), + ] { + write_file(root, relative_path, contents); + } +} + +pub(crate) fn write_file(root: &Path, relative_path: &str, contents: &str) { + let path = root.join(relative_path); + std::fs::create_dir_all(path.parent().expect("parent")).expect("parent folder"); + std::fs::write(path, contents).expect("project file"); +} + +pub(crate) fn excluded_files() -> HashSet { + HashSet::from([EXCLUDED_MODEL.to_owned()]) +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs b/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs new file mode 100644 index 0000000000..e55864a887 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_content_digest.rs @@ -0,0 +1,69 @@ +use crate::digest::main::content_digest::content_digest; +use crate::digest::main::hex_digest::hex_digest; +use crate::digest::tests::test_types::{ContentDigestTestCase, HexDigestTestCase}; + +#[test] +fn given_parts_when_digesting_then_only_identical_parts_match() { + let test_cases = [ + ContentDigestTestCase { + description: "identical parts", + left: &["cents", "@cents(amount)"], + right: &["cents", "@cents(amount)"], + expected_equal: true, + }, + ContentDigestTestCase { + description: "a boundary moved between parts", + left: &["cents", "@cents(amount)"], + right: &["cents@", "cents(amount)"], + expected_equal: false, + }, + ContentDigestTestCase { + description: "an empty part added", + left: &["cents"], + right: &["cents", ""], + expected_equal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + content_digest(test_case.left) == content_digest(test_case.right), + test_case.expected_equal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_digest_when_formatting_then_text_is_lowercase_hexadecimal() { + let test_cases = [ + HexDigestTestCase { + description: "one part", + parts: &["orders"], + expected_length: 64, + }, + HexDigestTestCase { + description: "no parts", + parts: &[], + expected_length: 64, + }, + ]; + + for test_case in test_cases { + let text = hex_digest(&content_digest(test_case.parts)); + + assert_eq!( + text.len(), + test_case.expected_length, + "{}", + test_case.description + ); + assert!( + text.chars() + .all(|character| matches!(character, '0'..='9' | 'a'..='f')), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs b/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs new file mode 100644 index 0000000000..c6d80d70fd --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_fingerprint_project_files.rs @@ -0,0 +1,124 @@ +use std::path::Path; + +use crate::digest::main::fingerprint_project_files::fingerprint_project_files; +use crate::digest::tests::helpers::{EXCLUDED_MODEL, excluded_files, write_file, write_project}; +use crate::digest::tests::test_types::{ + FingerprintFailureTestCase, FingerprintProjectFilesTestCase, +}; + +#[test] +fn given_project_edit_when_fingerprinting_then_only_covered_content_changes_it() { + let test_cases = [ + FingerprintProjectFilesTestCase { + description: "no edit", + edit: |_| {}, + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "macro body edit", + edit: |root| { + write_file( + root, + "macros/cents.py", + "def cents(amount):\n return 1\n", + ) + }, + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "helper module added", + edit: |root| write_file(root, "macros/_rounding.py", "SCALE = 2\n"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "macro file removed", + edit: |root| std::fs::remove_file(root.join("macros/cents.py")).expect("removed"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "file renamed with the same content", + edit: |root| { + std::fs::rename(root.join("macros/cents.py"), root.join("macros/money.py")) + .expect("renamed"); + }, + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "excluded model edit", + edit: |root| write_file(root, EXCLUDED_MODEL, "SELECT 2 AS id\n"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "new model is covered until the caller excludes it", + edit: |root| write_file(root, "models/customers.sql", "SELECT 1 AS id\n"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "root target folder edit", + edit: |root| write_file(root, "target/cache/store.bin", "new"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "nested target folder edit", + edit: |root| write_file(root, "macros/target/notes.txt", "covered"), + expected_changed: true, + }, + FingerprintProjectFilesTestCase { + description: "bytecode cache edit", + edit: |root| write_file(root, "macros/__pycache__/cents.pyc", "other"), + expected_changed: false, + }, + FingerprintProjectFilesTestCase { + description: "database pages rewritten", + edit: |root| write_file(root, "warehouse.duckdb", "other pages"), + expected_changed: false, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_project(project.path()); + let before = fingerprint_project_files(project.path(), &excluded_files()); + (test_case.edit)(project.path()); + let after = fingerprint_project_files(project.path(), &excluded_files()); + + assert!(before.is_ok(), "{}", test_case.description); + assert_eq!( + before.ok() != after.ok(), + test_case.expected_changed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unlistable_root_when_fingerprinting_then_it_fails() { + let test_cases = [ + FingerprintFailureTestCase { + description: "existing project", + root: Path::to_path_buf, + expected_failure: false, + }, + FingerprintFailureTestCase { + description: "missing project", + root: |root| root.join("missing"), + expected_failure: true, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_project(project.path()); + + let fingerprint = + fingerprint_project_files(&(test_case.root)(project.path()), &excluded_files()); + + assert_eq!( + fingerprint.is_err(), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_types.rs b/crates/sqlbuild-cache/src/digest/tests/test_types.rs new file mode 100644 index 0000000000..f6891edbc5 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_types.rs @@ -0,0 +1,29 @@ +use std::path::{Path, PathBuf}; + +pub(crate) struct ContentDigestTestCase { + pub(crate) description: &'static str, + pub(crate) left: &'static [&'static str], + pub(crate) right: &'static [&'static str], + pub(crate) expected_equal: bool, +} + +pub(crate) struct HexDigestTestCase { + pub(crate) description: &'static str, + pub(crate) parts: &'static [&'static str], + pub(crate) expected_length: usize, +} + +/// An edit applied to a fresh copy of the fingerprint test project. +pub(crate) type ProjectEdit = fn(&Path); + +pub(crate) struct FingerprintProjectFilesTestCase { + pub(crate) description: &'static str, + pub(crate) edit: ProjectEdit, + pub(crate) expected_changed: bool, +} + +pub(crate) struct FingerprintFailureTestCase { + pub(crate) description: &'static str, + pub(crate) root: fn(&Path) -> PathBuf, + pub(crate) expected_failure: bool, +} diff --git a/crates/sqlbuild-cache/src/digest/types.rs b/crates/sqlbuild-cache/src/digest/types.rs new file mode 100644 index 0000000000..bab2f3dc57 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/types.rs @@ -0,0 +1,4 @@ +//! Digest values shared by every native cache. + +/// A SHA-256 content digest. +pub type ContentDigest = [u8; 32]; diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs new file mode 100644 index 0000000000..c1e3b017ca --- /dev/null +++ b/crates/sqlbuild-cache/src/lib.rs @@ -0,0 +1,6 @@ +//! The shared native cache store and the content fingerprints its keys are built from. + +#![forbid(unsafe_code)] + +pub mod digest; +pub mod store; diff --git a/crates/sqlbuild-cache/src/store/_helpers/frame.rs b/crates/sqlbuild-cache/src/store/_helpers/frame.rs new file mode 100644 index 0000000000..91e8028df9 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/_helpers/frame.rs @@ -0,0 +1,74 @@ +//! Encode and verify whole store files. + +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; +use crate::store::constants::{STORE_FORMAT_VERSION, STORE_MAGIC}; +use crate::store::errors::StoreDecodeError; +use crate::store::models::{RecordReader, RecordWriter}; + +/// The decoded contents of one store file. +pub(crate) struct StoreFile { + pub(crate) kind: String, + pub(crate) environment: String, + pub(crate) generation: u64, + pub(crate) metadata: Vec, + pub(crate) entries: Vec<(ContentDigest, u64, Vec)>, +} + +/// Magic, format version, a digest of the body, then the body. +pub(crate) fn encode_store(file: &StoreFile) -> Vec { + let mut body: RecordWriter = RecordWriter::default(); + body.put_str(&file.kind); + body.put_str(&file.environment); + body.put_u64(file.generation); + body.put_bytes(&file.metadata); + body.put_u64(file.entries.len() as u64); + for (key, last_used, value) in &file.entries { + body.put_bytes(key); + body.put_u64(*last_used); + body.put_bytes(value); + } + let body: Vec = body.into_bytes(); + let mut bytes: Vec = Vec::with_capacity(body.len() + 44); + bytes.extend_from_slice(STORE_MAGIC); + bytes.extend_from_slice(&STORE_FORMAT_VERSION.to_le_bytes()); + bytes.extend_from_slice(&content_digest(&[&body])); + bytes.extend_from_slice(&body); + bytes +} + +/// The file's contents; another format, a damaged body or trailing bytes are decode errors. +pub(crate) fn decode_store(bytes: &[u8]) -> Result { + let rest: &[u8] = bytes + .strip_prefix(STORE_MAGIC.as_slice()) + .ok_or(StoreDecodeError)?; + let (version, rest) = rest.split_first_chunk::<4>().ok_or(StoreDecodeError)?; + if u32::from_le_bytes(*version) != STORE_FORMAT_VERSION { + return Err(StoreDecodeError); + } + let (checksum, body) = rest.split_first_chunk::<32>().ok_or(StoreDecodeError)?; + if content_digest(&[body]) != *checksum { + return Err(StoreDecodeError); + } + let mut reader: RecordReader<'_> = RecordReader::new(body); + let kind: String = reader.string()?; + let environment: String = reader.string()?; + let generation: u64 = reader.u64()?; + let metadata: Vec = reader.bytes()?.to_vec(); + let count: u64 = reader.u64()?; + let mut entries: Vec<(ContentDigest, u64, Vec)> = Vec::new(); + for _ in 0..count { + let key: ContentDigest = reader.bytes()?.try_into().map_err(|_| StoreDecodeError)?; + let last_used: u64 = reader.u64()?; + let value: Vec = reader.bytes()?.to_vec(); + entries.push((key, last_used, value)); + } + reader.finish()?; + Ok(StoreFile { + kind, + environment, + generation, + metadata, + entries, + }) +} diff --git a/crates/sqlbuild-cache/src/store/_helpers/mod.rs b/crates/sqlbuild-cache/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..70756e496c --- /dev/null +++ b/crates/sqlbuild-cache/src/store/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod frame; diff --git a/crates/sqlbuild-cache/src/store/constants.rs b/crates/sqlbuild-cache/src/store/constants.rs new file mode 100644 index 0000000000..ffda20c06d --- /dev/null +++ b/crates/sqlbuild-cache/src/store/constants.rs @@ -0,0 +1,8 @@ +//! The native store file format. + +/// The first bytes of every store file. +pub const STORE_MAGIC: &[u8; 8] = b"SQBSTORE"; +/// Bumped whenever the file layout changes; older files are discarded whole. +pub const STORE_FORMAT_VERSION: u32 = 1; +/// Saves an unused entry survives while the store's environment is unchanged. +pub const RETAINED_GENERATIONS: u64 = 64; diff --git a/crates/sqlbuild-cache/src/store/errors.rs b/crates/sqlbuild-cache/src/store/errors.rs new file mode 100644 index 0000000000..c055f393e3 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/errors.rs @@ -0,0 +1,5 @@ +//! Why stored bytes could not be read back. + +/// Stored bytes end early, carry unknown fields or were written by another format. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreDecodeError; diff --git a/crates/sqlbuild-cache/src/store/main/mod.rs b/crates/sqlbuild-cache/src/store/main/mod.rs new file mode 100644 index 0000000000..4960dcf371 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/main/mod.rs @@ -0,0 +1 @@ +pub mod open_native_store; diff --git a/crates/sqlbuild-cache/src/store/main/open_native_store.rs b/crates/sqlbuild-cache/src/store/main/open_native_store.rs new file mode 100644 index 0000000000..5abbecd712 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/main/open_native_store.rs @@ -0,0 +1,46 @@ +//! Open one cache kind's store file. + +use std::collections::HashMap; +use std::path::Path; + +use crate::store::_helpers::frame::{StoreFile, decode_store}; +use crate::store::errors::StoreDecodeError; +use crate::store::models::{NativeStore, StoredValue}; + +/// Load `kind` entries for exactly `environment`; missing, damaged or foreign files open empty. +pub fn open_native_store( + path: &Path, + kind: &str, + environment: &str, +) -> std::io::Result { + let mut store: NativeStore = NativeStore { + kind: kind.to_owned(), + environment: environment.to_owned(), + ..NativeStore::default() + }; + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(store), + Err(error) => return Err(error), + }; + let file: StoreFile = match decode_store(&bytes) { + Ok(file) if file.kind == kind && file.environment == environment => file, + Ok(_) | Err(StoreDecodeError) => return Ok(store), + }; + store.generation = file.generation; + store.metadata = file.metadata; + store.loaded = file.entries.len(); + store.entries = file + .entries + .into_iter() + .map(|(key, last_used, value)| { + let stored: StoredValue = StoredValue { + value, + last_used, + used: false, + }; + (key, stored) + }) + .collect::>(); + Ok(store) +} diff --git a/crates/sqlbuild-cache/src/store/mod.rs b/crates/sqlbuild-cache/src/store/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-cache/src/store/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/store/models.rs b/crates/sqlbuild-cache/src/store/models.rs new file mode 100644 index 0000000000..0b662e9704 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/models.rs @@ -0,0 +1,175 @@ +//! The shared native store: one file per cache kind, keyed by content digests. + +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use crate::digest::types::ContentDigest; +use crate::store::_helpers::frame::{StoreFile, encode_store}; +use crate::store::constants::RETAINED_GENERATIONS; +use crate::store::errors::StoreDecodeError; + +/// Length-prefixed fields of one stored value. +#[derive(Debug, Default)] +pub struct RecordWriter { + bytes: Vec, +} + +impl RecordWriter { + pub fn put_u64(&mut self, value: u64) { + self.bytes.extend_from_slice(&value.to_le_bytes()); + } + + pub fn put_bytes(&mut self, value: &[u8]) { + self.put_u64(value.len() as u64); + self.bytes.extend_from_slice(value); + } + + pub fn put_str(&mut self, value: &str) { + self.put_bytes(value.as_bytes()); + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// Reads the fields a [`RecordWriter`] wrote; reading past the end is a decode error. +#[derive(Debug)] +pub struct RecordReader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> RecordReader<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + pub fn u64(&mut self) -> Result { + let raw: [u8; 8] = self.take(8)?.try_into().map_err(|_| StoreDecodeError)?; + Ok(u64::from_le_bytes(raw)) + } + + pub fn bytes(&mut self) -> Result<&'a [u8], StoreDecodeError> { + let length: usize = usize::try_from(self.u64()?).map_err(|_| StoreDecodeError)?; + self.take(length) + } + + pub fn string(&mut self) -> Result { + String::from_utf8(self.bytes()?.to_vec()).map_err(|_| StoreDecodeError) + } + + pub fn finish(&self) -> Result<(), StoreDecodeError> { + if self.position == self.bytes.len() { + Ok(()) + } else { + Err(StoreDecodeError) + } + } + + fn take(&mut self, length: usize) -> Result<&'a [u8], StoreDecodeError> { + let end: usize = self.position.checked_add(length).ok_or(StoreDecodeError)?; + let taken: &'a [u8] = self.bytes.get(self.position..end).ok_or(StoreDecodeError)?; + self.position = end; + Ok(taken) + } +} + +/// One stored value and the save generation that last used it. +#[derive(Clone, Debug)] +pub(crate) struct StoredValue { + pub(crate) value: Vec, + pub(crate) last_used: u64, + pub(crate) used: bool, +} + +/// One cache kind's entries for one environment; entries are immutable once stored. +#[derive(Debug, Default)] +pub struct NativeStore { + pub(crate) kind: String, + pub(crate) environment: String, + pub(crate) generation: u64, + pub(crate) metadata: Vec, + pub(crate) entries: HashMap, + pub(crate) loaded: usize, + pub(crate) changed: bool, +} + +impl NativeStore { + /// The caller-owned validation data saved with the loaded entries; empty when none loaded. + pub fn metadata(&self) -> &[u8] { + &self.metadata + } + + /// How many entries the store opened with. + pub fn loaded_entries(&self) -> usize { + self.loaded + } + + /// Drop every loaded entry, for a caller whose metadata check failed. + pub fn discard(&mut self) { + self.entries.clear(); + self.metadata.clear(); + self.loaded = 0; + self.generation = 0; + } + + /// The value stored under `key`, which the next save keeps. + pub fn get(&mut self, key: &ContentDigest) -> Option<&[u8]> { + let stored: &mut StoredValue = self.entries.get_mut(key)?; + stored.used = true; + Some(&stored.value) + } + + /// Store a value computed in this process. + pub fn put(&mut self, key: ContentDigest, value: Vec) { + let stored: StoredValue = StoredValue { + value, + last_used: self.generation, + used: true, + }; + let _ = self.entries.insert(key, stored); + self.changed = true; + } + + /// Whether this process stored a value the file does not hold yet. + pub fn needs_save(&self) -> bool { + self.changed + } + + /// Atomically replace the file with used and recently used entries; returns entries written. + pub fn save(&self, path: &Path, metadata: &[u8]) -> std::io::Result { + let generation: u64 = self.generation + 1; + let mut entries: Vec<(ContentDigest, u64, Vec)> = self + .entries + .iter() + .filter(|(_, stored)| { + stored.used || generation - stored.last_used <= RETAINED_GENERATIONS + }) + .map(|(key, stored)| { + let last_used: u64 = if stored.used { + generation + } else { + stored.last_used + }; + (*key, last_used, stored.value.clone()) + }) + .collect(); + entries.sort_unstable_by_key(|(key, _, _)| *key); + let written: usize = entries.len(); + let bytes: Vec = encode_store(&StoreFile { + kind: self.kind.clone(), + environment: self.environment.clone(), + generation, + metadata: metadata.to_vec(), + entries, + }); + let directory: &Path = path.parent().unwrap_or_else(|| Path::new(".")); + std::fs::create_dir_all(directory)?; + let mut file: tempfile::NamedTempFile = tempfile::NamedTempFile::new_in(directory)?; + file.write_all(&bytes)?; + let _ = file.persist(path).map_err(|error| error.error)?; + Ok(written) + } +} diff --git a/crates/sqlbuild-cache/src/store/tests.rs b/crates/sqlbuild-cache/src/store/tests.rs new file mode 100644 index 0000000000..7b76c040a2 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_native_store.rs"] +mod native_store; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/store/tests/helpers.rs b/crates/sqlbuild-cache/src/store/tests/helpers.rs new file mode 100644 index 0000000000..044e993772 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/helpers.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; +use crate::store::errors::StoreDecodeError; +use crate::store::main::open_native_store::open_native_store; +use crate::store::models::{NativeStore, RecordReader, RecordWriter}; + +pub(crate) const KIND: &str = "macro-calls"; +pub(crate) const ENVIRONMENT: &str = "environment-a"; +pub(crate) const METADATA: &[u8] = b"module stamps"; +pub(crate) const VALUE: &[u8] = b"amount * 100"; + +pub(crate) fn key(text: &str) -> ContentDigest { + content_digest(&[text]) +} + +pub(crate) fn open(path: &Path) -> NativeStore { + open_native_store(path, KIND, ENVIRONMENT).expect("opened store") +} + +/// Save a store holding `VALUE` under the `cents` key. +pub(crate) fn saved_store(path: &Path) { + let mut store = open(path); + store.put(key("cents"), VALUE.to_vec()); + assert_eq!(store.save(path, METADATA).expect("saved"), 1); +} + +/// Flip one byte near the end of the file. +pub(crate) fn damage(path: &Path) { + let mut bytes = std::fs::read(path).expect("store bytes"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(path, bytes).expect("damaged store"); +} + +/// A record holding a string and a number. +pub(crate) fn record() -> Vec { + let mut writer = RecordWriter::default(); + writer.put_str("café"); + writer.put_u64(7); + writer.into_bytes() +} + +/// Read a string and a number, then require the end of the record. +pub(crate) fn read_record(bytes: &[u8]) -> Result<(String, u64), StoreDecodeError> { + let mut reader = RecordReader::new(bytes); + let text = reader.string()?; + let number = reader.u64()?; + reader.finish()?; + Ok((text, number)) +} diff --git a/crates/sqlbuild-cache/src/store/tests/test_native_store.rs b/crates/sqlbuild-cache/src/store/tests/test_native_store.rs new file mode 100644 index 0000000000..bd6128a871 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/test_native_store.rs @@ -0,0 +1,197 @@ +use crate::store::errors::StoreDecodeError; +use crate::store::main::open_native_store::open_native_store; +use crate::store::tests::helpers::{ + ENVIRONMENT, KIND, METADATA, VALUE, damage, key, open, read_record, record, saved_store, +}; +use crate::store::tests::test_types::{ + NativeStoreDiscardTestCase, NativeStoreOpenTestCase, NativeStoreRetentionTestCase, + RecordReaderTestCase, +}; + +#[test] +fn given_saved_store_when_opening_then_only_the_same_kind_and_environment_load() { + let test_cases = [ + NativeStoreOpenTestCase { + description: "same kind and environment", + kind: KIND, + environment: ENVIRONMENT, + edit: |_| {}, + expected_value: Some(VALUE), + expected_metadata: METADATA, + }, + NativeStoreOpenTestCase { + description: "another environment", + kind: KIND, + environment: "environment-b", + edit: |_| {}, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "another kind", + kind: "inventory", + environment: ENVIRONMENT, + edit: |_| {}, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "damaged file", + kind: KIND, + environment: ENVIRONMENT, + edit: damage, + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "truncated file", + kind: KIND, + environment: ENVIRONMENT, + edit: |path| std::fs::write(path, b"SQBSTORE").expect("truncated"), + expected_value: None, + expected_metadata: b"", + }, + NativeStoreOpenTestCase { + description: "missing file", + kind: KIND, + environment: ENVIRONMENT, + edit: |path| std::fs::remove_file(path).expect("removed"), + expected_value: None, + expected_metadata: b"", + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("cache/macro-calls.bin"); + saved_store(&path); + (test_case.edit)(&path); + + let mut store = + open_native_store(&path, test_case.kind, test_case.environment).expect("opened store"); + + assert_eq!( + store.get(&key("cents")).map(<[u8]>::to_vec), + test_case.expected_value.map(<[u8]>::to_vec), + "{}", + test_case.description + ); + assert_eq!( + store.metadata(), + test_case.expected_metadata, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_entry_unused_for_many_saves_when_saving_then_it_is_dropped_after_the_limit() { + let test_cases = [ + NativeStoreRetentionTestCase { + description: "unused for the retained number of saves", + unused_saves: 64, + expected_retained: true, + }, + NativeStoreRetentionTestCase { + description: "unused for one save more", + unused_saves: 65, + expected_retained: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + saved_store(&path); + for _ in 0..test_case.unused_saves { + let mut store = open(&path); + store.put(key("fresh"), VALUE.to_vec()); + let _ = store.save(&path, METADATA).expect("saved"); + } + + let mut store = open(&path); + + assert_eq!( + store.get(&key("cents")).is_some(), + test_case.expected_retained, + "{}", + test_case.description + ); + assert!( + store.get(&key("fresh")).is_some(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_loaded_store_when_saving_new_entries_then_only_discarded_entries_are_lost() { + let test_cases = [ + NativeStoreDiscardTestCase { + description: "kept", + prepare: |_| {}, + expected_old_entry: true, + }, + NativeStoreDiscardTestCase { + description: "discarded", + prepare: |store| store.discard(), + expected_old_entry: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + saved_store(&path); + let mut store = open(&path); + (test_case.prepare)(&mut store); + store.put(key("fresh"), VALUE.to_vec()); + let _ = store.save(&path, b"").expect("saved"); + + let mut reopened = open(&path); + + assert_eq!( + reopened.get(&key("cents")).is_some(), + test_case.expected_old_entry, + "{}", + test_case.description + ); + assert!( + reopened.get(&key("fresh")).is_some(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_record_bytes_when_reading_then_only_complete_records_decode() { + let test_cases = [ + RecordReaderTestCase { + description: "complete record", + bytes: record, + expected_record: Ok(("café".to_owned(), 7)), + }, + RecordReaderTestCase { + description: "truncated record", + bytes: || record()[..10].to_vec(), + expected_record: Err(StoreDecodeError), + }, + RecordReaderTestCase { + description: "trailing byte", + bytes: || [record(), vec![0]].concat(), + expected_record: Err(StoreDecodeError), + }, + ]; + + for test_case in test_cases { + assert_eq!( + read_record(&(test_case.bytes)()), + test_case.expected_record, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/store/tests/test_types.rs b/crates/sqlbuild-cache/src/store/tests/test_types.rs new file mode 100644 index 0000000000..01b77fa4c0 --- /dev/null +++ b/crates/sqlbuild-cache/src/store/tests/test_types.rs @@ -0,0 +1,34 @@ +use std::path::Path; + +use crate::store::errors::StoreDecodeError; +use crate::store::models::NativeStore; + +/// A change applied to a saved store file before it is opened again. +pub(crate) type StoreFileEdit = fn(&Path); + +pub(crate) struct NativeStoreOpenTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) environment: &'static str, + pub(crate) edit: StoreFileEdit, + pub(crate) expected_value: Option<&'static [u8]>, + pub(crate) expected_metadata: &'static [u8], +} + +pub(crate) struct NativeStoreRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) unused_saves: u64, + pub(crate) expected_retained: bool, +} + +pub(crate) struct NativeStoreDiscardTestCase { + pub(crate) description: &'static str, + pub(crate) prepare: fn(&mut NativeStore), + pub(crate) expected_old_entry: bool, +} + +pub(crate) struct RecordReaderTestCase { + pub(crate) description: &'static str, + pub(crate) bytes: fn() -> Vec, + pub(crate) expected_record: Result<(String, u64), StoreDecodeError>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 55aa620ffa..840752e5fa 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -21,6 +21,7 @@ mimalloc.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 6b882ff58b..3afd2c87d1 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -30,6 +30,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { scopes::registration::register(module)?; model_config::registration::register(module)?; crate::macro_bridge::main::register::register(module)?; + crate::native_store::main::register::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/lib.rs b/crates/sqlbuild-python/src/lib.rs index ba7c854c34..85ec55e0cc 100644 --- a/crates/sqlbuild-python/src/lib.rs +++ b/crates/sqlbuild-python/src/lib.rs @@ -2,3 +2,4 @@ pub mod bindings; mod macro_bridge; +mod native_store; diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs index ba5776e875..edc20a5d00 100644 --- a/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs +++ b/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs @@ -1,8 +1,15 @@ //! Python methods of the in-compile macro call memo. -use pyo3::exceptions::PyValueError; +use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::prelude::Python; use pyo3::{PyResult, pymethods}; +use sqlbuild_cache::store::main::open_native_store::open_native_store; +use sqlbuild_cache::store::models::NativeStore; use sqlbuild_render::macro_calls::models::MacroCallEntry; +use std::path::PathBuf; + +use crate::bindings::types::CompilerDetach; +use crate::macro_bridge::constants::MACRO_CALL_STORE_KIND; use crate::macro_bridge::_helpers::rows::{event_from_row, event_row}; use crate::macro_bridge::models::MacroCallMemo; @@ -57,4 +64,60 @@ impl MacroCallMemo { fn stats(&self) -> PyResult<(usize, usize, usize)> { Ok(self.inner.lock().map_err(poisoned)?.stats()) } + + /// Open the store at `path` for `environment`; returns its saved metadata and entry count. + fn attach_store( + &self, + py: Python<'_>, + path: PathBuf, + environment: &str, + ) -> PyResult<(Vec, usize)> { + let store: NativeStore = py + .compiler_detach(|| Ok(open_native_store(&path, MACRO_CALL_STORE_KIND, environment))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string()))?; + let opened = (store.metadata().to_vec(), store.loaded_entries()); + self.inner.lock().map_err(poisoned)?.attach_store(store); + Ok(opened) + } + + /// Drop the attached store's loaded entries after its metadata failed validation. + fn discard_store(&self) -> PyResult<()> { + if let Some(store) = self.inner.lock().map_err(poisoned)?.store_mut() { + store.discard(); + } + Ok(()) + } + + /// Store results of `class_id` under `class_text` and everything the store key covers. + fn set_persistent_class(&self, class_id: u64, class_text: &str) -> PyResult<()> { + self.inner + .lock() + .map_err(poisoned)? + .set_persistent_class(class_id, class_text); + Ok(()) + } + + /// Calls found in the store and calls recorded into it. + fn store_stats(&self) -> PyResult<(usize, usize)> { + Ok(self.inner.lock().map_err(poisoned)?.store_stats()) + } + + /// Atomically save the store when this compile recorded into it; returns entries written. + fn save_store( + &self, + py: Python<'_>, + path: PathBuf, + metadata: &[u8], + ) -> PyResult> { + let mut memo = self.inner.lock().map_err(poisoned)?; + let Some(store) = memo.store_mut().filter(|store| store.needs_save()) else { + return Ok(None); + }; + let store: &NativeStore = store; + py.compiler_detach(|| Ok(store.save(&path, metadata))) + .map_err(PyRuntimeError::new_err)? + .map(Some) + .map_err(|error| PyOSError::new_err(error.to_string())) + } } diff --git a/crates/sqlbuild-python/src/macro_bridge/constants.rs b/crates/sqlbuild-python/src/macro_bridge/constants.rs index 13ba6f0ade..7baaec13a2 100644 --- a/crates/sqlbuild-python/src/macro_bridge/constants.rs +++ b/crates/sqlbuild-python/src/macro_bridge/constants.rs @@ -4,3 +4,6 @@ pub(crate) const MACRO_USE: u8 = 0; pub(crate) const DECLARATION_READ: u8 = 1; pub(crate) const GENERATED_SQL: u8 = 2; pub(crate) const ARGUMENT_REFERENCE: u8 = 3; + +/// The native store kind holding recorded macro calls. +pub(crate) const MACRO_CALL_STORE_KIND: &str = "macro-calls"; diff --git a/crates/sqlbuild-python/src/native_store/_helpers/functions.rs b/crates/sqlbuild-python/src/native_store/_helpers/functions.rs new file mode 100644 index 0000000000..f70d170347 --- /dev/null +++ b/crates/sqlbuild-python/src/native_store/_helpers/functions.rs @@ -0,0 +1,34 @@ +//! Digests and project fingerprints of the shared native store, with the GIL released. + +use pyo3::exceptions::{PyOSError, PyRuntimeError}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_cache::digest::main::content_digest::content_digest as digest; +use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use std::collections::HashSet; +use std::path::PathBuf; + +use crate::bindings::types::CompilerDetach; + +/// Hexadecimal digest of length-prefixed UTF-8 `parts`, the digest every native cache key uses. +#[pyfunction] +pub(crate) fn content_digest(parts: Vec) -> String { + hex_digest(&digest(&parts)) +} + +/// Digest of every project file outside skipped folders and `excluded_files`, else `OSError`. +#[pyfunction] +pub(crate) fn fingerprint_project_files( + py: Python<'_>, + root: PathBuf, + excluded_files: Vec, +) -> PyResult { + let excluded: HashSet = excluded_files.into_iter().collect(); + let fingerprinted: ContentDigest = py + .compiler_detach(|| Ok(fingerprint(&root, &excluded))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string()))?; + Ok(hex_digest(&fingerprinted)) +} diff --git a/crates/sqlbuild-python/src/native_store/_helpers/mod.rs b/crates/sqlbuild-python/src/native_store/_helpers/mod.rs new file mode 100644 index 0000000000..8f03d97c29 --- /dev/null +++ b/crates/sqlbuild-python/src/native_store/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/native_store/main/mod.rs b/crates/sqlbuild-python/src/native_store/main/mod.rs new file mode 100644 index 0000000000..51101fd370 --- /dev/null +++ b/crates/sqlbuild-python/src/native_store/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod register; diff --git a/crates/sqlbuild-python/src/native_store/main/register.rs b/crates/sqlbuild-python/src/native_store/main/register.rs new file mode 100644 index 0000000000..518ccf8091 --- /dev/null +++ b/crates/sqlbuild-python/src/native_store/main/register.rs @@ -0,0 +1,12 @@ +//! Register the shared native store's functions on the extension module. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::native_store::_helpers::functions::{content_digest, fingerprint_project_files}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(content_digest, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/native_store/mod.rs b/crates/sqlbuild-python/src/native_store/mod.rs new file mode 100644 index 0000000000..7de28dc3ff --- /dev/null +++ b/crates/sqlbuild-python/src/native_store/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod _helpers; +pub(crate) mod main; diff --git a/crates/sqlbuild-render/Cargo.toml b/crates/sqlbuild-render/Cargo.toml index ac3a9cf79b..ace3d2d9c6 100644 --- a/crates/sqlbuild-render/Cargo.toml +++ b/crates/sqlbuild-render/Cargo.toml @@ -8,7 +8,11 @@ publish.workspace = true rust-version.workspace = true [dependencies] +sqlbuild-cache.workspace = true sqlbuild-core.workspace = true +[dev-dependencies] +tempfile.workspace = true + [lints] workspace = true diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs new file mode 100644 index 0000000000..29d46d32c7 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/entry_codec.rs @@ -0,0 +1,80 @@ +//! Stored bytes of one recorded macro call. + +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +use crate::macro_calls::models::{MacroCallEntry, MacroCallEvent}; + +const MACRO_USE: u64 = 0; +const DECLARATION_READ: u64 = 1; +const GENERATED_SQL: u64 = 2; +const ARGUMENT_REFERENCE: u64 = 3; + +pub(crate) fn encode_entry(entry: &MacroCallEntry) -> Vec { + let mut writer: RecordWriter = RecordWriter::default(); + writer.put_str(&entry.sql); + writer.put_u64(entry.relations.len() as u64); + for (kind, name) in &entry.relations { + writer.put_str(kind); + writer.put_str(name); + } + writer.put_u64(entry.events.len() as u64); + for event in &entry.events { + let (tag, first, second) = match event { + MacroCallEvent::MacroUse { name } => (MACRO_USE, name.as_str(), ""), + MacroCallEvent::DeclarationRead { kind, name } => { + (DECLARATION_READ, kind.as_str(), name.as_str()) + } + MacroCallEvent::GeneratedSql { macro_name, sql } => { + (GENERATED_SQL, macro_name.as_str(), sql.as_str()) + } + MacroCallEvent::ArgumentReference { kind, name } => { + (ARGUMENT_REFERENCE, kind.as_str(), name.as_str()) + } + }; + writer.put_u64(tag); + writer.put_str(first); + writer.put_str(second); + } + writer.into_bytes() +} + +/// The entry stored by [`encode_entry`]; bytes this build did not write are a decode error. +pub(crate) fn decode_entry(bytes: &[u8]) -> Result { + let mut reader: RecordReader<'_> = RecordReader::new(bytes); + let sql: String = reader.string()?; + let relation_count: u64 = reader.u64()?; + let mut relations: Vec<(String, String)> = Vec::new(); + for _ in 0..relation_count { + relations.push((reader.string()?, reader.string()?)); + } + let event_count: u64 = reader.u64()?; + let mut events: Vec = Vec::new(); + for _ in 0..event_count { + let tag: u64 = reader.u64()?; + let first: String = reader.string()?; + let second: String = reader.string()?; + events.push(match tag { + MACRO_USE if second.is_empty() => MacroCallEvent::MacroUse { name: first }, + DECLARATION_READ => MacroCallEvent::DeclarationRead { + kind: first, + name: second, + }, + GENERATED_SQL => MacroCallEvent::GeneratedSql { + macro_name: first, + sql: second, + }, + ARGUMENT_REFERENCE => MacroCallEvent::ArgumentReference { + kind: first, + name: second, + }, + _ => return Err(StoreDecodeError), + }); + } + reader.finish()?; + Ok(MacroCallEntry { + sql, + relations, + events, + }) +} diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs index e6dea55521..92e1d6be79 100644 --- a/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod entry_codec; pub(crate) mod offsets; pub(crate) mod scanner; diff --git a/crates/sqlbuild-render/src/macro_calls/models.rs b/crates/sqlbuild-render/src/macro_calls/models.rs index f125026a05..d2d5ada473 100644 --- a/crates/sqlbuild-render/src/macro_calls/models.rs +++ b/crates/sqlbuild-render/src/macro_calls/models.rs @@ -1,8 +1,14 @@ //! Macro call sites, recorded macro call results and the in-compile memo that replays them. +use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::NativeStore; use std::collections::HashMap; use std::sync::Arc; +use crate::macro_calls::_helpers::entry_codec::{decode_entry, encode_entry}; + /// One top-level `@macro(...)` call in an authored SQL string; offsets count code points. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MacroCallSite { @@ -55,28 +61,47 @@ pub struct MacroSpliceSpan { pub output_end: usize, } -/// Recorded macro call results of one compile, keyed by call class and exact call text. +/// One compile's recorded macro call results by class and call text, plus an optional store. #[derive(Debug, Default)] pub struct MacroCallMemo { entries: HashMap<(u64, String), Arc>, hits: usize, misses: usize, + store: Option, + class_digests: HashMap, + store_hits: usize, + store_records: usize, } impl MacroCallMemo { - /// Return the recorded result for the call, counting the lookup as a hit or a miss. + /// The recorded result of the call, found in this compile or, if persistent, in the store. pub fn lookup(&mut self, class_id: u64, call_text: &str) -> Option> { - let found = self.entries.get(&(class_id, call_text.to_owned())).cloned(); - if found.is_some() { + let key: (u64, String) = (class_id, call_text.to_owned()); + if let Some(found) = self.entries.get(&key).cloned() { self.hits += 1; - } else { - self.misses += 1; + return Some(found); + } + match self.stored_entry(class_id, call_text) { + Some(entry) => { + self.store_hits += 1; + let _ = self.entries.insert(key, Arc::clone(&entry)); + Some(entry) + } + None => { + self.misses += 1; + None + } } - found } - /// Record the result of one executed call. + /// Record the result of one executed call, storing it when its class is persistent. pub fn record(&mut self, class_id: u64, call_text: String, entry: MacroCallEntry) { + if let Some(store_key) = self.store_key(class_id, &call_text) + && let Some(store) = self.store.as_mut() + { + store.put(store_key, encode_entry(&entry)); + self.store_records += 1; + } let _ = self.entries.insert((class_id, call_text), Arc::new(entry)); } @@ -84,4 +109,44 @@ impl MacroCallMemo { pub fn stats(&self) -> (usize, usize, usize) { (self.hits, self.misses, self.entries.len()) } + + /// Attach a store whose keys already cover everything every persistent class depends on. + pub fn attach_store(&mut self, store: NativeStore) { + self.store = Some(store); + } + + /// The attached store, if any. + pub fn store_mut(&mut self) -> Option<&mut NativeStore> { + self.store.as_mut() + } + + /// Mark `class_id` persistent under `class_text`, which names all its results depend on. + pub fn set_persistent_class(&mut self, class_id: u64, class_text: &str) { + let _ = self + .class_digests + .insert(class_id, content_digest(&[class_text])); + } + + /// Calls found in the store and calls recorded into it. + pub fn store_stats(&self) -> (usize, usize) { + (self.store_hits, self.store_records) + } + + fn stored_entry(&mut self, class_id: u64, call_text: &str) -> Option> { + let store_key: ContentDigest = self.store_key(class_id, call_text)?; + let bytes: &[u8] = self.store.as_mut()?.get(&store_key)?; + match decode_entry(bytes) { + Ok(entry) => Some(Arc::new(entry)), + Err(StoreDecodeError) => None, + } + } + + fn store_key(&self, class_id: u64, call_text: &str) -> Option { + self.store.as_ref()?; + let class_digest = self.class_digests.get(&class_id)?; + Some(content_digest(&[ + class_digest.as_slice(), + call_text.as_bytes(), + ])) + } } diff --git a/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs index 181f047d89..2fa89bc6ec 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/helpers.rs @@ -1,5 +1,7 @@ +use sqlbuild_cache::store::main::open_native_store::open_native_store; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_core::text::models::PythonText; +use std::path::Path; use crate::macro_calls::models::{MacroCallEntry, MacroCallEvent, MacroCallMemo}; @@ -27,3 +29,48 @@ pub(crate) fn recorded_memo() -> (MacroCallMemo, MacroCallEntry) { pub(crate) fn python_312() -> PythonText { python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") } + +/// Record `entry` for `@cents(amount)` into a fresh store at `path`, persistent per `class_texts`. +pub(crate) fn saved_memo_store(path: &Path, class_texts: &[&str], entry: &MacroCallEntry) { + let mut memo = MacroCallMemo::default(); + memo.attach_store( + open_native_store(path, STORE_KIND, STORE_ENVIRONMENT).expect("opened store"), + ); + for class_text in class_texts { + memo.set_persistent_class(1, class_text); + } + memo.record(1, "@cents(amount)".to_owned(), entry.clone()); + let _ = memo + .store_mut() + .expect("attached store") + .save(path, b"") + .expect("saved store"); +} + +pub(crate) const STORE_KIND: &str = "macro-calls"; +pub(crate) const STORE_ENVIRONMENT: &str = "environment"; + +/// An entry with one event of every kind. +pub(crate) fn every_event_entry() -> MacroCallEntry { + MacroCallEntry { + sql: "SELECT * FROM __SQLBUILD_RELATION_1__ -- café".to_owned(), + relations: vec![("model".to_owned(), "orders".to_owned())], + events: vec![ + MacroCallEvent::MacroUse { + name: "cents".to_owned(), + }, + MacroCallEvent::DeclarationRead { + kind: "enum".to_owned(), + name: "order_status".to_owned(), + }, + MacroCallEvent::GeneratedSql { + macro_name: "cents".to_owned(), + sql: "__ref(\"orders\")".to_owned(), + }, + MacroCallEvent::ArgumentReference { + kind: "source".to_owned(), + name: "raw.orders".to_owned(), + }, + ], + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs index 5a9866a43b..0bae036198 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_memo.rs @@ -1,5 +1,10 @@ -use crate::macro_calls::tests::helpers::recorded_memo; -use crate::macro_calls::tests::test_types::MacroCallMemoTestCase; +use sqlbuild_cache::store::main::open_native_store::open_native_store; + +use crate::macro_calls::models::MacroCallMemo; +use crate::macro_calls::tests::helpers::{ + STORE_ENVIRONMENT, STORE_KIND, every_event_entry, recorded_memo, saved_memo_store, +}; +use crate::macro_calls::tests::test_types::{MacroCallMemoTestCase, MacroCallStoreTestCase}; #[test] fn given_recorded_call_when_looking_up_then_only_same_class_and_text_hit() { @@ -45,3 +50,73 @@ fn given_recorded_call_when_looking_up_then_only_same_class_and_text_hit() { ); } } + +#[test] +fn given_stored_call_when_a_new_memo_looks_it_up_then_only_the_same_persistent_class_hits() { + let test_cases = [ + MacroCallStoreTestCase { + description: "same persistent class and call text", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: true, + }, + MacroCallStoreTestCase { + description: "another persistent class", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@models/marts/_sqlbuild/_macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "another call text", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(total)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "class not persistent when recorded", + recorded_class_texts: &[], + looked_up_class_texts: &["cents@macros/cents.py"], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + MacroCallStoreTestCase { + description: "class not persistent when looked up", + recorded_class_texts: &["cents@macros/cents.py"], + looked_up_class_texts: &[], + call_text: "@cents(amount)", + expected_store_hit: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("macro-calls.bin"); + let entry = every_event_entry(); + saved_memo_store(&path, test_case.recorded_class_texts, &entry); + let mut memo = MacroCallMemo::default(); + memo.attach_store( + open_native_store(&path, STORE_KIND, STORE_ENVIRONMENT).expect("opened store"), + ); + for class_text in test_case.looked_up_class_texts { + memo.set_persistent_class(7, class_text); + } + + let found = memo.lookup(7, test_case.call_text); + + assert_eq!( + found.as_deref() == Some(&entry), + test_case.expected_store_hit, + "{}", + test_case.description + ); + assert_eq!( + memo.store_stats(), + (usize::from(test_case.expected_store_hit), 0), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs index c2c785e736..39af8176e4 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs @@ -24,3 +24,11 @@ pub(crate) struct MacroCallMemoTestCase { pub(crate) call_text: &'static str, pub(crate) expected_hit: bool, } + +pub(crate) struct MacroCallStoreTestCase { + pub(crate) description: &'static str, + pub(crate) recorded_class_texts: &'static [&'static str], + pub(crate) looked_up_class_texts: &'static [&'static str], + pub(crate) call_text: &'static str, + pub(crate) expected_store_hit: bool, +} diff --git a/fensu.toml b/fensu.toml index 82f714743b..7520b517e9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -305,6 +305,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ "sqlbuild-analysis", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d1c6aa1bc..dbbe8014c4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -203,6 +203,14 @@ class MacroCallMemo: entry: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]], ) -> None: ... def stats(self) -> tuple[int, int, int]: ... + def attach_store(self, path: str, environment: str) -> tuple[bytes, int]: ... + def discard_store(self) -> None: ... + def set_persistent_class(self, class_id: int, class_text: str) -> None: ... + def store_stats(self) -> tuple[int, int]: ... + def save_store(self, path: str, metadata: bytes) -> int | None: ... + +def content_digest(parts: list[str]) -> str: ... +def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... From af1bdc0aa0c813c4efcbe7adcccaf5208a0b78f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:49:36 +0100 Subject: [PATCH 05/11] feat(native): cache macro call results across compiles The native macro bridge now loads stored call results when the compile cache is enabled and saves the results it used or recorded after the render stage succeeds. A stored result is valid only in the same environment: native build, interpreter, import search path, SQLBUILD_* and SQB_* variables, and the content of every project file except model SQL files; loaded module files are stamped and rechecked before use. Each call is keyed by its text, the name, file, declaration identity and source of every macro in the call, the relations passed so far, and for context macros the whole context by value (vars, target, adapter, visible constants and enums). Values without stable text make the call uncached. An edit-sequence e2e test compiles each edit with the store and with --no-cache, with and without project reuse, and a broken store key must make it fail. The compiler differential compiles every authored and generated project a second time with a warm store. --- .../_helpers/corpus/corpus.py | 7 +- .../_helpers/running/execution.py | 7 +- scripts/compiler_differential/constants.py | 5 + scripts/compiler_differential/models.py | 3 +- .../_helpers/macro_bridge/call_store.py | 48 ++++ .../_helpers/macro_bridge/expansion.py | 9 +- .../compile/main/_build_compile_inputs.py | 14 +- .../macro_bridge/_helpers/__init__.py | 1 + .../_helpers/store_environment.py | 115 +++++++++ .../macro_bridge/_helpers/store_keys.py | 143 ++++++++++++ .../macro_bridge/classes/macro_bridge.py | 151 +++++++++--- .../compiler/macro_bridge/constants.py | 11 + .../compiler/macro_bridge/exceptions.py | 7 + .../main/run_with_macro_bridge.py | 9 +- src/sqlbuild/compiler/macro_bridge/models.py | 13 +- src/sqlbuild/compiler/macro_bridge/types.py | 1 + .../test_compiler_differential.py | 6 +- .../cli/commands/main/compile/_test_types.py | 30 +++ .../cli/commands/main/compile/helpers.py | 140 +++++++++++ .../compile/test_compile_macro_call_store.py | 219 ++++++++++++++++++ .../macro_bridge/_helpers/__init__.py | 0 .../macro_bridge/_helpers/_test_types.py | 86 +++++++ .../compiler/macro_bridge/_helpers/helpers.py | 94 ++++++++ .../_helpers/test_store_environment.py | 157 +++++++++++++ .../macro_bridge/_helpers/test_store_keys.py | 209 +++++++++++++++++ 25 files changed, 1432 insertions(+), 53 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py create mode 100644 src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py create mode 100644 src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py create mode 100644 src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py create mode 100644 src/sqlbuild/compiler/macro_bridge/exceptions.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index 805f8aa782..b217c7f0c0 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -23,6 +23,7 @@ FIXTURE_ROOT, PLAN, PROJECT_CONFIG_FILE, + STORE_WARM_COMPILE, WARM_COMPILE, ) from scripts.compiler_differential.models import ( @@ -91,7 +92,7 @@ def build_corpus( entries.extend( CorpusProject( name=f"dense/{dense_models}{'-custom-rules' if custom_rules else ''}", - commands=(COLD_COMPILE, WARM_COMPILE), + commands=(COLD_COMPILE, WARM_COMPILE, STORE_WARM_COMPILE), expected=EXPECT_SUCCESS, writer=DenseProject(model_count=dense_models, custom_rules=custom_rules).write, ) @@ -115,7 +116,7 @@ def _authored_project( project_dir: Path = source_dir if subdirectory is None else source_dir / subdirectory return CorpusProject( name=name, - commands=(COLD_COMPILE, WARM_COMPILE, *_plan_commands(project_dir)), + commands=(COLD_COMPILE, WARM_COMPILE, STORE_WARM_COMPILE, *_plan_commands(project_dir)), expected=expected, source_dir=source_dir, project_subdirectory=subdirectory, @@ -137,7 +138,7 @@ def _seed_project(seed: int) -> CorpusProject: generated: GeneratedProject = ProjectBuilder(seed).build() return CorpusProject( name=f"seed/{seed}", - commands=(COLD_COMPILE, PLAN), + commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN), expected=ExpectedOutcome( error_code=generated.expected_error_code, succeeding_commands=generated.succeeding_commands, diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 5e1b13c5c4..3c4b283cb0 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -69,9 +69,10 @@ def run_engine( _run_command( command=command, project_dir=project_dir, - environment=_command_environment( - engine=engine, options=options, capture_dir=capture_dir - ), + environment={ + **_command_environment(engine=engine, options=options, capture_dir=capture_dir), + **dict(command.environment), + }, python=options.python, ) ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 756d0a779f..de07f6da70 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -85,6 +85,11 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: WARM_COMPILE: DifferentialCommand = DifferentialCommand( label="compile-warm", arguments=("compile", "--json") ) +STORE_WARM_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile-store-warm", + arguments=("compile", "--json"), + environment=(("SQLBUILD_DISABLE_COMPILE_REUSE", "1"),), +) PLAN: DifferentialCommand = DifferentialCommand(label=PLAN_LABEL, arguments=("plan", "--json")) GENERATOR_DOMAINS: tuple[str, ...] = ("sales", "inventory", "support", "fulfillment", "billing") diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 5a399c2e9c..3af73b087b 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -9,10 +9,11 @@ @dataclass(frozen=True) class DifferentialCommand: - """One sqb invocation run identically under both engines.""" + """One sqb invocation run identically under both engines, with extra environment values.""" label: str arguments: tuple[str, ...] + environment: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py new file mode 100644 index 0000000000..de321d0c39 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -0,0 +1,48 @@ +"""Resolve the compile target and hand the running macro bridge its call store.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.spec.contracts.models import TargetConfig + + +def target_context_with_macro_call_store( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + no_cache: bool, +) -> tuple[str | None, TargetConfig | None, Path | None]: + """Resolve the target and cache directory, loading stored macro calls when a bridge runs.""" + + target_context: tuple[str | None, TargetConfig | None, Path | None] = ( + build_compile_target_context( + discovered_inputs=discovered_inputs, + selected_target=selected_target, + no_cache=no_cache, + ) + ) + bridge: MacroBridge | None = active_macro_bridge() + project_dir: Path | None = discovered_inputs.project_dir + compile_cache_dir: Path | None = target_context[2] + if bridge is not None and compile_cache_dir is not None and project_dir is not None: + bridge.attach_store( + cache_dir=compile_cache_dir, + project_dir=project_dir, + model_paths=_model_paths(discovered_inputs=discovered_inputs, project_dir=project_dir), + ) + return target_context + + +def _model_paths(*, discovered_inputs: DiscoveredProjectInputs, project_dir: Path) -> list[str]: + paths: list[str] = [] + for model_file in discovered_inputs.model_files: + try: + paths.append(model_file.file_path.relative_to(project_dir).as_posix()) + except ValueError: + continue + return paths diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py index 4c5f8caad4..a9f48cdcf3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Hashable from pathlib import Path from sqlbuild.compiler.compile._helpers.render.macros import ( @@ -28,7 +27,7 @@ GENERATED_SQL_EVENT, MACRO_USE_EVENT, ) -from sqlbuild.compiler.macro_bridge.models import MacroCallSite +from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind @@ -51,9 +50,9 @@ def expand_bridged_sql_macros( state=state, consumer_path=consumer_path ) sites: tuple[MacroCallSite, ...] | None = bridge.scan(sql) - call_classes: list[Hashable] = [] + call_classes: list[MacroCallClass] = [] for site in sites or (): - call_class: Hashable | None = bridge.call_class( + call_class: MacroCallClass | None = bridge.call_class( site=site, declarations=declarations, loaded_macros=state.loaded_macros, @@ -94,7 +93,7 @@ def _bridged_call_output( # noqa: PLR0913 declarations: DeclarationResolutionContext | None, bridge: MacroBridge, site: MacroCallSite, - call_class: Hashable, + call_class: MacroCallClass, ) -> str: facts: MacroExpansionFacts = state.facts call_text: str = sql[site.start : site.end] diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index ebee19a4b6..2b857fdbdb 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) -from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) @@ -31,6 +30,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( missing_description_diagnostics, ) +from sqlbuild.compiler.compile._helpers.macro_bridge.call_store import ( + target_context_with_macro_call_store, +) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -93,10 +95,12 @@ def build_compile_inputs( effective_target_name: str | None effective_target: TargetConfig | None compile_cache_dir: Path | None - effective_target_name, effective_target, compile_cache_dir = build_compile_target_context( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - no_cache=no_cache, + effective_target_name, effective_target, compile_cache_dir = ( + target_context_with_macro_call_store( + discovered_inputs=discovered_inputs, + selected_target=selected_target, + no_cache=no_cache, + ) ) effective_vars: dict[str, object] = build_effective_vars( diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py new file mode 100644 index 0000000000..a88eb85027 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py @@ -0,0 +1 @@ +"""Macro call store keys and the environment they are valid in.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py new file mode 100644 index 0000000000..7bcd8f0300 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -0,0 +1,115 @@ +"""The environment stored macro call results are valid in: build, interpreter and project files.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.macro_bridge.constants import ( + MACRO_CALL_STORE_ENVIRONMENT_VERSION, + MISSING_SEARCH_PATH_STAMP, + MODULE_STAMP_ROW_FIELDS, + PROJECT_ROOT_SEARCH_PATH_STAMP, + STORE_TRACKED_ENVIRONMENT_PREFIXES, + STORE_UNTRACKED_ENVIRONMENT_NAMES, +) +from sqlbuild.compiler.macro_bridge.types import ModuleStamp + + +def store_environment(*, project_dir: Path, model_paths: list[str]) -> str: + """Digest every input but model SQL files and context; OSError if a file is unreadable.""" + + fingerprint: str = _native.fingerprint_project_files(str(project_dir), model_paths) + return _native.content_digest( + [ + MACRO_CALL_STORE_ENVIRONMENT_VERSION, + _native.BUILD_IDENTITY, + sys.version, + sys.executable, + sys.platform, + str(sys.implementation.cache_tag), + json.dumps(_search_path_stamps(project_dir=project_dir)), + json.dumps(_tracked_environment()), + fingerprint, + ] + ) + + +def loaded_module_stamps() -> dict[str, ModuleStamp]: + """Stat every loaded module file, including modules macros imported while running.""" + + stamps: dict[str, ModuleStamp] = {} + for module in tuple(sys.modules.values()): + path: object = getattr(module, "__file__", None) + if not isinstance(path, str) or path in stamps: + continue + try: + status: os.stat_result = os.stat(path) + except OSError: + continue + stamps[path] = (status.st_mtime_ns, status.st_size) + return stamps + + +def module_stamps_metadata(stamps: dict[str, ModuleStamp]) -> bytes: + """Encode module stamps for the store's metadata.""" + + return json.dumps(sorted([path, *stamp] for path, stamp in stamps.items())).encode() + + +def unchanged_module_stamps(metadata: bytes) -> dict[str, ModuleStamp] | None: + """Return the stored module stamps when every module file is unchanged, else None.""" + + try: + rows: object = json.loads(metadata) + except ValueError: + return None + if not isinstance(rows, list): + return None + stamps: dict[str, ModuleStamp] = {} + for row in rows: + if not ( + isinstance(row, list) + and len(row) == MODULE_STAMP_ROW_FIELDS + and isinstance(row[0], str) + and isinstance(row[1], int) + and isinstance(row[2], int) + ): + return None + try: + status: os.stat_result = os.stat(row[0]) + except OSError: + return None + if (status.st_mtime_ns, status.st_size) != (row[1], row[2]): + return None + stamps[row[0]] = (row[1], row[2]) + return stamps + + +def _search_path_stamps(*, project_dir: Path) -> list[tuple[str, int]]: + project_root: str = os.path.realpath(project_dir) + stamps: list[tuple[str, int]] = [] + for entry in sys.path: + path: str = entry or "." + try: + mtime_ns: int = ( + PROJECT_ROOT_SEARCH_PATH_STAMP + if os.path.realpath(path) == project_root + else os.stat(path).st_mtime_ns + ) + except OSError: + mtime_ns = MISSING_SEARCH_PATH_STAMP + stamps.append((entry, mtime_ns)) + return stamps + + +def _tracked_environment() -> list[tuple[str, str]]: + return sorted( + (name, value) + for name, value in os.environ.items() + if name.startswith(STORE_TRACKED_ENVIRONMENT_PREFIXES) + and name not in STORE_UNTRACKED_ENVIRONMENT_NAMES + ) diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py new file mode 100644 index 0000000000..f507172914 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py @@ -0,0 +1,143 @@ +"""Value-based text of everything a stored macro call result depends on besides its call text.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import json +import types +from pathlib import PurePath + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + LoadedMacro, + MacroContext, +) +from sqlbuild.compiler.macro_bridge.constants import VALUE_RENDERER_FIELD +from sqlbuild.compiler.macro_bridge.exceptions import UncacheableValueError +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.python_nodes.models import SqlResourceRef + + +def macro_store_token(*, loaded: LoadedMacro, identity: DeclarationIdentity | None) -> str: + """Name a macro by its name, file, declaration identity and exact source.""" + + return _key_text( + [ + loaded.name, + loaded.relative_path.as_posix(), + _canonical(identity), + _native.content_digest([loaded.raw_source]), + ] + ) + + +def context_store_token( + *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None +) -> str | None: + """Digest the whole context a macro may read, or None when a value has no stable text.""" + + try: + text: str = _key_text( + [ + _canonical_context(macro_context), + None + if declarations is None + else [ + _canonical(declarations.constants), + _canonical(declarations.enums), + sorted(declarations.inaccessible_constants), + sorted(declarations.inaccessible_enums), + ], + ] + ) + except UncacheableValueError: + return None + return _native.content_digest([text]) + + +def call_class_store_text( + *, + macro_tokens: tuple[str, ...], + context_token: str | None, + prior_relations: tuple[SqlResourceRef, ...] | None, +) -> str: + """Text of one persistent call class: its macros, context and relations passed so far.""" + + return _key_text( + [ + list(macro_tokens), + context_token, + None + if prior_relations is None + else [[relation.kind.value, relation.name] for relation in prior_relations], + ] + ) + + +def _canonical_context(macro_context: MacroContext) -> object: + fields: list[object] = [] + for field in dataclasses.fields(macro_context): + value: object = getattr(macro_context, field.name) + fields.append( + [ + field.name, + _type_name(type(value)) + if field.name == VALUE_RENDERER_FIELD + else _canonical(value), + ] + ) + return [_type_name(type(macro_context)), fields] + + +def _canonical(value: object) -> object: # noqa: PLR0911 + if isinstance(value, enum.Enum): + return ["enum", _type_name(type(value)), _canonical(value.value)] + if value is None or isinstance(value, bool | str): + return value + if isinstance(value, int): + return ["int", str(value)] + if isinstance(value, float): + return ["float", value.hex()] + if isinstance(value, decimal.Decimal): + return ["decimal", str(value)] + if isinstance(value, datetime.date | datetime.time): + return [_type_name(type(value)), value.isoformat()] + if isinstance(value, datetime.timedelta): + return ["timedelta", value.days, value.seconds, value.microseconds] + if isinstance(value, bytes): + return ["bytes", value.hex()] + if isinstance(value, PurePath): + return ["path", value.as_posix()] + if isinstance(value, list | tuple): + return [_type_name(type(value)), [_canonical(item) for item in value]] + if isinstance(value, set | frozenset): + return [ + _type_name(type(value)), + sorted(json.dumps(_canonical(item), sort_keys=True) for item in value), + ] + if isinstance(value, dict | types.MappingProxyType): + return [ + "mapping", + [[_canonical(key), _canonical(item)] for key, item in value.items()], + ] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return [ + _type_name(type(value)), + [ + [field.name, _canonical(getattr(value, field.name))] + for field in dataclasses.fields(value) + ], + ] + raise UncacheableValueError(_type_name(type(value))) + + +def _type_name(kind: type) -> str: + return f"{kind.__module__}.{kind.__qualname__}" + + +def _key_text(value: object) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index ca378276c5..caa3708202 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -2,7 +2,9 @@ from __future__ import annotations +import logging from collections.abc import Hashable, Mapping +from pathlib import Path import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( @@ -11,8 +13,21 @@ LoadedMacro, MacroContext, ) -from sqlbuild.compiler.macro_bridge.models import MacroCallSite -from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + loaded_module_stamps, + module_stamps_metadata, + store_environment, + unchanged_module_stamps, +) +from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( + call_class_store_text, + context_store_token, + macro_store_token, +) +from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME +from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord, ModuleStamp +from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -26,8 +41,47 @@ def __init__(self, *, python_version: tuple[int, int], unicode_version: str) -> self._memo: _native.MacroCallMemo = _native.MacroCallMemo() self._class_ids: dict[Hashable, int] = {} self._retained: dict[int, object] = {} - self._name_tokens: dict[tuple[int, str], Hashable | None] = {} - self._context_tokens: dict[tuple[int, int], Hashable] = {} + self._name_tokens: dict[tuple[int, str], tuple[Hashable, str | None] | None] = {} + self._context_tokens: dict[tuple[int, int], tuple[Hashable, str | None]] = {} + self._store_path: Path | None = None + self._carried_module_stamps: dict[str, ModuleStamp] = {} + + def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[str]) -> None: + """Reuse call results stored by earlier compiles whose environment matches this one.""" + + if self._store_path is not None: + return + path: Path = cache_dir / MACRO_CALL_STORE_FILE_NAME + metadata: bytes + loaded: int + try: + environment: str = store_environment(project_dir=project_dir, model_paths=model_paths) + metadata, loaded = self._memo.attach_store(str(path), environment) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("Macro call store not loaded: %s", error) + return + stamps: dict[str, ModuleStamp] | None = unchanged_module_stamps(metadata) if loaded else {} + if stamps is None: + self._memo.discard_store() + stamps = {} + self._store_path = path + self._carried_module_stamps = stamps + self._name_tokens.clear() + self._context_tokens.clear() + + def save_store(self) -> None: + """Save the stored results this compile used or recorded, with the modules they ran on.""" + + if self._store_path is None: + return + stamps: dict[str, ModuleStamp] = { + **self._carried_module_stamps, + **loaded_module_stamps(), + } + try: + _ = self._memo.save_store(str(self._store_path), module_stamps_metadata(stamps)) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("Macro call store not saved: %s", error) def scan(self, sql: str) -> tuple[MacroCallSite, ...] | None: """Return the top-level call sites of `sql`, or None when Python must expand it.""" @@ -55,7 +109,7 @@ def call_class( declarations: DeclarationResolutionContext | None, loaded_macros: dict[str, LoadedMacro], macro_context: MacroContext, - ) -> Hashable | None: + ) -> MacroCallClass | None: """Return what a call's result may depend on besides its text, or None if unresolvable.""" owner: object = loaded_macros if declarations is None else declarations @@ -65,32 +119,54 @@ def call_class( _ = self._retained.setdefault(id(owner), owner) _ = self._retained.setdefault(id(macro_context), macro_context) tokens: list[Hashable] = [] + store_tokens: list[str | None] = [] for name in site.tree_names: - token: Hashable | None = self._name_token( + token: tuple[Hashable, str | None] | None = self._name_token( owner=owner, macros=macros, declarations=declarations, name=name ) if token is None: return None - tokens.append(token) + tokens.append(token[0]) + store_tokens.append(token[1]) injects_context: bool = any(macros[name].injects_context for name in site.tree_names) - context: Hashable | None = ( + context: tuple[Hashable, str | None] | None = ( self._context_token(owner=owner, declarations=declarations, macro_context=macro_context) if injects_context else None ) - return (tuple(tokens), context) + persistent_tokens: tuple[str, ...] = tuple( + token for token in store_tokens if token is not None + ) + return MacroCallClass( + key=(tuple(tokens), None if context is None else context[0]), + macro_store_tokens=persistent_tokens, + context_store_token=None if context is None else context[1], + persistent=len(persistent_tokens) == len(store_tokens) + and (context is None or context[1] is not None), + ) def lookup( self, *, - call_class: Hashable, + call_class: MacroCallClass, call_text: str, prior_relations: tuple[SqlResourceRef, ...] | None, ) -> tuple[int, MacroCallRecord | None]: - """Return the class id of a call and its recorded result, if one was recorded.""" + """Return the class id of a call and its recorded or stored result, if any.""" - class_key: Hashable = (call_class, prior_relations) - class_id: int = self._class_ids.setdefault(class_key, len(self._class_ids)) + class_key: Hashable = (call_class.key, prior_relations) + class_id: int | None = self._class_ids.get(class_key) + if class_id is None: + class_id = self._class_ids[class_key] = len(self._class_ids) + if self._store_path is not None and call_class.persistent: + self._memo.set_persistent_class( + class_id, + call_class_store_text( + macro_tokens=call_class.macro_store_tokens, + context_token=call_class.context_store_token, + prior_relations=prior_relations, + ), + ) found: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None = ( self._memo.lookup(class_id, call_text) ) @@ -147,6 +223,11 @@ def stats(self) -> tuple[int, int, int]: return self._memo.stats() + def store_stats(self) -> tuple[int, int]: + """Calls found in the store and calls recorded into it.""" + + return self._memo.store_stats() + def _name_token( self, *, @@ -154,21 +235,24 @@ def _name_token( macros: Mapping[str, LoadedMacro], declarations: DeclarationResolutionContext | None, name: str, - ) -> Hashable | None: + ) -> tuple[Hashable, str | None] | None: key: tuple[int, str] = (id(owner), name) if key in self._name_tokens: return self._name_tokens[key] loaded: LoadedMacro | None = macros.get(name) - token: Hashable | None = ( - None - if loaded is None - or (declarations is not None and name in declarations.inaccessible_macros) - else ( - name, - id(loaded), - declarations.macro_records[name].identity if declarations is not None else None, + token: tuple[Hashable, str | None] | None = None + if loaded is not None and ( + declarations is None or name not in declarations.inaccessible_macros + ): + identity: DeclarationIdentity | None = ( + declarations.macro_records[name].identity if declarations is not None else None + ) + token = ( + (name, id(loaded), identity), + macro_store_token(loaded=loaded, identity=identity) + if self._store_path is not None + else None, ) - ) self._name_tokens[key] = token return token @@ -178,18 +262,23 @@ def _context_token( owner: object, declarations: DeclarationResolutionContext | None, macro_context: MacroContext, - ) -> Hashable: + ) -> tuple[Hashable, str | None]: key: tuple[int, int] = (id(owner), id(macro_context)) - token: Hashable | None = self._context_tokens.get(key) + token: tuple[Hashable, str | None] | None = self._context_tokens.get(key) if token is None: token = ( - id(macro_context), - None - if declarations is None - else ( - tuple((name, id(value)) for name, value in declarations.constants.items()), - tuple((name, id(value)) for name, value in declarations.enums.items()), + ( + id(macro_context), + None + if declarations is None + else ( + tuple((name, id(value)) for name, value in declarations.constants.items()), + tuple((name, id(value)) for name, value in declarations.enums.items()), + ), ), + context_store_token(macro_context=macro_context, declarations=declarations) + if self._store_path is not None + else None, ) self._context_tokens[key] = token return token diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index aee9b7ede8..4ac7e5a682 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -4,6 +4,7 @@ from contextvars import ContextVar +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR from sqlbuild.compiler.references.types import SqlReferenceKind MACRO_USE_EVENT: int = 0 @@ -16,3 +17,13 @@ ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) +MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" +MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v1" +VALUE_RENDERER_FIELD: str = "_value_renderer" +PROJECT_ROOT_SEARCH_PATH_STAMP: int = -2 +MISSING_SEARCH_PATH_STAMP: int = -1 +MODULE_STAMP_ROW_FIELDS: int = 3 +STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") +STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( + {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} +) diff --git a/src/sqlbuild/compiler/macro_bridge/exceptions.py b/src/sqlbuild/compiler/macro_bridge/exceptions.py new file mode 100644 index 0000000000..e3e4181d26 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/exceptions.py @@ -0,0 +1,7 @@ +"""Macro bridge exceptions.""" + +from __future__ import annotations + + +class UncacheableValueError(Exception): + """A value a macro may read has no stable text, so calls that may read it are not stored.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py index 981f174d6e..030621e48e 100644 --- a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py @@ -15,7 +15,7 @@ def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: - """Run `stage` with the bridge; a failure re-runs Python, which raises the exact error.""" + """Run `stage` with the bridge, storing calls on success; a failure re-runs Python.""" python_version: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) if not _native.native_text_supported(python_version, unicodedata.unidata_version): @@ -30,12 +30,17 @@ def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: except Exception as error: native_error = error else: + bridge.save_store() hits, misses, recorded = bridge.stats() + store_hits, store_records = bridge.store_stats() logging.getLogger(__name__).debug( - "Native macro bridge: %d memo hits, %d misses, %d recorded calls", + "Native macro bridge: %d memo hits, %d store hits, %d misses, %d recorded calls, " + "%d stored calls", hits, + store_hits, misses, recorded, + store_records, ) return result finally: diff --git a/src/sqlbuild/compiler/macro_bridge/models.py b/src/sqlbuild/compiler/macro_bridge/models.py index 3f520b5f92..10e812364c 100644 --- a/src/sqlbuild/compiler/macro_bridge/models.py +++ b/src/sqlbuild/compiler/macro_bridge/models.py @@ -1,7 +1,8 @@ -"""Macro call sites found by the native scan.""" +"""Macro call sites found by the native scan and the classes of their results.""" from __future__ import annotations +from collections.abc import Hashable from dataclasses import dataclass @@ -14,3 +15,13 @@ class MacroCallSite: name: str tree_names: tuple[str, ...] typed_reference_text: bool + + +@dataclass(frozen=True) +class MacroCallClass: + """A call's inputs besides its text: by object identity for the memo, by value for the store.""" + + key: Hashable + macro_store_tokens: tuple[str, ...] + context_store_token: str | None + persistent: bool diff --git a/src/sqlbuild/compiler/macro_bridge/types.py b/src/sqlbuild/compiler/macro_bridge/types.py index 767a96c96c..c738b6a7e9 100644 --- a/src/sqlbuild/compiler/macro_bridge/types.py +++ b/src/sqlbuild/compiler/macro_bridge/types.py @@ -6,3 +6,4 @@ type MacroCallEvent = tuple[int, str, str] type MacroCallRecord = tuple[str, tuple[SqlResourceRef, ...], tuple[MacroCallEvent, ...]] +type ModuleStamp = tuple[int, int] diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 3faf8d875d..42665b68ca 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -117,7 +117,7 @@ def test_given_perturbed_native_engine_when_comparing_then_every_artifact_is_loc expected_lines=("DIFF project/waffle_shop",), expected_patterns=( r"- stage capture 0-compile/001-discovered_project_inputs\.json at /model_files/0/", - r"- stage capture 2-plan/001-discovered_project_inputs\.json at /model_files/0/", + r"- stage capture 3-plan/001-discovered_project_inputs\.json at /model_files/0/", ), expected_absent=(), ) @@ -151,7 +151,9 @@ def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_ expected_patterns=( r"- stage capture 0-compile/002-compile_project_inputs\.json at " r"/model_inputs/0/macro_deps/", - r"- stage capture 2-plan/002-compile_project_inputs\.json at " + r"- stage capture 2-compile-store-warm/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + r"- stage capture 3-plan/002-compile_project_inputs\.json at " r"/model_inputs/0/macro_deps/", ), expected_absent=("001-discovered_project_inputs.json",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 9dbdf12564..2b1063bd41 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -894,3 +894,33 @@ class NativeModelLoopParityTestCase: engines: tuple[str, str] expected_exit_codes: tuple[int, int] expected_report_text: str + + +@dataclass(frozen=True) +class MacroCallStoreEditStep: + """One edit compiled with the macro call store, and the executions it expects.""" + + description: str + edit: Callable[[Path], object] + expected_logged_calls: int + args: tuple[str, ...] = () + env: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True) +class MacroCallStoreEditSequenceTestCase: + """Edits applied in order, each compiled with the macro call store and with --no-cache.""" + + description: str + project_reuse: bool + steps: tuple[MacroCallStoreEditStep, ...] + expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class BrokenMacroCallStoreKeyTestCase: + """An edit the store key no longer covers, which the --no-cache oracle must catch.""" + + description: str + edit: Callable[[Path], None] + expected_matches_uncached: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 7fb69439ac..c4e5341e59 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -37,6 +37,7 @@ import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses +import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, @@ -63,6 +64,10 @@ SetOperationModel, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.integration.src.sqlbuild.compiler.compile.helpers import ( + MACRO_BRIDGE_PROJECT_FILES, + MACRO_CALL_LOG_ENV_VAR, +) _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( (0.50, 1_800), @@ -4263,3 +4268,138 @@ def prepare_reference_call_project(*, tmp_path: Path, staging_from: str, mart_fr ) assert seeded.returncode == 0, seeded.stdout + seeded.stderr return project_dir + + +MACRO_CALL_STORE_ENGINE: str = "native" +STORE_ENVIRONMENT_REGION_VAR: str = "SQB_STORE_TEST_REGION" +_REUSE_DISABLED_VALUES: dict[bool, str] = {True: "0", False: "1"} +_SOUTH_MODEL: str = "models/south/orders_south.sql" +_STORE_EXTERNAL_MODULE: str = "store_flavor_values" +_STORE_PROJECT_EXTRA_FILES: dict[str, str] = { + "macros/context_labels.py": ( + '"""Macros that read the target, the environment and an outside module."""\n\n' + "import os\n\n\n" + "def env_region() -> str:\n" + ' """Return the region from the environment."""\n' + f' return repr(os.environ.get("{STORE_ENVIRONMENT_REGION_VAR}", "none"))\n\n\n' + "def target_label(ctx) -> str:\n" + ' """Name the compile target."""\n' + " return repr(ctx.target_name)\n\n\n" + "def flavor() -> str:\n" + ' """Return a value from a module outside the project, imported while rendering."""\n' + f" import {_STORE_EXTERNAL_MODULE}\n\n" + f" return repr({_STORE_EXTERNAL_MODULE}.VALUE)\n" + ), + "macros/_label_values.py": ( + 'def label_value() -> str:\n """Return the label literal."""\n return "\'first\'"\n' + ), + "macros/labels.py": ( + "from macros._label_values import label_value\n\n\n" + 'def label() -> str:\n """Return the label from a helper module."""\n' + " return label_value()\n" + ), +} + + +class MacroCallStoreRun(NamedTuple): + """A compile with the macro call store, its --no-cache reference, and macro executions.""" + + incremental: CompileReuseRun + reference: CompileReuseRun + logged_calls: int + + @property + def matches(self) -> bool: + """Return whether exit code, report, and every compiled artifact are byte-identical.""" + + return _edit_outcome(self.incremental) == _edit_outcome(self.reference) + + +def prepare_macro_call_store_project(*, project_dir: Path, extlib: Path) -> None: + """Write a macro-heavy project whose macros read vars, target, constants and enums.""" + + for relative_path, content in MACRO_BRIDGE_PROJECT_FILES.items(): + write_project_file( + project_dir, relative_path, content.replace("WHERE id IN @generated_join()\n", "") + ) + for relative_path, content in _STORE_PROJECT_EXTRA_FILES.items(): + write_project_file(project_dir, relative_path, content) + replace_project_text( + project_dir, + "sqlbuild_project.toml", + '[targets.dev]\nschema = "analytics"\n', + '[targets.dev]\nschema = "analytics"\n\n[targets.prod]\nschema = "analytics"\n', + ) + replace_project_text( + project_dir, + _SOUTH_MODEL, + ' description "South orders",\n', + ' description "South orders",\n constants (_south_bonus 3),\n', + ) + replace_project_text( + project_dir, + _SOUTH_MODEL, + " 'quoted @cents(1)' AS quoted", + " @target_label() AS target_label,\n @flavor() AS flavor,\n @label() AS label,\n" + " @env_region() AS env_region,\n" + " 'quoted @cents(1)' AS quoted", + ) + write_store_flavor(extlib=extlib, value="sweet") + + +def write_store_flavor(*, extlib: Path, value: str) -> None: + """Write, or rewrite in place, the outside module the flavor macro imports.""" + + extlib.mkdir(parents=True, exist_ok=True) + (extlib / f"{_STORE_EXTERNAL_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") + + +def edit_label_helper(project_dir: Path, value: str) -> None: + """Change the literal the helper module behind the label macro returns.""" + + replace_project_text(project_dir, "macros/_label_values.py", "'first'", f"'{value}'") + + +def edit_south_model(project_dir: Path, old: str, new: str) -> None: + """Replace text in the south orders model.""" + + replace_project_text(project_dir, _SOUTH_MODEL, old, new) + + +def macro_call_store_compile( + *, + project_dir: Path, + extlib: Path, + log_path: Path, + project_reuse: bool, + args: tuple[str, ...] = (), + extra_env: tuple[tuple[str, str], ...] = (), +) -> MacroCallStoreRun: + """Compile with the macro call store, count macro executions, then compile with --no-cache.""" + + env: dict[str, str] = { + REUSE_DISABLE_ENV_VAR: _REUSE_DISABLED_VALUES[project_reuse], + MACRO_CALL_LOG_ENV_VAR: str(log_path), + "PYTHONPATH": str(extlib), + "PYTHONDONTWRITEBYTECODE": "1", + **dict(extra_env), + } + global_args: tuple[str, ...] = ("--compiler-engine", MACRO_CALL_STORE_ENGINE) + _ = log_path.write_text("", encoding="utf-8") + incremental: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=env, args=args, global_args=global_args + ) + logged_calls: int = len(log_path.read_text(encoding="utf-8").splitlines()) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=env, args=(*args, "--no-cache"), global_args=global_args + ) + return MacroCallStoreRun( + incremental=incremental, reference=reference, logged_calls=logged_calls + ) + + +def freeze_macro_call_store_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """Break the store key: every project and module state looks like the first one.""" + + monkeypatch.setattr(macro_bridge_class, "store_environment", lambda **_kwargs: "frozen") + monkeypatch.setattr(macro_bridge_class, "unchanged_module_stamps", lambda _metadata: {}) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py new file mode 100644 index 0000000000..1ce542bb33 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -0,0 +1,219 @@ +"""Compiles that reuse stored macro call results must match an uncached compile after every edit.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + BrokenMacroCallStoreKeyTestCase, + MacroCallStoreEditSequenceTestCase, + MacroCallStoreEditStep, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + MACRO_CALL_STORE_ENGINE, + STORE_ENVIRONMENT_REGION_VAR, + CompileReuseRun, + MacroCallStoreRun, + edit_label_helper, + edit_south_model, + freeze_macro_call_store_environment, + in_process_reuse_run, + macro_call_store_compile, + prepare_macro_call_store_project, + replace_project_text, + run_reuse_compile, + write_project_file, + write_store_flavor, +) +from tests.integration.src.sqlbuild.compiler.compile.helpers import MACRO_CALL_LOG_ENV_VAR + +_EDIT_STEPS: tuple[MacroCallStoreEditStep, ...] = ( + MacroCallStoreEditStep( + description="no_change", + edit=lambda _root: None, + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="model_comment", + edit=lambda root: edit_south_model(root, "FROM (", "-- reviewed\nFROM ("), + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="model_adds_a_call", + edit=lambda root: edit_south_model( + root, " @target_label()", ' @logged("tax") AS logged_tax,\n @target_label()' + ), + expected_logged_calls=1, + ), + MacroCallStoreEditStep( + description="model_private_constant_read_by_context_macro", + edit=lambda root: edit_south_model(root, "_south_bonus 3", "_south_bonus 4"), + expected_logged_calls=0, + ), + MacroCallStoreEditStep( + description="cli_vars_read_by_context_macro", + edit=lambda _root: None, + expected_logged_calls=0, + args=("--vars", '{"region": "south"}'), + ), + MacroCallStoreEditStep( + description="target_read_by_context_macro", + edit=lambda _root: None, + expected_logged_calls=0, + args=("--target", "prod"), + ), + MacroCallStoreEditStep( + description="environment_variable_read_by_macro", + edit=lambda _root: None, + expected_logged_calls=2, + env=((STORE_ENVIRONMENT_REGION_VAR, "south"),), + ), + MacroCallStoreEditStep( + description="project_var_edit", + edit=lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'region = "north"', 'region = "east"' + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="constant_file_edit", + edit=lambda root: replace_project_text( + root, "constants/base_rate.sql", "value 1)", "value 5)" + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="enum_edit", + edit=lambda root: replace_project_text( + root, "enums/order_status.sql", 'PLACED "placed"', 'PLACED "open"' + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="macro_body_edit", + edit=lambda root: replace_project_text( + root, "macros/common.py", 'return f"{column} * 100"', 'return f"{column} * 1000"' + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="helper_module_edit", + edit=lambda root: edit_label_helper(root, "other"), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="macro_file_added", + edit=lambda root: ( + write_project_file( + root, + "macros/badges.py", + 'def badge() -> str:\n """Return a badge literal."""\n return "\'gold\'"\n', + ), + edit_south_model(root, " @label() AS label,\n", " @badge() AS badge,\n"), + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="macro_file_removed", + edit=lambda root: ( + edit_south_model(root, " @badge() AS badge,\n", ""), + (root / "macros/badges.py").unlink(), + ), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="outside_module_rewritten_in_place", + edit=lambda root: write_store_flavor(extlib=root.parent / "extlib", value="salty"), + expected_logged_calls=2, + ), + MacroCallStoreEditStep( + description="model_comment_after_every_edit", + edit=lambda root: edit_south_model(root, "-- reviewed\n", "-- reviewed twice\n"), + expected_logged_calls=0, + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroCallStoreEditSequenceTestCase( + description="full_render_each_step", project_reuse=False, steps=_EDIT_STEPS + ), + MacroCallStoreEditSequenceTestCase( + description="with_project_and_render_reuse", project_reuse=True, steps=_EDIT_STEPS + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_with_macro_call_store_then_each_step_matches_uncached( + tmp_path: Path, test_case: MacroCallStoreEditSequenceTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "macro-calls.log" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + cold: MacroCallStoreRun = macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=log_path, + project_reuse=test_case.project_reuse, + ) + assert (cold.incremental.returncode, cold.matches) == (0, True), cold.incremental.stderr + + for step in test_case.steps: + _ = step.edit(project_dir) + run: MacroCallStoreRun = macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=log_path, + project_reuse=test_case.project_reuse, + args=step.args, + extra_env=step.env, + ) + + assert run.incremental.returncode == 0, (step.description, run.incremental.stderr) + assert run.matches is test_case.expected_matches_uncached, step.description + assert run.logged_calls == step.expected_logged_calls, step.description + + +@pytest.mark.parametrize( + "test_case", + [ + BrokenMacroCallStoreKeyTestCase( + description="helper_module_edit", + edit=lambda root: edit_label_helper(root, "other"), + expected_matches_uncached=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_store_key_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( + tmp_path: Path, + test_case: BrokenMacroCallStoreKeyTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(tmp_path / "macro-calls.log")) + freeze_macro_call_store_environment(monkeypatch) + cold: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + assert cold.returncode == 0, cold.stderr + test_case.edit(project_dir) + + broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, + env={"PYTHONPATH": str(extlib), "SQLBUILD_COMPILER_ENGINE": MACRO_CALL_STORE_ENGINE}, + args=("--no-cache",), + ) + + assert (broken.returncode, reference.returncode) == (0, 0) + assert (broken.compiled == reference.compiled) is test_case.expected_matches_uncached diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py new file mode 100644 index 0000000000..3e4bcf916f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from sqlbuild.compiler.compile.models import DeclarationResolutionContext, MacroContext +from sqlbuild.python_nodes.models import SqlResourceRef + + +@dataclass(frozen=True) +class ContextStoreTokenTestCase: + """Two macro contexts and whether their stored results may be shared.""" + + description: str + left: MacroContext + right: MacroContext + expected_equal: bool + left_declarations: DeclarationResolutionContext | None = None + right_declarations: DeclarationResolutionContext | None = None + + +@dataclass(frozen=True) +class ContextValueTestCase: + """A macro context value and whether calls that may read it can be stored.""" + + description: str + context: MacroContext + expected_stored: bool + + +@dataclass(frozen=True) +class CallClassStoreTextTestCase: + """A call class's context and relations, compared with a plain class of the same macros.""" + + description: str + context_token: str | None + prior_relations: tuple[SqlResourceRef, ...] | None + expected_equal: bool + + +@dataclass(frozen=True) +class MacroStoreTokenTestCase: + """A change to a loaded macro and whether its stored results stay valid.""" + + description: str + relative_path: Path + raw_source: str + expected_equal: bool + + +@dataclass(frozen=True) +class ModuleStampsRoundTripTestCase: + """Module stamps encoded as store metadata and validated back.""" + + description: str + module_text: str + expected_valid: bool + + +@dataclass(frozen=True) +class StoreEnvironmentTestCase: + """A project edit and whether stored macro calls stay valid after it.""" + + description: str + edit: Callable[[Path], object] + expected_unchanged: bool + + +@dataclass(frozen=True) +class StoreEnvironmentVariableTestCase: + """An environment variable change and whether stored macro calls stay valid after it.""" + + description: str + name: str + value: str + expected_unchanged: bool + + +@dataclass(frozen=True) +class ModuleStampsTestCase: + """Stored module stamps, a change to the module file, and whether they still validate.""" + + description: str + change: Callable[[Path], bytes] + expected_valid: bool diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py new file mode 100644 index 0000000000..25d9a6374d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import datetime +import decimal +import json +import os +from dataclasses import replace +from pathlib import Path + +from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext +from sqlbuild.compiler.discovery.models import ConstantDeclaration +from sqlbuild.sql_values.models import SqlLogicalType, SqlValue +from sqlbuild.sql_values.types import SqlValueKind + +BASE_CONTEXT: MacroContext = MacroContext( + adapter_name="duckdb", + sql_analysis_enabled=False, + target_name="dev", + vars={ + "region": "north", + "limits": [1, 2.5, decimal.Decimal("3.10")], + "since": datetime.date(2026, 1, 1), + }, +) + + +def with_vars(**values: object) -> MacroContext: + """The base context with some vars replaced.""" + + return replace(BASE_CONTEXT, vars={**BASE_CONTEXT.vars, **values}) + + +def constant_declarations(value: int) -> DeclarationResolutionContext: + """Declarations holding one integer constant.""" + + return DeclarationResolutionContext( + constants={ + "base_rate": ConstantDeclaration( + name="base_rate", + value=SqlValue(logical_type=SqlLogicalType(kind=SqlValueKind.INTEGER), value=value), + relative_path=Path("constants/base_rate.sql"), + ) + } + ) + + +def loaded_macro(*, relative_path: Path, raw_source: str) -> LoadedMacro: + """A loaded `cents` macro from the given file.""" + + return LoadedMacro( + name="cents", + file_path=Path("/project") / relative_path, + relative_path=relative_path, + raw_source=raw_source, + function=lambda column: column, + ) + + +def write_module(path: Path, text: str) -> bytes: + """Write a module file and return metadata stamping its current state.""" + + _ = path.write_text(text, encoding="utf-8") + status: os.stat_result = path.stat() + return json.dumps([[str(path), status.st_mtime_ns, status.st_size]]).encode() + + +def rewritten_module(path: Path) -> bytes: + """Stamp a module, then rewrite it with the same size and a later modification time.""" + + metadata: bytes = write_module(path, "VALUE = 1\n") + _ = path.write_text("VALUE = 2\n", encoding="utf-8") + status: os.stat_result = path.stat() + os.utime(path, ns=(status.st_atime_ns, status.st_mtime_ns + 1_000_000)) + return metadata + + +def removed_module(path: Path) -> bytes: + """Stamp a module, then remove it.""" + + metadata: bytes = write_module(path, "VALUE = 1\n") + path.unlink() + return metadata + + +def write_environment_project(root: Path) -> None: + """Write a model and a macro for store environment checks.""" + + (root / "models").mkdir() + (root / "macros").mkdir() + _ = (root / "models/orders.sql").write_text("SELECT 1\n", encoding="utf-8") + _ = (root / "macros/common.py").write_text("X = 1\n", encoding="utf-8") + + +EXCLUDED_MODEL_PATHS: list[str] = ["models/orders.sql"] diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py new file mode 100644 index 0000000000..98aebe2a09 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + module_stamps_metadata, + store_environment, + unchanged_module_stamps, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( + ModuleStampsRoundTripTestCase, + ModuleStampsTestCase, + StoreEnvironmentTestCase, + StoreEnvironmentVariableTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers.helpers import ( + EXCLUDED_MODEL_PATHS, + removed_module, + rewritten_module, + write_environment_project, + write_module, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleStampsTestCase( + description="unchanged module", + change=lambda path: write_module(path, "VALUE = 1\n"), + expected_valid=True, + ), + ModuleStampsTestCase( + description="module rewritten with the same size", + change=rewritten_module, + expected_valid=False, + ), + ModuleStampsTestCase( + description="module removed", change=removed_module, expected_valid=False + ), + ModuleStampsTestCase( + description="unreadable metadata", change=lambda _path: b"{", expected_valid=False + ), + ModuleStampsTestCase( + description="unexpected metadata shape", + change=lambda _path: b'[["flavor.py", "mtime", 1]]', + expected_valid=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_module_stamps_when_validating_then_only_unchanged_modules_pass( + tmp_path: Path, test_case: ModuleStampsTestCase +) -> None: + metadata: bytes = test_case.change(tmp_path / "flavor.py") + + stamps: dict[str, tuple[int, int]] | None = unchanged_module_stamps(metadata) + + assert (stamps is not None) is test_case.expected_valid + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleStampsRoundTripTestCase( + description="one module", module_text="VALUE = 1\n", expected_valid=True + ), + ], + ids=lambda case: case.description, +) +def test_given_module_stamps_when_encoding_then_validation_returns_them( + tmp_path: Path, test_case: ModuleStampsRoundTripTestCase +) -> None: + path: Path = tmp_path / "flavor.py" + _ = write_module(path, test_case.module_text) + stamps: dict[str, tuple[int, int]] = {str(path): (path.stat().st_mtime_ns, path.stat().st_size)} + + validated: dict[str, tuple[int, int]] | None = unchanged_module_stamps( + module_stamps_metadata(stamps) + ) + + assert (validated == stamps) is test_case.expected_valid + + +@pytest.mark.parametrize( + "test_case", + [ + StoreEnvironmentTestCase( + description="no edit", edit=lambda _root: None, expected_unchanged=True + ), + StoreEnvironmentTestCase( + description="model edit", + edit=lambda root: (root / "models/orders.sql").write_text("SELECT 2\n"), + expected_unchanged=True, + ), + StoreEnvironmentTestCase( + description="macro edit", + edit=lambda root: (root / "macros/common.py").write_text("X = 2\n"), + expected_unchanged=False, + ), + StoreEnvironmentTestCase( + description="model the caller does not exclude", + edit=lambda root: (root / "models/customers.sql").write_text("SELECT 1\n"), + expected_unchanged=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_edit_when_digesting_store_environment_then_only_model_edits_keep_it( + tmp_path: Path, test_case: StoreEnvironmentTestCase +) -> None: + write_environment_project(tmp_path) + original: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + _ = test_case.edit(tmp_path) + + edited: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + + assert (edited == original) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + StoreEnvironmentVariableTestCase( + description="variable a macro may read", + name="SQB_STORE_REGION", + value="south", + expected_unchanged=False, + ), + StoreEnvironmentVariableTestCase( + description="compiler engine selection", + name="SQLBUILD_COMPILER_ENGINE", + value="python", + expected_unchanged=True, + ), + StoreEnvironmentVariableTestCase( + description="variable outside the tracked prefixes", + name="STORE_REGION", + value="south", + expected_unchanged=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_environment_variable_change_when_digesting_store_environment_then_tracked_ones_count( + tmp_path: Path, test_case: StoreEnvironmentVariableTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + write_environment_project(tmp_path) + monkeypatch.delenv(test_case.name, raising=False) + original: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + monkeypatch.setenv(test_case.name, test_case.value) + + changed: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + + assert (changed == original) is test_case.expected_unchanged diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py new file mode 100644 index 0000000000..e32305902d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( + call_class_store_text, + context_store_token, + macro_store_token, +) +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( + CallClassStoreTextTestCase, + ContextStoreTokenTestCase, + ContextValueTestCase, + MacroStoreTokenTestCase, +) +from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers.helpers import ( + BASE_CONTEXT, + constant_declarations, + loaded_macro, + with_vars, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ContextStoreTokenTestCase( + description="equal values in separate objects", + left=with_vars(), + right=with_vars(), + expected_equal=True, + left_declarations=constant_declarations(1), + right_declarations=constant_declarations(1), + ), + ContextStoreTokenTestCase( + description="var value changed", + left=with_vars(), + right=with_vars(region="south"), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="integer and float var values", + left=with_vars(region=1), + right=with_vars(region=1.0), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="boolean and integer var values", + left=with_vars(region=1), + right=with_vars(region=True), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="var order changed", + left=with_vars(first=1, second=2), + right=replace(BASE_CONTEXT, vars={**BASE_CONTEXT.vars, "second": 2, "first": 1}), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="target changed", + left=with_vars(), + right=replace(BASE_CONTEXT, target_name="prod"), + expected_equal=False, + ), + ContextStoreTokenTestCase( + description="visible constant value changed", + left=with_vars(), + right=with_vars(), + expected_equal=False, + left_declarations=constant_declarations(1), + right_declarations=constant_declarations(2), + ), + ContextStoreTokenTestCase( + description="declarations missing on one side", + left=with_vars(), + right=with_vars(), + expected_equal=False, + left_declarations=constant_declarations(1), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_contexts_when_naming_their_store_class_then_only_equal_values_match( + test_case: ContextStoreTokenTestCase, +) -> None: + left: str | None = context_store_token( + macro_context=test_case.left, declarations=test_case.left_declarations + ) + right: str | None = context_store_token( + macro_context=test_case.right, declarations=test_case.right_declarations + ) + + assert left is not None + assert (left == right) is test_case.expected_equal + + +@pytest.mark.parametrize( + "test_case", + [ + ContextValueTestCase( + description="dates, decimals and floats", context=with_vars(), expected_stored=True + ), + ContextValueTestCase( + description="object var", context=with_vars(region=object()), expected_stored=False + ), + ContextValueTestCase( + description="nested object var", + context=with_vars(region={"key": [object()]}), + expected_stored=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_context_value_when_naming_store_class_then_only_stable_values_are_stored( + test_case: ContextValueTestCase, +) -> None: + token: str | None = context_store_token(macro_context=test_case.context, declarations=None) + + assert (token is not None) is test_case.expected_stored + + +@pytest.mark.parametrize( + "test_case", + [ + MacroStoreTokenTestCase( + description="same file and source", + relative_path=Path("macros/common.py"), + raw_source="def cents(column): ...\n", + expected_equal=True, + ), + MacroStoreTokenTestCase( + description="source edited", + relative_path=Path("macros/common.py"), + raw_source="def cents(column): return column\n", + expected_equal=False, + ), + MacroStoreTokenTestCase( + description="same source in a scoped file", + relative_path=Path("models/south/_sqlbuild/_macros/common.py"), + raw_source="def cents(column): ...\n", + expected_equal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_macro_change_when_naming_it_then_only_the_same_file_and_source_match( + test_case: MacroStoreTokenTestCase, +) -> None: + original: str = macro_store_token( + loaded=loaded_macro( + relative_path=Path("macros/common.py"), raw_source="def cents(column): ...\n" + ), + identity=None, + ) + + changed: str = macro_store_token( + loaded=loaded_macro(relative_path=test_case.relative_path, raw_source=test_case.raw_source), + identity=None, + ) + + assert (original == changed) is test_case.expected_equal + + +@pytest.mark.parametrize( + "test_case", + [ + CallClassStoreTextTestCase( + description="same class", context_token=None, prior_relations=None, expected_equal=True + ), + CallClassStoreTextTestCase( + description="no relations passed yet", + context_token=None, + prior_relations=(), + expected_equal=False, + ), + CallClassStoreTextTestCase( + description="one relation passed", + context_token=None, + prior_relations=(SqlResourceRef(kind=SqlResourceRefKind.MODEL, name="orders"),), + expected_equal=False, + ), + CallClassStoreTextTestCase( + description="context macro", + context_token="ctx", + prior_relations=None, + expected_equal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_call_class_inputs_when_naming_it_then_only_identical_inputs_match( + test_case: CallClassStoreTextTestCase, +) -> None: + plain: str = call_class_store_text( + macro_tokens=("cents",), context_token=None, prior_relations=None + ) + + text: str = call_class_store_text( + macro_tokens=("cents",), + context_token=test_case.context_token, + prior_relations=test_case.prior_relations, + ) + + assert (text == plain) is test_case.expected_equal From a55938203660156dc1622bf122c970458022b442 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:48:17 +0100 Subject: [PATCH 06/11] feat(native): run the macro bridge and call store only in the preview tier The macro bridge and the macro call store are registered as preview-tier native stages, so the default native engine and the Python engine never batch macro calls or open the store. The bindings move under the grouped bindings layout (render and cache domains). An e2e test compiles twice under each engine and checks macro executions and store files; the edit sequence also changes an @@ENV value passed to a macro. --- .../src/bindings/_helpers/cache/mod.rs | 1 + .../_helpers/cache/native_store.rs} | 10 ++- .../src/bindings/_helpers/mod.rs | 2 + .../_helpers/render/macro_call_memo.rs} | 20 +++-- .../_helpers/render/macro_call_rows.rs} | 20 +++-- .../_helpers/render/macro_calls.rs} | 24 ++--- .../src/bindings/_helpers/render/mod.rs | 3 + .../src/bindings/main/register.rs | 6 +- crates/sqlbuild-python/src/lib.rs | 2 - .../src/macro_bridge/_helpers/mod.rs | 3 - .../src/macro_bridge/constants.rs | 9 -- .../src/macro_bridge/main/mod.rs | 1 - .../src/macro_bridge/main/register.rs | 14 --- .../sqlbuild-python/src/macro_bridge/mod.rs | 5 -- .../src/macro_bridge/models.rs | 11 --- .../sqlbuild-python/src/macro_bridge/types.rs | 10 --- .../src/native_store/_helpers/mod.rs | 1 - .../src/native_store/main/mod.rs | 1 - .../src/native_store/main/register.rs | 12 --- .../sqlbuild-python/src/native_store/mod.rs | 2 - .../_helpers/macro_bridge/call_store.py | 9 +- .../frontier/_helpers/native_frontier.py | 7 +- src/sqlbuild/compiler/frontier/constants.py | 2 + src/sqlbuild/compiler/frontier/types.py | 2 + .../cli/commands/main/compile/_test_types.py | 10 +++ .../cli/commands/main/compile/helpers.py | 43 ++++++++- .../compile/test_compile_macro_call_store.py | 87 +++++++++++++++++-- .../compile/test_compiler_engine_switch.py | 2 +- .../src/sqlbuild/compiler/compile/helpers.py | 11 ++- .../compile/test_native_macro_bridge.py | 10 ++- .../frontier/test_native_stage_tiers.py | 36 ++++++++ 31 files changed, 259 insertions(+), 117 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs rename crates/sqlbuild-python/src/{native_store/_helpers/functions.rs => bindings/_helpers/cache/native_store.rs} (78%) rename crates/sqlbuild-python/src/{macro_bridge/_helpers/memo.rs => bindings/_helpers/render/macro_call_memo.rs} (85%) rename crates/sqlbuild-python/src/{macro_bridge/_helpers/rows.rs => bindings/_helpers/render/macro_call_rows.rs} (69%) rename crates/sqlbuild-python/src/{macro_bridge/_helpers/functions.rs => bindings/_helpers/render/macro_calls.rs} (67%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/constants.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/main/mod.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/main/register.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/mod.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/models.rs delete mode 100644 crates/sqlbuild-python/src/macro_bridge/types.rs delete mode 100644 crates/sqlbuild-python/src/native_store/_helpers/mod.rs delete mode 100644 crates/sqlbuild-python/src/native_store/main/mod.rs delete mode 100644 crates/sqlbuild-python/src/native_store/main/register.rs delete mode 100644 crates/sqlbuild-python/src/native_store/mod.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs new file mode 100644 index 0000000000..31c5e16613 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs @@ -0,0 +1 @@ +pub(crate) mod native_store; diff --git a/crates/sqlbuild-python/src/native_store/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs similarity index 78% rename from crates/sqlbuild-python/src/native_store/_helpers/functions.rs rename to crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs index f70d170347..f28b85d95b 100644 --- a/crates/sqlbuild-python/src/native_store/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -1,8 +1,8 @@ //! Digests and project fingerprints of the shared native store, with the GIL released. use pyo3::exceptions::{PyOSError, PyRuntimeError}; -use pyo3::prelude::{PyResult, Python}; -use pyo3::pyfunction; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::content_digest::content_digest as digest; use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; use sqlbuild_cache::digest::main::hex_digest::hex_digest; @@ -32,3 +32,9 @@ pub(crate) fn fingerprint_project_files( .map_err(|error| PyOSError::new_err(error.to_string()))?; Ok(hex_digest(&fingerprinted)) } + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(content_digest, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e5d5e03f58..7863a42fdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,7 +1,9 @@ pub(crate) mod analysis; pub(crate) mod boundary; +pub(crate) mod cache; pub(crate) mod discovery; pub(crate) mod model_config; +pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; pub(crate) mod sqltext; diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs similarity index 85% rename from crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs rename to crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index edc20a5d00..718dcff75f 100644 --- a/crates/sqlbuild-python/src/macro_bridge/_helpers/memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -1,19 +1,27 @@ -//! Python methods of the in-compile macro call memo. +//! The macro call memo class: one compile's recorded calls and its optional native store. use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; use pyo3::prelude::Python; -use pyo3::{PyResult, pymethods}; +use pyo3::{PyResult, pyclass, pymethods}; use sqlbuild_cache::store::main::open_native_store::open_native_store; use sqlbuild_cache::store::models::NativeStore; use sqlbuild_render::macro_calls::models::MacroCallEntry; use std::path::PathBuf; +use std::sync::Mutex; use crate::bindings::types::CompilerDetach; -use crate::macro_bridge::constants::MACRO_CALL_STORE_KIND; -use crate::macro_bridge::_helpers::rows::{event_from_row, event_row}; -use crate::macro_bridge::models::MacroCallMemo; -use crate::macro_bridge::types::EntryRow; +use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row}; + +/// The native store kind holding recorded macro calls. +const MACRO_CALL_STORE_KIND: &str = "macro-calls"; + +/// The macro call results one compile records and replays, keyed by call class and call text. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug, Default)] +pub(crate) struct MacroCallMemo { + pub(crate) inner: Mutex, +} fn poisoned(_: T) -> pyo3::PyErr { PyValueError::new_err("macro call memo is poisoned") diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs similarity index 69% rename from crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs rename to crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs index cc12477e93..78ccae8219 100644 --- a/crates/sqlbuild-python/src/macro_bridge/_helpers/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -1,13 +1,23 @@ -//! Conversions between native macro bridge models and their Python rows. +//! Python rows of macro call sites, splice spans and recorded call events. use pyo3::PyResult; use pyo3::exceptions::PyValueError; use sqlbuild_render::macro_calls::models::{MacroCallEvent, MacroCallSite, MacroSpliceSpan}; -use crate::macro_bridge::constants::{ - ARGUMENT_REFERENCE, DECLARATION_READ, GENERATED_SQL, MACRO_USE, -}; -use crate::macro_bridge::types::{EventRow, SiteRow, SpanRow}; +/// Event tags shared with `sqlbuild.compiler.macro_bridge.constants`. +const MACRO_USE: u8 = 0; +const DECLARATION_READ: u8 = 1; +const GENERATED_SQL: u8 = 2; +const ARGUMENT_REFERENCE: u8 = 3; + +/// One call site: start, end, name, tree names and whether typed reference text appears. +pub(crate) type SiteRow = (usize, usize, String, Vec, bool); +/// One substitution span: source start and end, output start and end. +pub(crate) type SpanRow = (usize, usize, usize, usize); +/// One recorded event: its tag and two text fields. +pub(crate) type EventRow = (u8, String, String); +/// One recorded call: its SQL, added relations and events. +pub(crate) type EntryRow = (String, Vec<(String, String)>, Vec); pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { ( diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs similarity index 67% rename from crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs rename to crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index bcfd897d80..1d4677fa4a 100644 --- a/crates/sqlbuild-python/src/macro_bridge/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -1,20 +1,17 @@ -//! Call-site scanning and splicing for the Python macro bridge, with the GIL released. +//! Macro call-site scanning and splicing for the Python macro bridge, with the GIL released. -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{PyResult, Python}; -use pyo3::pyfunction; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; use sqlbuild_render::macro_calls::models::ScanDeferral; +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::render::macro_call_memo::MacroCallMemo; +use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_row, span_row}; use crate::bindings::types::CompilerDetach; -use crate::macro_bridge::_helpers::rows::{site_row, span_row}; -use crate::macro_bridge::types::{SiteRow, SpanRow}; - -fn compiler_error(message: String) -> pyo3::PyErr { - PyRuntimeError::new_err(message) -} /// Every top-level macro call site of `sql`, or `None` when Python must expand the string. #[pyfunction] @@ -51,3 +48,10 @@ pub(crate) fn splice_macro_calls( spliced.ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; Ok((rendered, spans.into_iter().map(span_row).collect())) } + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; + module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs new file mode 100644 index 0000000000..2a04cacf7b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod macro_call_memo; +pub(crate) mod macro_call_rows; +pub(crate) mod macro_calls; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 3afd2c87d1..64ff38a3ba 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,7 +4,7 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, boundary, discovery, model_config, rules, scopes, sqltext, + analysis, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, }; use crate::bindings::models; @@ -29,8 +29,8 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; - crate::macro_bridge::main::register::register(module)?; - crate::native_store::main::register::register(module)?; + render::macro_calls::register(module)?; + cache::native_store::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/lib.rs b/crates/sqlbuild-python/src/lib.rs index 85ec55e0cc..7c2526111e 100644 --- a/crates/sqlbuild-python/src/lib.rs +++ b/crates/sqlbuild-python/src/lib.rs @@ -1,5 +1,3 @@ #![forbid(unsafe_code)] pub mod bindings; -mod macro_bridge; -mod native_store; diff --git a/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs b/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs deleted file mode 100644 index bbdf7c8b94..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod functions; -pub(crate) mod memo; -pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/macro_bridge/constants.rs b/crates/sqlbuild-python/src/macro_bridge/constants.rs deleted file mode 100644 index 7baaec13a2..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/constants.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Macro call event tags shared with `sqlbuild.compiler.macro_bridge.constants`. - -pub(crate) const MACRO_USE: u8 = 0; -pub(crate) const DECLARATION_READ: u8 = 1; -pub(crate) const GENERATED_SQL: u8 = 2; -pub(crate) const ARGUMENT_REFERENCE: u8 = 3; - -/// The native store kind holding recorded macro calls. -pub(crate) const MACRO_CALL_STORE_KIND: &str = "macro-calls"; diff --git a/crates/sqlbuild-python/src/macro_bridge/main/mod.rs b/crates/sqlbuild-python/src/macro_bridge/main/mod.rs deleted file mode 100644 index 51101fd370..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod register; diff --git a/crates/sqlbuild-python/src/macro_bridge/main/register.rs b/crates/sqlbuild-python/src/macro_bridge/main/register.rs deleted file mode 100644 index 06cf3f3316..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/main/register.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Register the macro bridge's functions and classes on the extension module. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; - -use crate::macro_bridge::_helpers::functions::{scan_macro_call_sites, splice_macro_calls}; -use crate::macro_bridge::models::MacroCallMemo; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; - module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/macro_bridge/mod.rs b/crates/sqlbuild-python/src/macro_bridge/mod.rs deleted file mode 100644 index c376860cb1..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod constants; -pub(crate) mod main; -pub(crate) mod models; -pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/macro_bridge/models.rs b/crates/sqlbuild-python/src/macro_bridge/models.rs deleted file mode 100644 index 0287377097..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/models.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Python classes of the macro bridge. - -use pyo3::pyclass; -use std::sync::Mutex; - -/// The macro call results one compile records and replays, keyed by call class and call text. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug, Default)] -pub(crate) struct MacroCallMemo { - pub(crate) inner: Mutex, -} diff --git a/crates/sqlbuild-python/src/macro_bridge/types.rs b/crates/sqlbuild-python/src/macro_bridge/types.rs deleted file mode 100644 index c8339393ae..0000000000 --- a/crates/sqlbuild-python/src/macro_bridge/types.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Python-facing rows of the macro bridge. - -/// One call site: start, end, name, tree names and whether typed reference text appears. -pub(crate) type SiteRow = (usize, usize, String, Vec, bool); -/// One substitution span: source start and end, output start and end. -pub(crate) type SpanRow = (usize, usize, usize, usize); -/// One recorded event: its tag and two text fields. -pub(crate) type EventRow = (u8, String, String); -/// One recorded call: its SQL, added relations and events. -pub(crate) type EntryRow = (String, Vec<(String, String)>, Vec); diff --git a/crates/sqlbuild-python/src/native_store/_helpers/mod.rs b/crates/sqlbuild-python/src/native_store/_helpers/mod.rs deleted file mode 100644 index 8f03d97c29..0000000000 --- a/crates/sqlbuild-python/src/native_store/_helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/native_store/main/mod.rs b/crates/sqlbuild-python/src/native_store/main/mod.rs deleted file mode 100644 index 51101fd370..0000000000 --- a/crates/sqlbuild-python/src/native_store/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod register; diff --git a/crates/sqlbuild-python/src/native_store/main/register.rs b/crates/sqlbuild-python/src/native_store/main/register.rs deleted file mode 100644 index 518ccf8091..0000000000 --- a/crates/sqlbuild-python/src/native_store/main/register.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Register the shared native store's functions on the extension module. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; - -use crate::native_store::_helpers::functions::{content_digest, fingerprint_project_files}; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(content_digest, module)?)?; - module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/native_store/mod.rs b/crates/sqlbuild-python/src/native_store/mod.rs deleted file mode 100644 index 7de28dc3ff..0000000000 --- a/crates/sqlbuild-python/src/native_store/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index de321d0c39..faf6a00985 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -6,6 +6,8 @@ from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.spec.contracts.models import TargetConfig @@ -29,7 +31,12 @@ def target_context_with_macro_call_store( bridge: MacroBridge | None = active_macro_bridge() project_dir: Path | None = discovered_inputs.project_dir compile_cache_dir: Path | None = target_context[2] - if bridge is not None and compile_cache_dir is not None and project_dir is not None: + if ( + bridge is not None + and compile_cache_dir is not None + and project_dir is not None + and native_stage_enabled(NativeStage.MACRO_CALL_STORE) + ): bridge.attach_store( cache_dir=compile_cache_dir, project_dir=project_dir, diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py index 1505be9c60..031f90ca47 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -2,7 +2,8 @@ from collections.abc import Callable -from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import CompilerStage, NativeStage from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge @@ -16,6 +17,8 @@ def native_frontier[T]( if native_stage is not None: return native_stage() - if until is CompilerStage.COMPILE_PROJECT_INPUTS: + if until is CompilerStage.COMPILE_PROJECT_INPUTS and native_stage_enabled( + NativeStage.MACRO_CALLS + ): return run_with_macro_bridge(stage=python_stage) return python_stage() diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 83909b5197..b10c7aab0c 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -22,6 +22,8 @@ NativeStage.REFERENCE_EXTRACTION: NativeStageTier.PREVIEW, NativeStage.DECLARATION_FILES: NativeStageTier.PREVIEW, NativeStage.MODEL_LOOP: NativeStageTier.PREVIEW, + NativeStage.MACRO_CALLS: NativeStageTier.PREVIEW, + NativeStage.MACRO_CALL_STORE: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 82a422b4d2..b1dd2339c0 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -20,6 +20,8 @@ class NativeStage(StrEnum): REFERENCE_EXTRACTION = "reference_extraction" DECLARATION_FILES = "declaration_files" MODEL_LOOP = "model_loop" + MACRO_CALLS = "macro_calls" + MACRO_CALL_STORE = "macro_call_store" class NativeStageTier(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 2b1063bd41..e107c1e2fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -924,3 +924,13 @@ class BrokenMacroCallStoreKeyTestCase: description: str edit: Callable[[Path], None] expected_matches_uncached: bool + + +@dataclass(frozen=True) +class EngineMacroCallGateTestCase: + """Repeated full compiles under one engine and whether the bridge and store took part.""" + + description: str + engine: str + expected_logged_calls: tuple[int, ...] + expected_store_files: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index c4e5341e59..4b4c80e8b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -57,6 +57,7 @@ from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, IncrementalEditStep, @@ -4270,8 +4271,9 @@ def prepare_reference_call_project(*, tmp_path: Path, staging_from: str, mart_fr return project_dir -MACRO_CALL_STORE_ENGINE: str = "native" +MACRO_CALL_STORE_ENGINE: str = "native-preview" STORE_ENVIRONMENT_REGION_VAR: str = "SQB_STORE_TEST_REGION" +STORE_ARGUMENT_ENV_VAR: str = "STORE_TEST_ARGUMENT" _REUSE_DISABLED_VALUES: dict[bool, str] = {True: "0", False: "1"} _SOUTH_MODEL: str = "models/south/orders_south.sql" _STORE_EXTERNAL_MODULE: str = "store_flavor_values" @@ -4382,6 +4384,7 @@ def macro_call_store_compile( MACRO_CALL_LOG_ENV_VAR: str(log_path), "PYTHONPATH": str(extlib), "PYTHONDONTWRITEBYTECODE": "1", + STORE_ARGUMENT_ENV_VAR: "first", **dict(extra_env), } global_args: tuple[str, ...] = ("--compiler-engine", MACRO_CALL_STORE_ENGINE) @@ -4403,3 +4406,41 @@ def freeze_macro_call_store_environment(monkeypatch: pytest.MonkeyPatch) -> None monkeypatch.setattr(macro_bridge_class, "store_environment", lambda **_kwargs: "frozen") monkeypatch.setattr(macro_bridge_class, "unchanged_module_stamps", lambda _metadata: {}) + + +class EngineMacroCallRuns(NamedTuple): + """Full compiles under one engine: exit codes, macro executions, and macro call stores.""" + + returncodes: tuple[int, ...] + logged_calls: tuple[int, ...] + store_files: tuple[str, ...] + + +def engine_macro_call_runs( + *, project_dir: Path, extlib: Path, log_path: Path, engine: str, runs: int +) -> EngineMacroCallRuns: + """Compile fully `runs` times under an engine selected by environment; empty is the default.""" + + env: dict[str, str] = { + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + MACRO_CALL_LOG_ENV_VAR: str(log_path), + "PYTHONPATH": str(extlib), + "PYTHONDONTWRITEBYTECODE": "1", + } + returncodes: list[int] = [] + logged_calls: list[int] = [] + for _ in range(runs): + _ = log_path.write_text("", encoding="utf-8") + returncodes.append(run_reuse_compile(project_dir=project_dir, env=env).returncode) + logged_calls.append(len(log_path.read_text(encoding="utf-8").splitlines())) + return EngineMacroCallRuns( + returncodes=tuple(returncodes), + logged_calls=tuple(logged_calls), + store_files=tuple( + sorted( + path.relative_to(project_dir).as_posix() + for path in (project_dir / "target").rglob(MACRO_CALL_STORE_FILE_NAME) + ) + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 1ce542bb33..56ca19af5f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -8,16 +8,20 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( BrokenMacroCallStoreKeyTestCase, + EngineMacroCallGateTestCase, MacroCallStoreEditSequenceTestCase, MacroCallStoreEditStep, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( MACRO_CALL_STORE_ENGINE, + STORE_ARGUMENT_ENV_VAR, STORE_ENVIRONMENT_REGION_VAR, CompileReuseRun, + EngineMacroCallRuns, MacroCallStoreRun, edit_label_helper, edit_south_model, + engine_macro_call_runs, freeze_macro_call_store_environment, in_process_reuse_run, macro_call_store_compile, @@ -64,10 +68,25 @@ expected_logged_calls=0, args=("--target", "prod"), ), + MacroCallStoreEditStep( + description="model_adds_env_interpolated_argument", + edit=lambda root: edit_south_model( + root, + " @target_label()", + f' @logged("@@ENV:{STORE_ARGUMENT_ENV_VAR}") AS logged_env,\n @target_label()', + ), + expected_logged_calls=1, + ), + MacroCallStoreEditStep( + description="env_interpolated_argument_changes_call_text", + edit=lambda _root: None, + expected_logged_calls=1, + env=((STORE_ARGUMENT_ENV_VAR, "second"),), + ), MacroCallStoreEditStep( description="environment_variable_read_by_macro", edit=lambda _root: None, - expected_logged_calls=2, + expected_logged_calls=3, env=((STORE_ENVIRONMENT_REGION_VAR, "south"),), ), MacroCallStoreEditStep( @@ -75,33 +94,33 @@ edit=lambda root: replace_project_text( root, "sqlbuild_project.toml", 'region = "north"', 'region = "east"' ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="constant_file_edit", edit=lambda root: replace_project_text( root, "constants/base_rate.sql", "value 1)", "value 5)" ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="enum_edit", edit=lambda root: replace_project_text( root, "enums/order_status.sql", 'PLACED "placed"', 'PLACED "open"' ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="macro_body_edit", edit=lambda root: replace_project_text( root, "macros/common.py", 'return f"{column} * 100"', 'return f"{column} * 1000"' ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="helper_module_edit", edit=lambda root: edit_label_helper(root, "other"), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="macro_file_added", @@ -113,7 +132,7 @@ ), edit_south_model(root, " @label() AS label,\n", " @badge() AS badge,\n"), ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="macro_file_removed", @@ -121,12 +140,12 @@ edit_south_model(root, " @badge() AS badge,\n", ""), (root / "macros/badges.py").unlink(), ), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="outside_module_rewritten_in_place", edit=lambda root: write_store_flavor(extlib=root.parent / "extlib", value="salty"), - expected_logged_calls=2, + expected_logged_calls=3, ), MacroCallStoreEditStep( description="model_comment_after_every_edit", @@ -217,3 +236,53 @@ def test_given_broken_store_key_when_compiling_an_edit_then_the_oracle_reports_a assert (broken.returncode, reference.returncode) == (0, 0) assert (broken.compiled == reference.compiled) is test_case.expected_matches_uncached + + +@pytest.mark.parametrize( + "test_case", + [ + EngineMacroCallGateTestCase( + description="default_engine", + engine="", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="python", + engine="python", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="native", + engine="native", + expected_logged_calls=(3, 3), + expected_store_files=(), + ), + EngineMacroCallGateTestCase( + description="native_preview", + engine="native-preview", + expected_logged_calls=(1, 0), + expected_store_files=("target/cache/compiler-native-preview-v1/macro-calls.bin",), + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_compiling_repeatedly_then_only_preview_batches_and_stores_macro_calls( + tmp_path: Path, test_case: EngineMacroCallGateTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + + runs: EngineMacroCallRuns = engine_macro_call_runs( + project_dir=project_dir, + extlib=extlib, + log_path=tmp_path / "macro-calls.log", + engine=test_case.engine, + runs=2, + ) + + assert runs.returncodes == (0, 0) + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.store_files == test_case.expected_store_files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py index dcabbd8a75..18eef94924 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -121,7 +121,7 @@ def test_given_macro_heavy_project_when_compiling_with_each_engine_then_outputs_ ) native_run: CompileReuseRun = engine_reuse_compile( project_dir=write_project(root=tmp_path / "native", files=test_case.files), - engine="native", + engine="native-preview", ) compiled_text: str = b"".join(python_run.compiled.values()).decode() diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 36e6be4a4a..fe10c48a49 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -25,7 +25,11 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_INVOCATION_ID_MASK, + STAGE_CAPTURE_INVOCATION_ID_PATTERN, +) from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge @@ -254,7 +258,10 @@ def render_compile_inputs( def comparable_capture(inputs: CompileProjectInputs) -> str: """Render compile inputs as canonical JSON without the per-run identifier.""" - return _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)) + return STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub( + STAGE_CAPTURE_INVOCATION_ID_MASK, + _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)), + ) def render_error( diff --git a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py index fa60262c11..2650ab34a3 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py +++ b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py @@ -78,7 +78,9 @@ def test_given_macro_heavy_project_when_rendering_with_each_engine_then_inputs_a render_compile_inputs(project_dir=project_dir, engine="python", monkeypatch=monkeypatch) ) native_capture: str = comparable_capture( - render_compile_inputs(project_dir=project_dir, engine="native", monkeypatch=monkeypatch) + render_compile_inputs( + project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch + ) ) assert native_capture == python_capture @@ -102,7 +104,7 @@ def test_given_repeated_macro_calls_when_rendering_natively_then_each_call_class ) -> None: project_dir: Path = write_project(root=tmp_path / "project", files=test_case.files) executions: dict[str, int] = {} - for engine in ("python", "native"): + for engine in ("python", "native-preview"): log_path: Path = tmp_path / f"{engine}.log" monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(log_path)) _ = render_compile_inputs(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) @@ -110,7 +112,7 @@ def test_given_repeated_macro_calls_when_rendering_natively_then_each_call_class assert executions == { "python": test_case.expected_python_executions, - "native": test_case.expected_native_executions, + "native-preview": test_case.expected_native_executions, } @@ -171,7 +173,7 @@ def test_given_failing_render_when_rendering_natively_then_raises_the_python_err project_dir=project_dir, engine="python", monkeypatch=monkeypatch ) native_error: tuple[type[BaseException], str, type[object]] = render_error( - project_dir=project_dir, engine="native", monkeypatch=monkeypatch + project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch ) assert native_error == python_error diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 3f004a9f97..02134d10f3 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -103,6 +103,42 @@ stage=NativeStage.MODEL_LOOP, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_macro_calls", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MACRO_CALLS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="python_macro_call_store", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_macro_calls", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MACRO_CALLS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_macro_call_store", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_macro_calls", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MACRO_CALLS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="native_preview_macro_call_store", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MACRO_CALL_STORE, + expected_enabled=True, + ), ], ids=lambda case: case.description, ) From f8d8e5321189ebf04074d291f7f1a05bb9c0852a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:48:17 +0100 Subject: [PATCH 07/11] fix(native): scan nested macro calls in one pass without recursion Collecting the names of nested macro calls recursed once per nesting level and rescanned the arguments at each level, so deep nesting was quadratic and very deep nesting overflowed the native stack. One pass with an explicit stack now covers any depth in linear time. --- .../src/macro_calls/_helpers/scanner.rs | 63 ++++++++++++---- .../tests/test_scan_macro_call_sites.rs | 71 ++++++++++++++++++- .../src/macro_calls/tests/test_types.rs | 7 ++ .../compiler/macro_bridge/classes/__init__.py | 0 .../macro_bridge/classes/_test_types.py | 13 ++++ .../macro_bridge/classes/test_macro_bridge.py | 53 ++++++++++++++ 6 files changed, 194 insertions(+), 13 deletions(-) create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/classes/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs index 49ed169cb8..71cef2c882 100644 --- a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -34,15 +34,45 @@ pub(crate) fn top_level_calls( Ok(calls) } -/// Return the names of every call nested in `args`, depth first, with repeats. +/// Names of every call nested in `args` in source order, in one pass without recursion. pub(crate) fn nested_names(python: PythonText, args: &str) -> Result, ScanDeferral> { + let bytes: &[u8] = args.as_bytes(); let mut names: Vec = Vec::new(); - for call in top_level_calls(python, args)? { - let inner = nested_names(python, &args[call.open + 1..call.close])?; - names.push(call.name); - names.extend(inner); + let mut open_call_depths: Vec = Vec::new(); + let mut depth: usize = 0; + let mut index: usize = 0; + while index < bytes.len() { + match bytes[index] { + b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, + b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), + b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), + b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, + b'@' if macro_call_starts_at(python, args, index)? => { + let (name, open) = call_header(python, args, index)?; + names.push(name); + depth += 1; + open_call_depths.push(depth); + index = open + 1; + } + b'(' => { + depth += 1; + index += 1; + } + b')' => { + if open_call_depths.last() == Some(&depth) { + let _ = open_call_depths.pop(); + } + depth = depth.checked_sub(1).ok_or(ScanDeferral)?; + index += 1; + } + _ => index += 1, + } + } + if open_call_depths.is_empty() { + Ok(names) + } else { + Err(ScanDeferral) } - Ok(names) } /// Whether call arguments mention a typed reference function name anywhere. @@ -51,20 +81,29 @@ pub(crate) fn mentions_typed_reference(args: &str) -> bool { } fn call_at(python: PythonText, text: &str, start: usize) -> Result { - let name_end = identifier_end(python, text, start + 1); - let open = skip_whitespace(text, name_end); - if text.as_bytes().get(open) != Some(&b'(') { - return Err(ScanDeferral); - } + let (name, open) = call_header(python, text, start)?; let close = matching_paren(text.as_bytes(), open)?; Ok(ScannedCall { start, open, close, - name: text[start + 1..name_end].to_owned(), + name, }) } +fn call_header( + python: PythonText, + text: &str, + start: usize, +) -> Result<(String, usize), ScanDeferral> { + let name_end = identifier_end(python, text, start + 1); + let open = skip_whitespace(text, name_end); + if text.as_bytes().get(open) != Some(&b'(') { + return Err(ScanDeferral); + } + Ok((text[start + 1..name_end].to_owned(), open)) +} + fn next_macro_start( python: PythonText, text: &str, diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs index b77c6b4632..0ee44440e9 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -1,6 +1,10 @@ use crate::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites; use crate::macro_calls::tests::helpers::python_312; -use crate::macro_calls::tests::test_types::{ExpectedSite, ScanMacroCallSitesTestCase}; +use std::time::Instant; + +use crate::macro_calls::tests::test_types::{ + DeepNestingTestCase, ExpectedSite, ScanMacroCallSitesTestCase, +}; #[test] fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansion() { @@ -74,6 +78,21 @@ fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansio sql: "x @@var @m()", expected_sites: Some(&[(8, 12, "m", &["m"], false)]), }, + ScanMacroCallSitesTestCase { + description: "nested names follow source order across siblings and plain parentheses", + sql: "@a((@b(@c(1)), (@d()) ), \"@x()\", @c())", + expected_sites: Some(&[(0, 38, "a", &["a", "b", "c", "d"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a quoted parenthesis inside a nested call does not close it", + sql: "@a(@b(')'))", + expected_sites: Some(&[(0, 11, "a", &["a", "b"], false)]), + }, + ScanMacroCallSitesTestCase { + description: "a nested name without its parenthesis defers", + sql: "@a(@bb c(1))", + expected_sites: None, + }, ScanMacroCallSitesTestCase { description: "parentheses inside quoted arguments do not close the call", sql: "@m(')', \"(\", `)`) z", @@ -111,3 +130,53 @@ fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansio ); } } + +#[test] +fn given_deeply_nested_calls_when_scanning_then_time_is_linear_and_nothing_overflows() { + let test_cases = [ + DeepNestingTestCase { + description: "one thousand levels", + depth: 1_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + DeepNestingTestCase { + description: "twenty thousand levels", + depth: 20_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + DeepNestingTestCase { + description: "one hundred thousand levels", + depth: 100_000, + expected_tree_names: &["m"], + expected_max_seconds: 2.0, + }, + ]; + + for test_case in test_cases { + let sql = format!( + "{}x{}", + "@m(".repeat(test_case.depth), + ")".repeat(test_case.depth) + ); + let started = Instant::now(); + + let sites = scan_macro_call_sites(python_312(), &sql).expect("scanned"); + + assert!( + started.elapsed().as_secs_f64() < test_case.expected_max_seconds, + "{}", + test_case.description + ); + assert_eq!( + sites + .iter() + .map(|site| site.tree_names.clone()) + .collect::>(), + vec![test_case.expected_tree_names.to_vec()], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs index 39af8176e4..b58c34f5a4 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs @@ -32,3 +32,10 @@ pub(crate) struct MacroCallStoreTestCase { pub(crate) call_text: &'static str, pub(crate) expected_store_hit: bool, } + +pub(crate) struct DeepNestingTestCase { + pub(crate) description: &'static str, + pub(crate) depth: usize, + pub(crate) expected_tree_names: &'static [&'static str], + pub(crate) expected_max_seconds: f64, +} diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py new file mode 100644 index 0000000000..33c8dbcefc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/_test_types.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DeepNestingScanTestCase: + """Macro calls nested `depth` levels deep and the scan the bridge must return quickly.""" + + description: str + depth: int + expected_tree_names: tuple[tuple[str, ...], ...] + expected_max_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py new file mode 100644 index 0000000000..758149c91a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sys +import time +import unicodedata + +import pytest + +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.models import MacroCallSite +from tests.unit.src.sqlbuild.compiler.macro_bridge.classes._test_types import ( + DeepNestingScanTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeepNestingScanTestCase( + description="one thousand levels", + depth=1_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + DeepNestingScanTestCase( + description="twenty thousand levels", + depth=20_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + DeepNestingScanTestCase( + description="one hundred thousand levels", + depth=100_000, + expected_tree_names=(("m",),), + expected_max_seconds=2.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_deeply_nested_calls_when_scanning_then_the_bridge_returns_promptly( + test_case: DeepNestingScanTestCase, +) -> None: + bridge: MacroBridge = MacroBridge( + python_version=(sys.version_info[0], sys.version_info[1]), + unicode_version=unicodedata.unidata_version, + ) + sql: str = "@m(" * test_case.depth + "x" + ")" * test_case.depth + started: float = time.perf_counter() + + sites: tuple[MacroCallSite, ...] | None = bridge.scan(sql) + + assert time.perf_counter() - started < test_case.expected_max_seconds + assert tuple(site.tree_names for site in sites or ()) == test_case.expected_tree_names From 2fedbe404815b533097ec4e55019bd9d6c86236c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:48:17 +0100 Subject: [PATCH 08/11] docs: state that macros must be deterministic Compile reuse, the per-compile macro call memo and the macro call cache all reuse a macro's result for the same call and context. Document the contract, which inputs are compared, and that environment variables read directly are compared only for SQLBUILD_ and SQB_ names. --- .../sqlbuild/references/docs/cli/compile.md | 4 +++- .../references/docs/concepts/macros.md | 23 +++++++++++++++++++ website/src/content/docs/docs/cli/compile.mdx | 4 +++- .../src/content/docs/docs/concepts/macros.mdx | 22 ++++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index e28a528a3f..c2e24834a3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -109,7 +109,9 @@ anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.id files count only by presence. Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs -must produce the same output. A compile whose templates read `run.id` is never reused. +must produce the same output. Macro results are also reused within a compile and may be kept between +compiles; see [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). A +compile whose templates read `run.id` is never reused. Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when `SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 7c28c84d6a..2902c08b5f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/ - Keep implementation details private - Use macros in tests - Use macros in hooks +- Macros must be deterministic - Where macros work - Next steps @@ -213,6 +214,28 @@ SELECT 1 AS id See [SQL Hooks](models/hooks/sql.md) for hook lifecycle and context syntax. +## Macros must be deterministic + +A macro must return the same result whenever it is called with the same arguments and sees the same +context. Avoid module-level state such as counters, random values, the current time, and reads of +files or environment variables that the call does not receive. + +SQLBuild relies on this contract to reuse macro results: + +- [Compile reuse](../cli/compile.md) replays stored renders without running macros again. +- A compile may run each distinct macro call once and reuse its result for identical calls + elsewhere in the project. +- Macro call results may be kept between compiles under `target/cache/`. They are recomputed when + macro or other Python files, configuration, declarations, seeds, sources, variables, the target, + the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps + them, and a changed call is always run again. + +Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and +`@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, +and visible constants and enums. Environment variables a macro reads directly, for example with +`os.environ`, are compared only when their names start with `SQLBUILD_` or `SQB_`, the same set +compile reuse tracks. A macro whose result changes for any other reason may return a stale result. + ## Where macros work Macros are supported in: diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 6c24b05777..b07bbd8c1c 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -100,7 +100,9 @@ anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.id files count only by presence. Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs -must produce the same output. A compile whose templates read `run.id` is never reused. +must produce the same output. Macro results are also reused within a compile and may be kept between +compiles; see [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). A +compile whose templates read `run.id` is never reused. Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when `SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index b6f389f70d..9c1bba7045 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -204,6 +204,28 @@ SELECT 1 AS id See [SQL Hooks](/docs/concepts/models/hooks/sql) for hook lifecycle and context syntax. +## Macros must be deterministic + +A macro must return the same result whenever it is called with the same arguments and sees the same +context. Avoid module-level state such as counters, random values, the current time, and reads of +files or environment variables that the call does not receive. + +SQLBuild relies on this contract to reuse macro results: + +- [Compile reuse](/docs/cli/compile) replays stored renders without running macros again. +- A compile may run each distinct macro call once and reuse its result for identical calls + elsewhere in the project. +- Macro call results may be kept between compiles under `target/cache/`. They are recomputed when + macro or other Python files, configuration, declarations, seeds, sources, variables, the target, + the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps + them, and a changed call is always run again. + +Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and +`@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, +and visible constants and enums. Environment variables a macro reads directly, for example with +`os.environ`, are compared only when their names start with `SQLBUILD_` or `SQB_`, the same set +compile reuse tracks. A macro whose result changes for any other reason may return a stale result. + ## Where macros work Macros are supported in: From b60f8f644e68e8ce293ab2aa0724b6a80fd03642 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:43:04 +0100 Subject: [PATCH 09/11] fix(native): trust stored macro calls only for code identified by content Module stamps were read when the store was saved, so a module imported before its file changed could be stored under the new file's stamps and replayed by later processes; unreadable modules (zip members, moved files) were skipped, and timestamps can be preserved or coarse. Only the first compile of a process now uses and saves the store, so code loaded by earlier compiles can no longer differ from disk. Loaded modules outside the interpreter and SQLBuild are identified by a BLAKE3 digest of their backing file, at attach, after recorded calls and at save; a module whose code cannot be read from a plain file disables the store for that compile. Installed compiled extensions are covered by their distributions' RECORD files, and the project fingerprint must be unchanged between attach and save. The process start time is no longer read, so the store also works without /proc. --- Cargo.lock | 38 +++- Cargo.toml | 1 + crates/sqlbuild-cache/Cargo.toml | 1 + .../src/digest/main/digest_files.rs | 18 ++ crates/sqlbuild-cache/src/digest/main/mod.rs | 1 + crates/sqlbuild-cache/src/digest/tests.rs | 2 + .../src/digest/tests/test_digest_files.rs | 55 ++++++ .../src/digest/tests/test_types.rs | 6 + .../bindings/_helpers/cache/native_store.rs | 18 ++ .../references/docs/concepts/macros.md | 5 +- src/sqlbuild/_native.pyi | 1 + .../_helpers/macro_bridge/call_store.py | 7 +- .../_helpers/store_environment.py | 187 ++++++++++++++---- .../macro_bridge/classes/macro_bridge.py | 87 ++++++-- .../compiler/macro_bridge/constants.py | 10 +- .../macro_bridge/main/_claim_first_compile.py | 11 ++ src/sqlbuild/compiler/macro_bridge/models.py | 8 + src/sqlbuild/compiler/macro_bridge/types.py | 1 - .../cli/commands/main/compile/_test_types.py | 31 +++ .../cli/commands/main/compile/helpers.py | 176 ++++++++++++++++- .../compile/test_compile_macro_call_store.py | 156 +++++++++++++++ .../macro_bridge/_helpers/_test_types.py | 19 +- .../compiler/macro_bridge/_helpers/helpers.py | 97 ++++++++- .../_helpers/test_store_environment.py | 147 +++++++++++--- .../src/content/docs/docs/concepts/macros.mdx | 5 +- 25 files changed, 975 insertions(+), 113 deletions(-) create mode 100644 crates/sqlbuild-cache/src/digest/main/digest_files.rs create mode 100644 crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs create mode 100644 src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py diff --git a/Cargo.lock b/Cargo.lock index 6f292fc245..03a1b44287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "autocfg" version = "1.5.1" @@ -38,6 +44,20 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", + "rayon-core", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -83,6 +103,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -92,6 +118,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -646,7 +681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -675,6 +710,7 @@ dependencies = [ name = "sqlbuild-cache" version = "0.139.0" dependencies = [ + "blake3", "rayon", "sha2", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 8dd0df853d..2f610cd372 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0a polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0af282829d92efeeb36b6a820be2271e1da49", default-features = false, features = [ "dialect-duckdb", ] } +blake3 = { version = "1", features = ["rayon"] } quote = "1" rayon = "1" saphyr-parser = "0.1" diff --git a/crates/sqlbuild-cache/Cargo.toml b/crates/sqlbuild-cache/Cargo.toml index 6ffbf21d5c..c24885ac4b 100644 --- a/crates/sqlbuild-cache/Cargo.toml +++ b/crates/sqlbuild-cache/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +blake3.workspace = true rayon.workspace = true sha2.workspace = true tempfile.workspace = true diff --git a/crates/sqlbuild-cache/src/digest/main/digest_files.rs b/crates/sqlbuild-cache/src/digest/main/digest_files.rs new file mode 100644 index 0000000000..a659f8c0b0 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/digest_files.rs @@ -0,0 +1,18 @@ +//! BLAKE3 content digests of individual files, read and hashed in parallel. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use std::path::PathBuf; + +use crate::digest::types::ContentDigest; + +/// The digest of each file in `paths`; BLAKE3 keeps large compiled modules cheap to hash. +pub fn digest_files(paths: &[PathBuf]) -> Vec> { + paths.par_iter().map(file_digest).collect() +} + +fn file_digest(path: &PathBuf) -> std::io::Result { + let bytes: Vec = std::fs::read(path)?; + let mut hasher = blake3::Hasher::new(); + let _ = hasher.update_rayon(&bytes); + Ok(*hasher.finalize().as_bytes()) +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs index d9c3e7727b..c738ef926d 100644 --- a/crates/sqlbuild-cache/src/digest/main/mod.rs +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -1,3 +1,4 @@ pub mod content_digest; +pub mod digest_files; pub mod fingerprint_project_files; pub mod hex_digest; diff --git a/crates/sqlbuild-cache/src/digest/tests.rs b/crates/sqlbuild-cache/src/digest/tests.rs index 712cba961e..7ab98d75a0 100644 --- a/crates/sqlbuild-cache/src/digest/tests.rs +++ b/crates/sqlbuild-cache/src/digest/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_content_digest.rs"] mod content_digest; +#[path = "tests/test_digest_files.rs"] +mod digest_files; #[path = "tests/test_fingerprint_project_files.rs"] mod fingerprint_project_files; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs b/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs new file mode 100644 index 0000000000..2497649562 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_digest_files.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use crate::digest::main::digest_files::digest_files; +use crate::digest::tests::helpers::write_file; +use crate::digest::tests::test_types::DigestFilesTestCase; + +#[test] +fn given_paths_when_digesting_files_then_readable_files_digest_their_bytes() { + let test_cases = [ + DigestFilesTestCase { + description: "regular file", + path: |root| root.join("macros/cents.py"), + expected_digest_of: Some("def cents(amount):\n return amount\n"), + }, + DigestFilesTestCase { + description: "missing file", + path: |root| root.join("macros/missing.py"), + expected_digest_of: None, + }, + DigestFilesTestCase { + description: "member path inside an archive", + path: |root| root.join("macros/cents.py/inner.py"), + expected_digest_of: None, + }, + DigestFilesTestCase { + description: "directory", + path: |root| root.join("macros"), + expected_digest_of: None, + }, + ]; + + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + write_file( + project.path(), + "macros/cents.py", + "def cents(amount):\n return amount\n", + ); + let path: PathBuf = (test_case.path)(project.path()); + + let digests: Vec> = + digest_files(&[path]).into_iter().map(Result::ok).collect(); + + assert_eq!( + digests, + vec![ + test_case + .expected_digest_of + .map(|text| *blake3::hash(text.as_bytes()).as_bytes()) + ], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_types.rs b/crates/sqlbuild-cache/src/digest/tests/test_types.rs index f6891edbc5..3ecb41715d 100644 --- a/crates/sqlbuild-cache/src/digest/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/digest/tests/test_types.rs @@ -27,3 +27,9 @@ pub(crate) struct FingerprintFailureTestCase { pub(crate) root: fn(&Path) -> PathBuf, pub(crate) expected_failure: bool, } + +pub(crate) struct DigestFilesTestCase { + pub(crate) description: &'static str, + pub(crate) path: fn(&Path) -> PathBuf, + pub(crate) expected_digest_of: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs index f28b85d95b..0cd6e1c9a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -4,6 +4,7 @@ use pyo3::exceptions::{PyOSError, PyRuntimeError}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::content_digest::content_digest as digest; +use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each; use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; @@ -33,8 +34,25 @@ pub(crate) fn fingerprint_project_files( Ok(hex_digest(&fingerprinted)) } +/// Hexadecimal content digest of each file in `paths`, or `None` where it cannot be read. +#[pyfunction] +pub(crate) fn digest_files(py: Python<'_>, paths: Vec) -> PyResult>> { + let digests: Vec> = py + .compiler_detach(|| Ok(digest_each(&paths))) + .map_err(PyRuntimeError::new_err)?; + Ok(digests.iter().map(readable_digest).collect()) +} + +fn readable_digest(digest: &std::io::Result) -> Option { + match digest { + Ok(digest) => Some(hex_digest(digest)), + Err(_unreadable) => None, + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(content_digest, module)?)?; module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; + module.add_function(wrap_pyfunction!(digest_files, module)?)?; Ok(()) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 2902c08b5f..808371a6c9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -228,7 +228,10 @@ SQLBuild relies on this contract to reuse macro results: - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when macro or other Python files, configuration, declarations, seeds, sources, variables, the target, the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps - them, and a changed call is always run again. + them, and a changed call is always run again. Only the first compile in a process uses and saves + them: later compiles in the same process, such as in a notebook or a long-running server, run their + macros again. A compile that has loaded Python code it cannot read from a plain file, such as a + module imported from a zip archive, neither uses nor saves them. Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and `@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index dbbe8014c4..83909e19fc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -211,6 +211,7 @@ class MacroCallMemo: def content_digest(parts: list[str]) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... +def digest_files(paths: list[str]) -> list[str | None]: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index faf6a00985..8e8c3238f2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge +from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.spec.contracts.models import TargetConfig @@ -19,8 +20,9 @@ def target_context_with_macro_call_store( selected_target: str | None, no_cache: bool, ) -> tuple[str | None, TargetConfig | None, Path | None]: - """Resolve the target and cache directory, loading stored macro calls when a bridge runs.""" + """Resolve the target and cache directory; a process's first compile may load stored calls.""" + first_compile: bool = claim_first_compile() target_context: tuple[str | None, TargetConfig | None, Path | None] = ( build_compile_target_context( discovered_inputs=discovered_inputs, @@ -32,7 +34,8 @@ def target_context_with_macro_call_store( project_dir: Path | None = discovered_inputs.project_dir compile_cache_dir: Path | None = target_context[2] if ( - bridge is not None + first_compile + and bridge is not None and compile_cache_dir is not None and project_dir is not None and native_stage_enabled(NativeStage.MACRO_CALL_STORE) diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py index 7bcd8f0300..67bbca6d4d 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -1,28 +1,41 @@ -"""The environment stored macro call results are valid in: build, interpreter and project files.""" +"""The environment stored macro call results are valid in: build, interpreter, code and files.""" from __future__ import annotations +import functools +import importlib.machinery import json import os import sys +import sysconfig +from collections.abc import Iterable from pathlib import Path import sqlbuild._native as _native from sqlbuild.compiler.macro_bridge.constants import ( + INSTALLED_DISTRIBUTION_SUFFIX, + INSTALLED_PACKAGE_DIRECTORY_NAMES, + INSTALLED_RECORD_FILE_NAME, + INTERPRETER_MODULE_ORIGINS, MACRO_CALL_STORE_ENVIRONMENT_VERSION, MISSING_SEARCH_PATH_STAMP, - MODULE_STAMP_ROW_FIELDS, + MODULE_DIGEST_ROW_FIELDS, PROJECT_ROOT_SEARCH_PATH_STAMP, STORE_TRACKED_ENVIRONMENT_PREFIXES, STORE_UNTRACKED_ENVIRONMENT_NAMES, ) -from sqlbuild.compiler.macro_bridge.types import ModuleStamp +from sqlbuild.compiler.macro_bridge.models import ModuleSources -def store_environment(*, project_dir: Path, model_paths: list[str]) -> str: - """Digest every input but model SQL files and context; OSError if a file is unreadable.""" +def project_fingerprint(*, project_dir: Path, model_paths: list[str]) -> str: + """Digest every project file but model SQL files; OSError if a file is unreadable.""" + + return _native.fingerprint_project_files(str(project_dir), model_paths) + + +def store_environment(*, project_dir: Path, fingerprint: str) -> str: + """Digest the build, interpreter, import path, installed packages, variables and project.""" - fingerprint: str = _native.fingerprint_project_files(str(project_dir), model_paths) return _native.content_digest( [ MACRO_CALL_STORE_ENVIRONMENT_VERSION, @@ -33,60 +46,148 @@ def store_environment(*, project_dir: Path, model_paths: list[str]) -> str: str(sys.implementation.cache_tag), json.dumps(_search_path_stamps(project_dir=project_dir)), json.dumps(_tracked_environment()), + json.dumps(_installed_distribution_records()), fingerprint, ] ) -def loaded_module_stamps() -> dict[str, ModuleStamp]: - """Stat every loaded module file, including modules macros imported while running.""" +def module_sources(modules: Iterable[object]) -> ModuleSources: + """Files backing `modules` outside the interpreter and SQLBuild, and whether all have one.""" + + paths: list[str] = [] + for module in modules: + path: str | None + known: bool + path, known = _backing_file(module) + if not known: + return ModuleSources(paths=(), complete=False) + if path is not None and not _covered_without_content(path): + paths.append(path) + return ModuleSources(paths=tuple(paths), complete=True) - stamps: dict[str, ModuleStamp] = {} - for module in tuple(sys.modules.values()): - path: object = getattr(module, "__file__", None) - if not isinstance(path, str) or path in stamps: - continue - try: - status: os.stat_result = os.stat(path) - except OSError: - continue - stamps[path] = (status.st_mtime_ns, status.st_size) - return stamps +def digest_module_files(paths: Iterable[str]) -> dict[str, str] | None: + """Content digest of each module file, or None when any file cannot be read.""" + + ordered: list[str] = list(dict.fromkeys(paths)) + digests: list[str | None] = _native.digest_files(ordered) + if any(digest is None for digest in digests): + return None + return {path: digest for path, digest in zip(ordered, digests, strict=True) if digest} -def module_stamps_metadata(stamps: dict[str, ModuleStamp]) -> bytes: - """Encode module stamps for the store's metadata.""" - return json.dumps(sorted([path, *stamp] for path, stamp in stamps.items())).encode() +def module_digests_metadata(digests: dict[str, str]) -> bytes: + """Encode module file digests for the store's metadata.""" + return json.dumps(sorted([path, digest] for path, digest in digests.items())).encode() -def unchanged_module_stamps(metadata: bytes) -> dict[str, ModuleStamp] | None: - """Return the stored module stamps when every module file is unchanged, else None.""" + +def unchanged_module_digests(*, metadata: bytes, known: dict[str, str]) -> dict[str, str] | None: + """Return the stored module digests when every module file still has them, else None.""" try: rows: object = json.loads(metadata) except ValueError: return None - if not isinstance(rows, list): + if not isinstance(rows, list) or not all(map(_is_digest_row, rows)): return None - stamps: dict[str, ModuleStamp] = {} - for row in rows: - if not ( - isinstance(row, list) - and len(row) == MODULE_STAMP_ROW_FIELDS - and isinstance(row[0], str) - and isinstance(row[1], int) - and isinstance(row[2], int) - ): - return None - try: - status: os.stat_result = os.stat(row[0]) - except OSError: - return None - if (status.st_mtime_ns, status.st_size) != (row[1], row[2]): - return None - stamps[row[0]] = (row[1], row[2]) - return stamps + stored: dict[str, str] = {row[0]: row[1] for row in rows} + unknown: list[str] = [path for path in stored if path not in known] + current: dict[str, str] | None = digest_module_files(unknown) + if current is None: + return None + current.update((path, known[path]) for path in stored if path in known) + return stored if current == stored else None + + +def _is_digest_row(row: object) -> bool: + return ( + isinstance(row, list) + and len(row) == MODULE_DIGEST_ROW_FIELDS + and all(isinstance(field, str) for field in row) + ) + + +def _backing_file(module: object) -> tuple[str | None, bool]: + spec: object = getattr(module, "__spec__", None) + file: object = getattr(module, "__file__", None) + if spec is None: + return (file, True) if isinstance(file, str) and os.path.isfile(file) else (None, True) + origin: object = getattr(spec, "origin", None) + if getattr(spec, "has_location", False) and isinstance(origin, str): + return origin, True + if origin in INTERPRETER_MODULE_ORIGINS: + return None, True + if ( + origin is None + and file is None + and getattr(spec, "submodule_search_locations", None) is not None + ): + return None, True + return None, False + + +def _covered_without_content(path: str) -> bool: + installed, interpreter, own = _excluded_prefixes() + if path.startswith(own): + return True + if path.startswith(installed): + return path.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)) + return path.startswith(interpreter) + + +@functools.cache +def _excluded_prefixes() -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + return ( + _directory_prefixes(_installed_directories()), + _directory_prefixes(sysconfig.get_paths().get(name) for name in ("stdlib", "platstdlib")), + _directory_prefixes([os.path.dirname(_native.__file__)]), + ) + + +@functools.cache +def _installed_directories() -> tuple[str, ...]: + paths: dict[str, str] = sysconfig.get_paths() + return tuple( + dict.fromkeys( + [ + *(paths[name] for name in ("purelib", "platlib") if name in paths), + *( + entry + for entry in sys.path + if os.path.basename(entry) in INSTALLED_PACKAGE_DIRECTORY_NAMES + ), + ] + ) + ) + + +def _installed_distribution_records() -> list[tuple[str, str | None]]: + records: list[str] = [] + for directory in _installed_directories(): + records.extend(_distribution_records(directory)) + records.sort() + return list(zip(records, _native.digest_files(records), strict=True)) + + +def _distribution_records(directory: str) -> list[str]: + try: + with os.scandir(directory) as entries: + return [ + os.path.join(entry.path, INSTALLED_RECORD_FILE_NAME) + for entry in entries + if entry.name.endswith(INSTALLED_DISTRIBUTION_SUFFIX) + ] + except OSError: + return [] + + +def _directory_prefixes(directories: Iterable[str | None]) -> tuple[str, ...]: + prefixes: list[str] = [] + for directory in filter(None, directories): + prefixes.extend(os.path.join(form, "") for form in (directory, os.path.realpath(directory))) + return tuple(dict.fromkeys(prefixes)) def _search_path_stamps(*, project_dir: Path) -> list[tuple[str, int]]: diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index caa3708202..b073556bc5 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import sys from collections.abc import Hashable, Mapping from pathlib import Path @@ -14,10 +15,12 @@ MacroContext, ) from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( - loaded_module_stamps, - module_stamps_metadata, + digest_module_files, + module_digests_metadata, + module_sources, + project_fingerprint, store_environment, - unchanged_module_stamps, + unchanged_module_digests, ) from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( call_class_store_text, @@ -25,8 +28,8 @@ macro_store_token, ) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME -from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite -from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord, ModuleStamp +from sqlbuild.compiler.macro_bridge.models import MacroCallClass, MacroCallSite, ModuleSources +from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -44,10 +47,13 @@ def __init__(self, *, python_version: tuple[int, int], unicode_version: str) -> self._name_tokens: dict[tuple[int, str], tuple[Hashable, str | None] | None] = {} self._context_tokens: dict[tuple[int, int], tuple[Hashable, str | None]] = {} self._store_path: Path | None = None - self._carried_module_stamps: dict[str, ModuleStamp] = {} + self._store_trusted: bool = False + self._module_digests: dict[str, str] = {} + self._observed_modules: dict[str, object] = {} + self._project_inputs: tuple[Path, list[str], str] = (Path(), [], "") def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[str]) -> None: - """Reuse call results stored by earlier compiles whose environment matches this one.""" + """Reuse call results stored by earlier compiles of the same code and project files.""" if self._store_path is not None: return @@ -55,34 +61,75 @@ def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[ metadata: bytes loaded: int try: - environment: str = store_environment(project_dir=project_dir, model_paths=model_paths) + if not self._observe_modules(): + logging.getLogger(__name__).debug("Macro call store not used: unknown module code") + return + fingerprint: str = project_fingerprint(project_dir=project_dir, model_paths=model_paths) + environment: str = store_environment(project_dir=project_dir, fingerprint=fingerprint) metadata, loaded = self._memo.attach_store(str(path), environment) except (OSError, RuntimeError) as error: logging.getLogger(__name__).debug("Macro call store not loaded: %s", error) return - stamps: dict[str, ModuleStamp] | None = unchanged_module_stamps(metadata) if loaded else {} - if stamps is None: + carried: dict[str, str] | None = ( + unchanged_module_digests(metadata=metadata, known=self._module_digests) + if loaded + else {} + ) + if carried is None: self._memo.discard_store() - stamps = {} + carried = {} self._store_path = path - self._carried_module_stamps = stamps + self._store_trusted = True + self._module_digests = {**carried, **self._module_digests} + self._project_inputs = (project_dir, model_paths, fingerprint) self._name_tokens.clear() self._context_tokens.clear() def save_store(self) -> None: - """Save the stored results this compile used or recorded, with the modules they ran on.""" + """Save the results this compile used or recorded, unless code or files it read changed.""" - if self._store_path is None: + if self._store_path is None or not self._store_trusted: return - stamps: dict[str, ModuleStamp] = { - **self._carried_module_stamps, - **loaded_module_stamps(), - } + project_dir, model_paths, fingerprint = self._project_inputs try: - _ = self._memo.save_store(str(self._store_path), module_stamps_metadata(stamps)) + if ( + not self._observe_modules() + or digest_module_files(self._module_digests) != self._module_digests + or project_fingerprint(project_dir=project_dir, model_paths=model_paths) + != fingerprint + ): + logging.getLogger(__name__).debug( + "Macro call store not saved: code or files changed" + ) + return + _ = self._memo.save_store( + str(self._store_path), module_digests_metadata(self._module_digests) + ) except (OSError, RuntimeError) as error: logging.getLogger(__name__).debug("Macro call store not saved: %s", error) + def _observe_modules(self) -> bool: + """Digest the files of modules loaded since the last look; False when one is unknown.""" + + loaded: dict[str, object] = dict(sys.modules) + added: list[object] = [ + module + for name, module in loaded.items() + if self._observed_modules.get(name) is not module + ] + self._observed_modules = loaded + sources: ModuleSources = module_sources(added) + digests: dict[str, str] | None = ( + digest_module_files(sources.paths) if sources.complete else None + ) + if digests is None or any( + self._module_digests.get(path, digest) != digest for path, digest in digests.items() + ): + self._store_trusted = False + return False + self._module_digests.update(digests) + return True + def scan(self, sql: str) -> tuple[MacroCallSite, ...] | None: """Return the top-level call sites of `sql`, or None when Python must expand it.""" @@ -192,6 +239,8 @@ def record( ) -> None: """Record the result of one executed call for later calls of the same class and text.""" + if self._store_trusted and len(sys.modules) != len(self._observed_modules): + _ = self._observe_modules() self._memo.record( class_id, call_text, diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 4ac7e5a682..50b84b4929 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools from contextvars import ContextVar from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR @@ -17,12 +18,17 @@ ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) +COMPILES_STARTED: itertools.count[int] = itertools.count() MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" -MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v1" +MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v2" VALUE_RENDERER_FIELD: str = "_value_renderer" PROJECT_ROOT_SEARCH_PATH_STAMP: int = -2 MISSING_SEARCH_PATH_STAMP: int = -1 -MODULE_STAMP_ROW_FIELDS: int = 3 +MODULE_DIGEST_ROW_FIELDS: int = 2 +INSTALLED_PACKAGE_DIRECTORY_NAMES: frozenset[str] = frozenset({"site-packages", "dist-packages"}) +INSTALLED_DISTRIBUTION_SUFFIX: str = ".dist-info" +INSTALLED_RECORD_FILE_NAME: str = "RECORD" +INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} diff --git a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py new file mode 100644 index 0000000000..ef5d3668c6 --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py @@ -0,0 +1,11 @@ +"""Count the compiles this process has started, so only its first one uses the macro call store.""" + +from __future__ import annotations + +from sqlbuild.compiler.macro_bridge.constants import COMPILES_STARTED + + +def claim_first_compile() -> bool: + """Record that a compile started; True only for the first compile of this process.""" + + return next(COMPILES_STARTED) == 0 diff --git a/src/sqlbuild/compiler/macro_bridge/models.py b/src/sqlbuild/compiler/macro_bridge/models.py index 10e812364c..91ece77cc5 100644 --- a/src/sqlbuild/compiler/macro_bridge/models.py +++ b/src/sqlbuild/compiler/macro_bridge/models.py @@ -25,3 +25,11 @@ class MacroCallClass: macro_store_tokens: tuple[str, ...] context_store_token: str | None persistent: bool + + +@dataclass(frozen=True) +class ModuleSources: + """Files backing loaded modules, and whether every module's backing could be identified.""" + + paths: tuple[str, ...] + complete: bool diff --git a/src/sqlbuild/compiler/macro_bridge/types.py b/src/sqlbuild/compiler/macro_bridge/types.py index c738b6a7e9..767a96c96c 100644 --- a/src/sqlbuild/compiler/macro_bridge/types.py +++ b/src/sqlbuild/compiler/macro_bridge/types.py @@ -6,4 +6,3 @@ type MacroCallEvent = tuple[int, str, str] type MacroCallRecord = tuple[str, tuple[SqlResourceRef, ...], tuple[MacroCallEvent, ...]] -type ModuleStamp = tuple[int, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index e107c1e2fe..5d058fe1be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -5,11 +5,15 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple import pytest from tests.integration.src.sqlbuild.compiler.pipeline._test_types import PreparedCompile +if TYPE_CHECKING: + from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import CompileReuseRun + @dataclass(frozen=True) class SourceSemanticBindingCase: @@ -934,3 +938,30 @@ class EngineMacroCallGateTestCase: engine: str expected_logged_calls: tuple[int, ...] expected_store_files: tuple[str, ...] + + +class StaleStoreArrangement(NamedTuple): + """What an arrangement step may use to compile and edit before the fresh-process check.""" + + project_dir: Path + extlib: Path + monkeypatch: pytest.MonkeyPatch + capsys: pytest.CaptureFixture[str] + + +@dataclass(frozen=True) +class StaleMacroModuleStoreTestCase: + """An outside module a macro imported, edited after this process imported it.""" + + description: str + arrange: Callable[[StaleStoreArrangement], None] + compile_afresh: Callable[[StaleStoreArrangement], CompileReuseRun] + expected_flavor: str + + +@dataclass(frozen=True) +class SecondCompileStoreTestCase: + """Two compiles in one process; the second must run its macros without the store.""" + + description: str + expected_second_matches_first: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 4b4c80e8b1..58fcdaf275 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -14,6 +14,7 @@ import subprocess import sys import time +import zipfile from bisect import bisect_left from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout @@ -34,6 +35,7 @@ import sqlbuild.cli.compile_reuse._helpers.store as reuse_store import sqlbuild.compiler.compile._helpers.assembly.binding_waves as binding_waves import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses @@ -63,6 +65,7 @@ IncrementalEditStep, SemanticCorpusCase, SetOperationModel, + StaleStoreArrangement, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb from tests.integration.src.sqlbuild.compiler.compile.helpers import ( @@ -4275,8 +4278,16 @@ def prepare_reference_call_project(*, tmp_path: Path, staging_from: str, mart_fr STORE_ENVIRONMENT_REGION_VAR: str = "SQB_STORE_TEST_REGION" STORE_ARGUMENT_ENV_VAR: str = "STORE_TEST_ARGUMENT" _REUSE_DISABLED_VALUES: dict[bool, str] = {True: "0", False: "1"} +STALE_FLAVOR_EDIT: str = "salty" _SOUTH_MODEL: str = "models/south/orders_south.sql" _STORE_EXTERNAL_MODULE: str = "store_flavor_values" +_ZIPPED_FLAVOR_MODULE: str = "zipped_flavor_values" +_STORE_IMPORTED_MODULES: tuple[str, ...] = ( + _STORE_EXTERNAL_MODULE, + _ZIPPED_FLAVOR_MODULE, + "macros", + "macros._label_values", +) _STORE_PROJECT_EXTRA_FILES: dict[str, str] = { "macros/context_labels.py": ( '"""Macros that read the target, the environment and an outside module."""\n\n' @@ -4405,7 +4416,170 @@ def freeze_macro_call_store_environment(monkeypatch: pytest.MonkeyPatch) -> None """Break the store key: every project and module state looks like the first one.""" monkeypatch.setattr(macro_bridge_class, "store_environment", lambda **_kwargs: "frozen") - monkeypatch.setattr(macro_bridge_class, "unchanged_module_stamps", lambda _metadata: {}) + monkeypatch.setattr(macro_bridge_class, "unchanged_module_digests", lambda **_kwargs: {}) + monkeypatch.setattr(macro_bridge_class.MacroBridge, "_observe_modules", lambda _bridge: True) + monkeypatch.setattr(call_store_module, "claim_first_compile", lambda: True) + + +def pretend_fresh_process(monkeypatch: pytest.MonkeyPatch) -> None: + """Treat this test process as one no compile has run in yet.""" + + compiles: Iterator[int] = itertools.count() + monkeypatch.setattr(call_store_module, "claim_first_compile", lambda: next(compiles) == 0) + + +def logged_in_process_compile( + *, project_dir: Path, log_path: Path, capsys: pytest.CaptureFixture[str] +) -> int: + """Compile in this process and return how many macro calls ran.""" + + _ = log_path.write_text("", encoding="utf-8") + _ = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + return len(log_path.read_text(encoding="utf-8").splitlines()) + + +def _save_after(monkeypatch: pytest.MonkeyPatch, change: Callable[[], object]) -> None: + original: Callable[[macro_bridge_class.MacroBridge], None] = ( + macro_bridge_class.MacroBridge.save_store + ) + pending: Iterator[Callable[[], object]] = iter([change]) + + def change_then_save(bridge: macro_bridge_class.MacroBridge) -> None: + _ = [next_change() for next_change in itertools.islice(pending, 1)] + original(bridge) + + monkeypatch.setattr(macro_bridge_class.MacroBridge, "save_store", change_then_save) + + +def recompile_in_process_after_edit(arrangement: StaleStoreArrangement) -> None: + """Compile twice in one process, editing the flavor module between the compiles.""" + + pretend_fresh_process(arrangement.monkeypatch) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def edit_flavor_while_saving(arrangement: StaleStoreArrangement) -> None: + """Compile in a fresh process that edits the flavor module just before saving the store.""" + + pretend_fresh_process(arrangement.monkeypatch) + _save_after( + arrangement.monkeypatch, + lambda: write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT), + ) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def move_flavor_while_saving(arrangement: StaleStoreArrangement) -> None: + """Move the loaded flavor away before saving, then write a new one, keeping folder times.""" + + module_path: Path = arrangement.extlib / f"{_STORE_EXTERNAL_MODULE}.py" + folder: os.stat_result = arrangement.extlib.stat() + pretend_fresh_process(arrangement.monkeypatch) + _save_after( + arrangement.monkeypatch, lambda: module_path.rename(module_path.with_suffix(".moved")) + ) + _ = in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + os.utime(arrangement.extlib, ns=(folder.st_atime_ns, folder.st_mtime_ns)) + + +def rezip_flavor_between_processes(arrangement: StaleStoreArrangement) -> None: + """Serve the flavor from a zip archive, compile in a process, then rebuild the archive.""" + + write_zipped_flavor(extlib=arrangement.extlib, value="sweet") + _ = fresh_process_compile(project_dir=arrangement.project_dir, extlib=arrangement.extlib) + write_zipped_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + + +def backdate_flavor_between_processes(arrangement: StaleStoreArrangement) -> None: + """Compile in a process, then replace the flavor keeping its size and modification time.""" + + module_path: Path = arrangement.extlib / f"{_STORE_EXTERNAL_MODULE}.py" + _ = fresh_process_compile(project_dir=arrangement.project_dir, extlib=arrangement.extlib) + status: os.stat_result = module_path.stat() + write_store_flavor(extlib=arrangement.extlib, value=STALE_FLAVOR_EDIT) + os.utime(module_path, ns=(status.st_atime_ns, status.st_mtime_ns)) + + +def write_zipped_flavor(*, extlib: Path, value: str) -> None: + """Make the flavor module re-export a value from a module inside a zip archive.""" + + archive: Path = extlib / "flavors.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(f"{_ZIPPED_FLAVOR_MODULE}.py", f"VALUE = {value!r}\n") + _ = (extlib / f"{_STORE_EXTERNAL_MODULE}.py").write_text( + f"import sys\n\nsys.path.insert(0, {str(archive)!r})\n" + f"from {_ZIPPED_FLAVOR_MODULE} import VALUE\n", + encoding="utf-8", + ) + + +def fresh_process_compile(*, project_dir: Path, extlib: Path) -> MacroCallStoreRun: + """Compile with the macro call store in a new process, then with --no-cache.""" + + return macro_call_store_compile( + project_dir=project_dir, + extlib=extlib, + log_path=project_dir.parent / "fresh-calls.log", + project_reuse=False, + ) + + +def compile_in_new_process(arrangement: StaleStoreArrangement) -> CompileReuseRun: + """Compile with the macro call store in a new process.""" + + return fresh_process_compile( + project_dir=arrangement.project_dir, extlib=arrangement.extlib + ).incremental + + +def compile_as_new_process_here(arrangement: StaleStoreArrangement) -> CompileReuseRun: + """Compile in this process as if it had just started, re-importing the flavor modules.""" + + forget_store_flavor_module() + pretend_fresh_process(arrangement.monkeypatch) + return in_process_reuse_run(project_dir=arrangement.project_dir, capsys=arrangement.capsys) + + +def uncached_reference_compile(*, project_dir: Path, extlib: Path) -> CompileReuseRun: + """Compile with --no-cache in a new process, the oracle for stored results.""" + + return run_reuse_compile( + project_dir=project_dir, + env={"PYTHONPATH": str(extlib), COMPILER_ENGINE_ENV_VAR: MACRO_CALL_STORE_ENGINE}, + args=("--no-cache",), + ) + + +def block_proc_reads(extlib: Path) -> None: + """Make Python file reads under /proc fail in processes importing from `extlib`.""" + + _ = (extlib / "sitecustomize.py").write_text( + "import builtins\nimport io\n\n_open = io.open\n\n\n" + "def _guarded_open(file, *args, **kwargs):\n" + " if str(file).startswith('/proc'):\n" + " raise FileNotFoundError(file)\n" + " return _open(file, *args, **kwargs)\n\n\n" + "builtins.open = io.open = _guarded_open\n", + encoding="utf-8", + ) + + +def forget_store_flavor_module() -> None: + """Drop the flavor and project macro helper modules in-process compiles imported.""" + + _ = [sys.modules.pop(name, None) for name in _STORE_IMPORTED_MODULES] + + +def store_files(project_dir: Path) -> dict[str, bytes]: + """Every macro call store file under the project target folder, by relative path.""" + + return { + path.relative_to(project_dir).as_posix(): path.read_bytes() + for path in sorted((project_dir / "target").rglob(MACRO_CALL_STORE_FILE_NAME)) + } class EngineMacroCallRuns(NamedTuple): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 56ca19af5f..a224622ee0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from pathlib import Path import pytest @@ -11,6 +12,9 @@ EngineMacroCallGateTestCase, MacroCallStoreEditSequenceTestCase, MacroCallStoreEditStep, + SecondCompileStoreTestCase, + StaleMacroModuleStoreTestCase, + StaleStoreArrangement, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( MACRO_CALL_STORE_ENGINE, @@ -19,15 +23,28 @@ CompileReuseRun, EngineMacroCallRuns, MacroCallStoreRun, + backdate_flavor_between_processes, + block_proc_reads, + compile_as_new_process_here, + compile_in_new_process, + edit_flavor_while_saving, edit_label_helper, edit_south_model, engine_macro_call_runs, + forget_store_flavor_module, freeze_macro_call_store_environment, in_process_reuse_run, + logged_in_process_compile, macro_call_store_compile, + move_flavor_while_saving, prepare_macro_call_store_project, + pretend_fresh_process, + recompile_in_process_after_edit, replace_project_text, + rezip_flavor_between_processes, run_reuse_compile, + store_files, + uncached_reference_compile, write_project_file, write_store_flavor, ) @@ -286,3 +303,142 @@ def test_given_engine_when_compiling_repeatedly_then_only_preview_batches_and_st assert runs.returncodes == (0, 0) assert runs.logged_calls == test_case.expected_logged_calls assert runs.store_files == test_case.expected_store_files + + +@pytest.mark.parametrize( + "test_case", + [ + StaleMacroModuleStoreTestCase( + description="edited_between_two_compiles_of_one_process", + arrange=recompile_in_process_after_edit, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="edited_between_store_attach_and_save", + arrange=edit_flavor_while_saving, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="moved_away_while_loaded", + arrange=move_flavor_while_saving, + compile_afresh=compile_as_new_process_here, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="served_from_a_rebuilt_zip_archive", + arrange=rezip_flavor_between_processes, + compile_afresh=compile_in_new_process, + expected_flavor="'salty'", + ), + StaleMacroModuleStoreTestCase( + description="replaced_keeping_size_and_modification_time", + arrange=backdate_flavor_between_processes, + compile_afresh=compile_in_new_process, + expected_flavor="'salty'", + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_module_changed_after_use_when_compiling_in_a_new_process_then_no_stale_replay( + tmp_path: Path, + test_case: StaleMacroModuleStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(tmp_path / "macro-calls.log")) + monkeypatch.setattr(sys, "dont_write_bytecode", True) + forget_store_flavor_module() + arrangement: StaleStoreArrangement = StaleStoreArrangement( + project_dir=project_dir, extlib=extlib, monkeypatch=monkeypatch, capsys=capsys + ) + test_case.arrange(arrangement) + + fresh: CompileReuseRun = test_case.compile_afresh(arrangement) + forget_store_flavor_module() + reference: CompileReuseRun = uncached_reference_compile(project_dir=project_dir, extlib=extlib) + + assert (fresh.returncode, reference.returncode) == (0, 0), fresh.stderr + assert fresh.compiled == reference.compiled + assert test_case.expected_flavor in b"".join(fresh.compiled.values()).decode() + + +@pytest.mark.parametrize( + "test_case", + [ + SecondCompileStoreTestCase( + description="second_compile_of_a_process", expected_second_matches_first=True + ) + ], + ids=lambda case: case.description, +) +def test_given_second_compile_in_one_process_when_compiling_then_the_store_is_not_touched( + tmp_path: Path, + test_case: SecondCompileStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "macro-calls.log" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + monkeypatch.syspath_prepend(str(extlib)) + monkeypatch.setenv("SQLBUILD_COMPILER_ENGINE", MACRO_CALL_STORE_ENGINE) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_REUSE", "1") + monkeypatch.setenv(MACRO_CALL_LOG_ENV_VAR, str(log_path)) + forget_store_flavor_module() + pretend_fresh_process(monkeypatch) + first: int = logged_in_process_compile( + project_dir=project_dir, log_path=log_path, capsys=capsys + ) + stored: dict[str, bytes] = store_files(project_dir) + + second: int = logged_in_process_compile( + project_dir=project_dir, log_path=log_path, capsys=capsys + ) + forget_store_flavor_module() + + assert first > 0 + assert stored != {} + assert (second == first) is test_case.expected_second_matches_first + assert store_files(project_dir) == stored + + +@pytest.mark.parametrize( + "test_case", + [ + EngineMacroCallGateTestCase( + description="native_preview_without_proc", + engine="native-preview", + expected_logged_calls=(1, 0), + expected_store_files=("target/cache/compiler-native-preview-v1/macro-calls.bin",), + ) + ], + ids=lambda case: case.description, +) +def test_given_no_proc_filesystem_when_compiling_repeatedly_then_the_store_still_serves_calls( + tmp_path: Path, test_case: EngineMacroCallGateTestCase +) -> None: + project_dir: Path = tmp_path / "project" + extlib: Path = tmp_path / "extlib" + prepare_macro_call_store_project(project_dir=project_dir, extlib=extlib) + block_proc_reads(extlib) + + runs: EngineMacroCallRuns = engine_macro_call_runs( + project_dir=project_dir, + extlib=extlib, + log_path=tmp_path / "macro-calls.log", + engine=test_case.engine, + runs=2, + ) + + assert runs.returncodes == (0, 0) + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.store_files == test_case.expected_store_files diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py index 3e4bcf916f..d8d73b71f8 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -50,8 +50,8 @@ class MacroStoreTokenTestCase: @dataclass(frozen=True) -class ModuleStampsRoundTripTestCase: - """Module stamps encoded as store metadata and validated back.""" +class ModuleDigestsRoundTripTestCase: + """Module digests encoded as store metadata and validated back.""" description: str module_text: str @@ -78,9 +78,20 @@ class StoreEnvironmentVariableTestCase: @dataclass(frozen=True) -class ModuleStampsTestCase: - """Stored module stamps, a change to the module file, and whether they still validate.""" +class ModuleDigestsTestCase: + """Stored module digests, a change to the module file, and whether they still validate.""" description: str change: Callable[[Path], bytes] expected_valid: bool + + +@dataclass(frozen=True) +class ModuleSourcesTestCase: + """A loaded module and the file the store digests for it, if its code can be identified.""" + + description: str + module: Callable[[Path], object] + expected_paths: Callable[[Path], tuple[str, ...]] + expected_complete: bool + expected_digestible: bool diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py index 25d9a6374d..34953aa1bc 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -4,11 +4,21 @@ import decimal import json import os +import sys +import types +import zipfile from dataclasses import replace +from importlib.machinery import ModuleSpec +from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext from sqlbuild.compiler.discovery.models import ConstantDeclaration +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( + project_fingerprint, + store_environment, +) from sqlbuild.sql_values.models import SqlLogicalType, SqlValue from sqlbuild.sql_values.types import SqlValueKind @@ -57,31 +67,93 @@ def loaded_macro(*, relative_path: Path, raw_source: str) -> LoadedMacro: def write_module(path: Path, text: str) -> bytes: - """Write a module file and return metadata stamping its current state.""" + """Write a module file and return metadata recording its current content digest.""" _ = path.write_text(text, encoding="utf-8") - status: os.stat_result = path.stat() - return json.dumps([[str(path), status.st_mtime_ns, status.st_size]]).encode() + return json.dumps([[str(path), _native.digest_files([str(path)])[0]]]).encode() -def rewritten_module(path: Path) -> bytes: - """Stamp a module, then rewrite it with the same size and a later modification time.""" +def backdated_module(path: Path) -> bytes: + """Record a module, then replace it with other bytes of the same size and modification time.""" metadata: bytes = write_module(path, "VALUE = 1\n") - _ = path.write_text("VALUE = 2\n", encoding="utf-8") status: os.stat_result = path.stat() - os.utime(path, ns=(status.st_atime_ns, status.st_mtime_ns + 1_000_000)) + _ = path.write_text("VALUE = 2\n", encoding="utf-8") + os.utime(path, ns=(status.st_atime_ns, status.st_mtime_ns)) return metadata def removed_module(path: Path) -> bytes: - """Stamp a module, then remove it.""" + """Record a module, then remove it.""" metadata: bytes = write_module(path, "VALUE = 1\n") path.unlink() return metadata +def file_module(root: Path) -> object: + """A module loaded from a source file.""" + + path: Path = root / "flavor.py" + _ = path.write_text("VALUE = 1\n", encoding="utf-8") + spec: ModuleSpec | None = spec_from_file_location("flavor", path) + assert spec is not None + return module_from_spec(spec) + + +def zip_member_module(root: Path) -> object: + """A module whose location is a member of a zip archive on the import path.""" + + archive: Path = root / "flavors.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("zipped_flavor.py", "VALUE = 1\n") + return _located_module("zipped_flavor", str(archive / "zipped_flavor.py")) + + +def _located_module(name: str, origin: str) -> object: + spec: ModuleSpec = ModuleSpec(name, None, origin=origin) + spec.has_location = True + return module_from_spec(spec) + + +def moved_module(root: Path) -> object: + """A module loaded from a file that has since been moved away.""" + + module: object = file_module(root) + _ = (root / "flavor.py").rename(root / "flavor_moved.py") + return module + + +def custom_loader_module(_root: Path) -> object: + """A module from a loader that reports no file location.""" + + return module_from_spec(ModuleSpec("generated_flavor", None, origin="generated")) + + +def synthetic_module(_root: Path) -> object: + """A module object created in code, without an import spec.""" + + return types.ModuleType("synthetic_flavor") + + +def interpreter_module(_root: Path) -> object: + """A module built into the interpreter.""" + + return sys.modules["sys"] + + +def standard_library_module(_root: Path) -> object: + """A standard library module loaded from source.""" + + return sys.modules["json"] + + +def sqlbuild_module(_root: Path) -> object: + """A module of SQLBuild itself.""" + + return sys.modules["sqlbuild.compiler.macro_bridge.constants"] + + def write_environment_project(root: Path) -> None: """Write a model and a macro for store environment checks.""" @@ -92,3 +164,12 @@ def write_environment_project(root: Path) -> None: EXCLUDED_MODEL_PATHS: list[str] = ["models/orders.sql"] + + +def environment_of(root: Path) -> str: + """The store environment of a project, excluding its one model file.""" + + return store_environment( + project_dir=root, + fingerprint=project_fingerprint(project_dir=root, model_paths=EXCLUDED_MODEL_PATHS), + ) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py index 98aebe2a09..30e76f55d3 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py @@ -5,83 +5,95 @@ import pytest from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( - module_stamps_metadata, - store_environment, - unchanged_module_stamps, + digest_module_files, + module_digests_metadata, + module_sources, + unchanged_module_digests, ) +from sqlbuild.compiler.macro_bridge.models import ModuleSources from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( - ModuleStampsRoundTripTestCase, - ModuleStampsTestCase, + ModuleDigestsRoundTripTestCase, + ModuleDigestsTestCase, + ModuleSourcesTestCase, StoreEnvironmentTestCase, StoreEnvironmentVariableTestCase, ) from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers.helpers import ( - EXCLUDED_MODEL_PATHS, + backdated_module, + custom_loader_module, + environment_of, + file_module, + interpreter_module, + moved_module, removed_module, - rewritten_module, + sqlbuild_module, + standard_library_module, + synthetic_module, write_environment_project, write_module, + zip_member_module, ) @pytest.mark.parametrize( "test_case", [ - ModuleStampsTestCase( + ModuleDigestsTestCase( description="unchanged module", change=lambda path: write_module(path, "VALUE = 1\n"), expected_valid=True, ), - ModuleStampsTestCase( - description="module rewritten with the same size", - change=rewritten_module, + ModuleDigestsTestCase( + description="module replaced keeping its size and modification time", + change=backdated_module, expected_valid=False, ), - ModuleStampsTestCase( + ModuleDigestsTestCase( description="module removed", change=removed_module, expected_valid=False ), - ModuleStampsTestCase( + ModuleDigestsTestCase( description="unreadable metadata", change=lambda _path: b"{", expected_valid=False ), - ModuleStampsTestCase( + ModuleDigestsTestCase( description="unexpected metadata shape", - change=lambda _path: b'[["flavor.py", "mtime", 1]]', + change=lambda _path: b'[["flavor.py", 1]]', expected_valid=False, ), ], ids=lambda case: case.description, ) -def test_given_stored_module_stamps_when_validating_then_only_unchanged_modules_pass( - tmp_path: Path, test_case: ModuleStampsTestCase +def test_given_stored_module_digests_when_validating_then_only_unchanged_contents_pass( + tmp_path: Path, test_case: ModuleDigestsTestCase ) -> None: metadata: bytes = test_case.change(tmp_path / "flavor.py") - stamps: dict[str, tuple[int, int]] | None = unchanged_module_stamps(metadata) + digests: dict[str, str] | None = unchanged_module_digests(metadata=metadata, known={}) - assert (stamps is not None) is test_case.expected_valid + assert (digests is not None) is test_case.expected_valid @pytest.mark.parametrize( "test_case", [ - ModuleStampsRoundTripTestCase( + ModuleDigestsRoundTripTestCase( description="one module", module_text="VALUE = 1\n", expected_valid=True ), ], ids=lambda case: case.description, ) -def test_given_module_stamps_when_encoding_then_validation_returns_them( - tmp_path: Path, test_case: ModuleStampsRoundTripTestCase +def test_given_module_digests_when_encoding_then_validation_returns_them( + tmp_path: Path, test_case: ModuleDigestsRoundTripTestCase ) -> None: path: Path = tmp_path / "flavor.py" _ = write_module(path, test_case.module_text) - stamps: dict[str, tuple[int, int]] = {str(path): (path.stat().st_mtime_ns, path.stat().st_size)} + digests: dict[str, str] | None = digest_module_files([str(path)]) + assert digests is not None - validated: dict[str, tuple[int, int]] | None = unchanged_module_stamps( - module_stamps_metadata(stamps) + validated: dict[str, str] | None = unchanged_module_digests( + metadata=module_digests_metadata(digests), known={} ) - assert (validated == stamps) is test_case.expected_valid + assert (validated == digests) is test_case.expected_valid @pytest.mark.parametrize( @@ -112,10 +124,10 @@ def test_given_project_edit_when_digesting_store_environment_then_only_model_edi tmp_path: Path, test_case: StoreEnvironmentTestCase ) -> None: write_environment_project(tmp_path) - original: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + original: str = environment_of(tmp_path) _ = test_case.edit(tmp_path) - edited: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + edited: str = environment_of(tmp_path) assert (edited == original) is test_case.expected_unchanged @@ -149,9 +161,84 @@ def test_given_environment_variable_change_when_digesting_store_environment_then ) -> None: write_environment_project(tmp_path) monkeypatch.delenv(test_case.name, raising=False) - original: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + original: str = environment_of(tmp_path) monkeypatch.setenv(test_case.name, test_case.value) - changed: str = store_environment(project_dir=tmp_path, model_paths=EXCLUDED_MODEL_PATHS) + changed: str = environment_of(tmp_path) assert (changed == original) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleSourcesTestCase( + description="source file", + module=file_module, + expected_paths=lambda root: (str(root / "flavor.py"),), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="zip archive member", + module=zip_member_module, + expected_paths=lambda root: (str(root / "flavors.zip" / "zipped_flavor.py"),), + expected_complete=True, + expected_digestible=False, + ), + ModuleSourcesTestCase( + description="file moved after loading", + module=moved_module, + expected_paths=lambda root: (str(root / "flavor.py"),), + expected_complete=True, + expected_digestible=False, + ), + ModuleSourcesTestCase( + description="loader without a file location", + module=custom_loader_module, + expected_paths=lambda _root: (), + expected_complete=False, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="module object created in code", + module=synthetic_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="built into the interpreter", + module=interpreter_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="standard library source", + module=standard_library_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ModuleSourcesTestCase( + description="sqlbuild itself", + module=sqlbuild_module, + expected_paths=lambda _root: (), + expected_complete=True, + expected_digestible=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_module_when_identifying_its_code_then_only_readable_files_are_trusted( + tmp_path: Path, test_case: ModuleSourcesTestCase +) -> None: + module: object = test_case.module(tmp_path) + + sources: ModuleSources = module_sources([module]) + + assert sources == ModuleSources( + paths=test_case.expected_paths(tmp_path), complete=test_case.expected_complete + ) + assert (digest_module_files(sources.paths) is not None) is test_case.expected_digestible diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 9c1bba7045..8e3fda3c32 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -218,7 +218,10 @@ SQLBuild relies on this contract to reuse macro results: - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when macro or other Python files, configuration, declarations, seeds, sources, variables, the target, the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps - them, and a changed call is always run again. + them, and a changed call is always run again. Only the first compile in a process uses and saves + them: later compiles in the same process, such as in a notebook or a long-running server, run their + macros again. A compile that has loaded Python code it cannot read from a plain file, such as a + module imported from a zip archive, neither uses nor saves them. Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and `@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`, From 66301fca5ba126d535aae4eca38cc0a0f9e10378 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:16:31 +0100 Subject: [PATCH 10/11] test(native): replay macro-returned reference call errors for every consumer --- .../cli/commands/main/compile/_test_types.py | 9 ++ .../cli/commands/main/compile/helpers.py | 90 +++++++++++++++++++ .../compile/test_compile_macro_call_store.py | 51 +++++++++++ 3 files changed, 150 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 5d058fe1be..90da17fbf2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -900,6 +900,15 @@ class NativeModelLoopParityTestCase: expected_report_text: str +@dataclass(frozen=True) +class MacroReferenceCallStoreTestCase: + """Repeated compiles of models sharing a macro that returns a rejected reference call.""" + + description: str + engine: str + expected_logged_calls: tuple[int, ...] + + @dataclass(frozen=True) class MacroCallStoreEditStep: """One edit compiled with the macro call store, and the executions it expects.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 58fcdaf275..59ba8d67f5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4274,6 +4274,96 @@ def prepare_reference_call_project(*, tmp_path: Path, staging_from: str, mart_fr return project_dir +_MACRO_REFERENCE_CALL_MODELS: tuple[str, ...] = ("customer_orders", "customer_returns") +_DIAGNOSTIC_FIELDS: tuple[str, ...] = ( + "code", + "message", + "resource_name", + "path", + "line", + "column", + "help", +) + + +class MacroReferenceCallRuns(NamedTuple): + """Repeated compiles: exit codes, macro executions, and the diagnostics of each run.""" + + returncodes: tuple[int, ...] + logged_calls: tuple[int, ...] + diagnostics: tuple[tuple[tuple[object, ...], ...], ...] + + +def prepare_macro_reference_call_project(*, tmp_path: Path) -> Path: + """Write a project whose models share a macro returning a reference call compile rejects.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _REFERENCE_CALL_PROJECT_TOML, + "sources/raw.yml": _REFERENCE_CALL_SOURCES_YML, + "models/staging/customers.sql": ( + 'MODEL (description "Customers.", materialized view);\n\n' + 'SELECT order_id AS customer_id FROM __source("raw_orders")\n' + ), + "models/marts/_sqlbuild/_macros/customer_relation.py": ( + '"""A macro that returns an unquoted reference call."""\n\n' + "import os\n\n\n" + "def customer_relation() -> str:\n" + ' """Return the customers relation, logging each execution."""\n' + f' path = os.environ.get("{MACRO_CALL_LOG_ENV_VAR}")\n' + " if path:\n" + ' with open(path, "a", encoding="utf-8") as log:\n' + ' log.write("customer_relation\\n")\n' + ' return "__ref(customers)"\n' + ), + **{ + f"models/marts/{name}.sql": ( + f'MODEL (description "{name}.", materialized view);\n\n' + "SELECT customer_id FROM @customer_relation()\n" + ) + for name in _MACRO_REFERENCE_CALL_MODELS + }, + }, + ) + + +def _diagnostic_fields(item: dict[str, object]) -> tuple[object, ...]: + return tuple(item.get(field) for field in _DIAGNOSTIC_FIELDS) + + +def macro_reference_call_runs( + *, project_dir: Path, log_path: Path, engine: str, runs: int +) -> MacroReferenceCallRuns: + """Compile `runs` times in fresh processes, keeping the macro call store between runs.""" + + env: dict[str, str] = { + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + MACRO_CALL_LOG_ENV_VAR: str(log_path), + } + returncodes: list[int] = [] + logged_calls: list[int] = [] + diagnostics: list[tuple[tuple[object, ...], ...]] = [] + for _ in range(runs): + _ = log_path.write_text("", encoding="utf-8") + compiled: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json"), env=env + ) + returncodes.append(compiled.returncode) + logged_calls.append(len(log_path.read_text(encoding="utf-8").splitlines())) + items: list[dict[str, object]] = cast( + list[dict[str, object]], json.loads(compiled.stdout)["diagnostics"] + ) + diagnostics.append(tuple(_diagnostic_fields(item) for item in items)) + return MacroReferenceCallRuns( + returncodes=tuple(returncodes), + logged_calls=tuple(logged_calls), + diagnostics=tuple(diagnostics), + ) + + MACRO_CALL_STORE_ENGINE: str = "native-preview" STORE_ENVIRONMENT_REGION_VAR: str = "SQB_STORE_TEST_REGION" STORE_ARGUMENT_ENV_VAR: str = "STORE_TEST_ARGUMENT" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index a224622ee0..5e1f98e710 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -12,6 +12,7 @@ EngineMacroCallGateTestCase, MacroCallStoreEditSequenceTestCase, MacroCallStoreEditStep, + MacroReferenceCallStoreTestCase, SecondCompileStoreTestCase, StaleMacroModuleStoreTestCase, StaleStoreArrangement, @@ -23,6 +24,7 @@ CompileReuseRun, EngineMacroCallRuns, MacroCallStoreRun, + MacroReferenceCallRuns, backdate_flavor_between_processes, block_proc_reads, compile_as_new_process_here, @@ -36,8 +38,10 @@ in_process_reuse_run, logged_in_process_compile, macro_call_store_compile, + macro_reference_call_runs, move_flavor_while_saving, prepare_macro_call_store_project, + prepare_macro_reference_call_project, pretend_fresh_process, recompile_in_process_after_edit, replace_project_text, @@ -50,6 +54,11 @@ ) from tests.integration.src.sqlbuild.compiler.compile.helpers import MACRO_CALL_LOG_ENV_VAR +_MACRO_REFERENCE_CALL_RUNS: int = 2 +_MACRO_REFERENCE_CALL_MESSAGE: str = ( + "__ref(customers) is not a valid __ref() call, returned by macro customer_relation()" +) +_MACRO_REFERENCE_CALL_PATH: str = "models/marts/_sqlbuild/_macros/customer_relation.py" _EDIT_STEPS: tuple[MacroCallStoreEditStep, ...] = ( MacroCallStoreEditStep( description="no_change", @@ -305,6 +314,48 @@ def test_given_engine_when_compiling_repeatedly_then_only_preview_batches_and_st assert runs.store_files == test_case.expected_store_files +@pytest.mark.parametrize( + "test_case", + [ + MacroReferenceCallStoreTestCase( + description="python", + engine="python", + expected_logged_calls=(2, 2), + ), + MacroReferenceCallStoreTestCase( + description="native_preview_memo_then_store", + engine="native-preview", + expected_logged_calls=(1, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_returning_rejected_reference_call_when_replaying_then_reports_match_python( + tmp_path: Path, test_case: MacroReferenceCallStoreTestCase +) -> None: + oracle: MacroReferenceCallRuns = macro_reference_call_runs( + project_dir=prepare_macro_reference_call_project(tmp_path=tmp_path / "python"), + log_path=tmp_path / "python.log", + engine="python", + runs=1, + ) + + runs: MacroReferenceCallRuns = macro_reference_call_runs( + project_dir=prepare_macro_reference_call_project(tmp_path=tmp_path / "engine"), + log_path=tmp_path / "engine.log", + engine=test_case.engine, + runs=_MACRO_REFERENCE_CALL_RUNS, + ) + + assert [diagnostic[:4] for diagnostic in oracle.diagnostics[0]] == [ + ("P012", _MACRO_REFERENCE_CALL_MESSAGE, name, _MACRO_REFERENCE_CALL_PATH) + for name in ("customer_orders", "customer_returns") + ] + assert runs.returncodes == (1,) * _MACRO_REFERENCE_CALL_RUNS + assert runs.logged_calls == test_case.expected_logged_calls + assert runs.diagnostics == oracle.diagnostics * _MACRO_REFERENCE_CALL_RUNS + + @pytest.mark.parametrize( "test_case", [ From 331adbc8bd4715db1719a54db20c58d58cb3b919 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:05:49 +0100 Subject: [PATCH 11/11] refactor(native): raise the shared native stage mismatch error from the macro bridge --- .../compiler/compile/_helpers/macro_bridge/expansion.py | 9 +++++---- src/sqlbuild/compiler/compile/exceptions.py | 4 ---- .../compiler/macro_bridge/main/run_with_macro_bridge.py | 4 ++-- .../macro_bridge/main/test_run_with_macro_bridge.py | 5 +++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py index a9f48cdcf3..6c46bb639c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -14,12 +14,13 @@ ) from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import MACRO_TOKEN -from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeRenderMismatchError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationResolutionContext, ExpansionSpan, MacroExpansionState, ) +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ( ARGUMENT_REFERENCE_EVENT, @@ -139,7 +140,7 @@ def _bridged_call_output( # noqa: PLR0913 "directly in SQL" ) if next_index != site.end: - raise NativeRenderMismatchError( + raise NativeStageMismatchError( f"Native macro call scan of '{consumer_path}' ended at {site.end}, Python at " f"{next_index}" ) @@ -177,7 +178,7 @@ def _replay_macro_call_event( ) elif tag == DECLARATION_READ_EVENT: if declarations is None: - raise NativeRenderMismatchError("Replayed a declaration read without declarations") + raise NativeStageMismatchError("Replayed a declaration read without declarations") _record_macro_declaration_usage( name=second, kind=DeclarationKind(first), declarations=declarations, state=state ) @@ -198,4 +199,4 @@ def _replay_macro_call_event( (SqlResourceRef(kind=SqlResourceRefKind(first), name=second),) ) else: - raise NativeRenderMismatchError(f"Unknown macro call event {tag}") + raise NativeStageMismatchError(f"Unknown macro call event {tag}") diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 2dbd139c49..5d849a9b26 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -19,10 +19,6 @@ class MacroDeclarationLookupError(CompileInputError, KeyError): """Retain authored diagnostics while honoring the Mapping lookup contract.""" -class NativeRenderMismatchError(RuntimeError): - """Raised when native rendering disagrees with Python, so the Python stage must re-run.""" - - class NativeSqlTestResponseError(CompileInputError): """Raised when native SQL-test extraction returns malformed data.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py index 030621e48e..70b7ee99e6 100644 --- a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py @@ -9,7 +9,7 @@ from contextvars import Token import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import NativeRenderMismatchError +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE @@ -46,7 +46,7 @@ def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: finally: ACTIVE_MACRO_BRIDGE.reset(token) _ = stage() - raise NativeRenderMismatchError( + raise NativeStageMismatchError( "The native macro bridge failed where the Python render succeeded; " "run with SQLBUILD_COMPILER_ENGINE=python" ) from native_error diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py index d529893f7e..5b556aeb0d 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py @@ -4,7 +4,8 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeRenderMismatchError +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge from tests.unit.src.sqlbuild.compiler.macro_bridge.main._test_types import ( @@ -58,7 +59,7 @@ def test_given_succeeding_stage_when_running_with_macro_bridge_then_runs_once_br description="native-only failure is reported as a divergence", stage_with_bridge=failed, stage_without_bridge=rendered, - expected_error=NativeRenderMismatchError, + expected_error=NativeStageMismatchError, expected_bridged_runs=[True, False], ), ],