fix(compiler)!: reject unknown schema-change values and follow dialect quoting in compile scanners

crates/sqlbuild-rules-native/Cargo.toml

@@ -30,6 +30,9 @@ stacker.workspace = true
 sqlparser.workspace = true
 toml.workspace = true
 
+[build-dependencies]
+sha2.workspace = true
+
 [dev-dependencies]
 tempfile.workspace = true
 

crates/sqlbuild-rules-native/build.rs

@@ -0,0 +1,81 @@
+//! Derive a content identity for this native build so local caches notice source changes.
+//!
+//! The identity hashes every file under the listed crate entries plus the workspace lockfile,
+//! keyed by `/`-separated paths relative to the crate root so it matches across platforms.
+//! Entries that do not exist (for example the lockfile outside a workspace checkout) are left
+//! out of both the hash and the `rerun-if-changed` list, because Cargo reruns a build script on
+//! every build when a watched path is missing.
+
+use sha2::{Digest, Sha256};
+use std::path::{Component, Path, PathBuf};
+
+const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"];
+const WORKSPACE_LOCKFILE: &str = "../../Cargo.lock";
+
+fn main() -> Result<(), String> {
+    let crate_dir =
+        PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?);
+    let mut files: Vec<(String, PathBuf)> = Vec::new();
+    for entry in HASHED_ENTRIES {
+        let path = crate_dir.join(entry);
+        if path.exists() {
+            println!("cargo:rerun-if-changed={}", path.display());
+            collect_files(&crate_dir, &path, &mut files)?;
+        }
+    }
+    files.sort();
+    let lockfile = crate_dir.join(WORKSPACE_LOCKFILE);
+    if lockfile.is_file() {
+        println!("cargo:rerun-if-changed={}", lockfile.display());
+        files.push((WORKSPACE_LOCKFILE.to_owned(), lockfile));
+    }
+    let mut digest = Sha256::new();
+    for (label, file) in &files {
+        digest.update(label.as_bytes());
+        digest.update([0]);
+        digest.update(std::fs::read(file).map_err(|error| error.to_string())?);
+        digest.update([0]);
+    }
+    let identity = format!("{:x}", digest.finalize());
+    println!(
+        "cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}",
+        &identity[..16]
+    );
+    Ok(())
+}
+
+fn portable_label(crate_dir: &Path, path: &Path) -> Result<String, String> {
+    let relative = path
+        .strip_prefix(crate_dir)
+        .map_err(|error| error.to_string())?;
+    let parts: Vec<String> = relative
+        .components()
+        .map(|component| match component {
+            Component::Normal(part) => Ok(part.to_string_lossy().into_owned()),
+            other => Err(format!("unexpected path component {other:?}")),
+        })
+        .collect::<Result<_, _>>()?;
+    Ok(parts.join("/"))
+}
+
+fn collect_files(
+    crate_dir: &Path,
+    path: &Path,
+    files: &mut Vec<(String, PathBuf)>,
+) -> Result<(), String> {
+    if path.is_file() {
+        files.push((portable_label(crate_dir, path)?, path.to_path_buf()));
+        return Ok(());
+    }
+    if !path.is_dir() {
+        return Ok(());
+    }
+    for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? {
+        collect_files(
+            crate_dir,
+            &entry.map_err(|error| error.to_string())?.path(),
+            files,
+        )?;
+    }
+    Ok(())
+}

crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs

@@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple};
 use pyo3::{FromPyObject, pyfunction, wrap_pyfunction};
 
 use crate::configuration::main::load;
-use crate::constants::API_VERSION;
+use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY};
 use crate::engine::main::evaluate;
 use crate::models::CatalogueResponse;
 use crate::rules::main::{catalogue, selected_codes};
@@ -469,5 +469,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?;
     module.add_function(wrap_pyfunction!(skill_freshness, module)?)?;
     module.add("API_VERSION", API_VERSION)?;
+    module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?;
     Ok(())
 }

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs

@@ -242,7 +242,7 @@ fn is_generated_cte_name(name: &str) -> bool {
 
 /// Distinct lowercase identifiers outside strings and comments, in first-appearance order.
 fn identifier_tokens(sql: &str, patterns: &SqlTestPatterns) -> Vec<String> {
-    let protected = protected_ranges(&patterns.protected, sql);
+    let protected = protected_ranges(&patterns.lexical, sql);
     let mut seen: HashSet<String> = HashSet::new();
     let mut tokens: Vec<String> = Vec::new();
     let mut push = |token: &str| {

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs

@@ -5,21 +5,21 @@ use std::collections::HashSet;
 use regex::{Captures, Regex};
 
 use crate::compiler::_helpers::sql_tests::planning::compile_error;
-use crate::sql_scan::main::matching_paren::matching_paren;
+use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
+use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
 use crate::sql_scan::main::skip_whitespace::skip_whitespace;
-use crate::sql_scan::models::QuotePolicy;
-use crate::sql_scan::models::Unclosed;
+use crate::sql_scan::models::{LexicalSyntax, Unclosed};
 
 pub(crate) fn replace_callable_markers<F>(
     sql: &str,
     pattern: &Regex,
-    protected_pattern: &Regex,
+    syntax: &LexicalSyntax,
     mut replacement: F,
 ) -> Result<(String, HashSet<String>), String>
 where
     F: FnMut(&str, &str) -> Option<(String, bool)>,
 {
-    let protected = protected_ranges(protected_pattern, sql);
+    let protected = protected_ranges(syntax, sql);
     let mut output = String::with_capacity(sql.len());
     let mut reached: HashSet<String> = HashSet::new();
     let mut cursor = 0;
@@ -34,7 +34,7 @@ where
         if !sql[suffix_start..].starts_with('(') {
             continue;
         }
-        let suffix_end = matching_paren_end(sql, suffix_start)?;
+        let suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
         let Some(name) = captures.get(1).map(|value| value.as_str()) else {
             continue;
         };
@@ -53,8 +53,8 @@ where
     Ok((output, reached))
 }
 
-fn matching_paren_end(sql: &str, open: usize) -> Result<usize, String> {
-    matching_paren(sql.as_bytes(), open, QuotePolicy::COMPILER)
+fn matching_paren_end(sql: &str, open: usize, syntax: &LexicalSyntax) -> Result<usize, String> {
+    dialect_matching_paren(sql.as_bytes(), open, syntax)
         .map(|close| close + 1)
         .map_err(|error| {
             compile_error(match error {
@@ -68,27 +68,27 @@ fn matching_paren_end(sql: &str, open: usize) -> Result<usize, String> {
 pub(crate) fn replace_named_markers<F>(
     sql: &str,
     pattern: &Regex,
-    protected_pattern: &Regex,
+    syntax: &LexicalSyntax,
     mut replacement: F,
 ) -> String
 where
     F: FnMut(&str) -> Option<String>,
 {
-    replace_marker_captures(sql, pattern, protected_pattern, |captures| {
+    replace_marker_captures(sql, pattern, syntax, |captures| {
         replacement(captures.get(1)?.as_str())
     })
 }
 
 pub(crate) fn replace_dbt_ref_markers<F>(
     sql: &str,
     pattern: &Regex,
-    protected_pattern: &Regex,
+    syntax: &LexicalSyntax,
     mut replacement: F,
 ) -> String
 where
     F: FnMut(&str) -> Option<String>,
 {
-    replace_marker_captures(sql, pattern, protected_pattern, |captures| {
+    replace_marker_captures(sql, pattern, syntax, |captures| {
         let first = captures.get(1)?.as_str();
         let name = captures.get(2).map_or_else(
             || first.to_string(),
@@ -101,13 +101,13 @@ where
 fn replace_marker_captures<F>(
     sql: &str,
     pattern: &Regex,
-    protected_pattern: &Regex,
+    syntax: &LexicalSyntax,
     mut replacement: F,
 ) -> String
 where
     F: FnMut(&Captures<'_>) -> Option<String>,
 {
-    let protected = protected_ranges(protected_pattern, sql);
+    let protected = protected_ranges(syntax, sql);
     let mut output = String::with_capacity(sql.len());
     let mut cursor = 0;
     for captures in pattern.captures_iter(sql) {
@@ -128,8 +128,8 @@ where
     output
 }
 
-pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec<String> {
-    let protected = protected_ranges(protected_pattern, sql);
+pub(crate) fn marker_names(pattern: &Regex, syntax: &LexicalSyntax, sql: &str) -> Vec<String> {
+    let protected = protected_ranges(syntax, sql);
     let mut names: Vec<String> = Vec::new();
     for captures in pattern.captures_iter(sql) {
         let Some(full) = captures.get(0) else {
@@ -145,11 +145,8 @@ pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str
     names
 }
 
-pub(crate) fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> {
-    pattern
-        .find_iter(sql)
-        .map(|value| (value.start(), value.end()))
-        .collect()
+pub(crate) fn protected_ranges(syntax: &LexicalSyntax, sql: &str) -> Vec<(usize, usize)> {
+    dialect_non_code_ranges(sql, syntax)
 }
 
 pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool {