feat(native): resolve model configuration natively behind the preview engine

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "crates/sqlbuild-config",
     "crates/sqlbuild-discovery",
     "crates/sqlbuild-scopes",
+    "crates/sqlbuild-model-config",
     "crates/sqlbuild-analysis",
     "crates/sqlbuild-rules",
     "crates/sqlbuild-python",
@@ -21,6 +22,7 @@ order = [
     "sqlbuild-config",
     "sqlbuild-discovery",
     "sqlbuild-scopes",
+    "sqlbuild-model-config",
     "sqlbuild-analysis",
     "sqlbuild-rules",
     "sqlbuild-python",
@@ -70,6 +72,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" }
 sqlbuild-core = { path = "crates/sqlbuild-core" }
 sqlbuild-discovery = { path = "crates/sqlbuild-discovery" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }
+sqlbuild-model-config = { path = "crates/sqlbuild-model-config" }
 sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
 sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 stacker = "0.1"

crates/README.md

@@ -11,6 +11,7 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. |
 | `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. |
 | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. |
+| `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. |
 | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. |
 | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. |
 | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. |
@@ -21,12 +22,13 @@ Dependencies point one way, from the top of this graph to the bottom:
 
 ```text
 sqlbuild-python
-  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config
-  -> sqlbuild-sqltext -> sqlbuild-core
+  -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-scopes
+  -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core
 ```
 
-`sqlbuild-config` does not depend on the crates below it today, and `sqlbuild-scopes` uses only
-`sqlbuild-sqltext`; their place in the order fixes which crates may use them. The JSON emitter
+`sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and
+`sqlbuild-scopes` uses only `sqlbuild-sqltext`; their place in the order fixes which crates may
+use them. The JSON emitter
 lives in `sqlbuild-core` so that any layer can produce text that must equal Python's
 `json.dumps` output.
 

crates/sqlbuild-model-config/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "sqlbuild-model-config"
+version.workspace = true
+description = "Native model configuration for SQLBuild: MODEL header metadata parsed with Python's exact rules."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs

@@ -0,0 +1,42 @@
+//! Python's `@[A-Za-z_][A-Za-z0-9_]*\s*\(` search over one string.
+
+use crate::config_presence::models::Presence;
+
+/// Search `text` for a macro call opening, deferring where Python's `\s` decides.
+pub(crate) fn macro_call(text: &str) -> Presence {
+    let bytes = text.as_bytes();
+    let mut deferred = false;
+    for (at, _) in text.match_indices('@') {
+        let mut index = at + 1;
+        if !bytes
+            .get(index)
+            .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_')
+        {
+            continue;
+        }
+        while bytes
+            .get(index)
+            .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_')
+        {
+            index += 1;
+        }
+        while bytes.get(index).copied().is_some_and(is_python_ascii_space) {
+            index += 1;
+        }
+        match bytes.get(index) {
+            Some(b'(') => return Presence::Present,
+            Some(byte) if !byte.is_ascii() => deferred = true,
+            _ => {}
+        }
+    }
+    if deferred {
+        Presence::Deferred
+    } else {
+        Presence::Absent
+    }
+}
+
+/// ASCII characters Python's `\s` matches in a `str` pattern.
+fn is_python_ascii_space(byte: u8) -> bool {
+    matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ')
+}

crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs

@@ -0,0 +1,3 @@
+pub(crate) mod macro_calls;
+pub(crate) mod scan;
+pub(crate) mod template_tokens;

crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs

@@ -0,0 +1,36 @@
+//! The recursive config scan shared by every presence check.
+
+use crate::config_presence::models::Presence;
+use crate::types::{AuthoredNode, NodeKind};
+
+/// Scan mapping values, list and tuple items and strings like Python's recursive config scans.
+pub(crate) fn scan<N: AuthoredNode>(
+    node: &N,
+    text_presence: &impl Fn(&str) -> Presence,
+) -> Presence {
+    match node.kind() {
+        NodeKind::Str => node.with_text(text_presence).unwrap_or(Presence::Deferred),
+        NodeKind::Map => any_present(node.entries().iter().map(|(_, value)| value), text_presence),
+        NodeKind::List | NodeKind::Tuple => any_present(node.items().iter(), text_presence),
+        _ => Presence::Absent,
+    }
+}
+
+fn any_present<'node, N: AuthoredNode + 'node>(
+    nodes: impl Iterator<Item = &'node N>,
+    text_presence: &impl Fn(&str) -> Presence,
+) -> Presence {
+    let mut deferred = false;
+    for node in nodes {
+        match scan(node, text_presence) {
+            Presence::Present => return Presence::Present,
+            Presence::Deferred => deferred = true,
+            Presence::Absent => {}
+        }
+    }
+    if deferred {
+        Presence::Deferred
+    } else {
+        Presence::Absent
+    }
+}

crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs

@@ -0,0 +1,13 @@
+//! Python's `TEMPLATE_OPEN_TOKEN in value` over one string.
+
+use crate::config_presence::constants::TEMPLATE_OPEN_TOKEN;
+use crate::config_presence::models::Presence;
+
+/// Return whether `text` holds a template opening.
+pub(crate) fn template_token(text: &str) -> Presence {
+    if text.contains(TEMPLATE_OPEN_TOKEN) {
+        Presence::Present
+    } else {
+        Presence::Absent
+    }
+}

crates/sqlbuild-model-config/src/config_presence/constants.rs

@@ -0,0 +1,4 @@
+//! Tokens the config presence scans look for.
+
+/// The opening of a `${...}` template.
+pub const TEMPLATE_OPEN_TOKEN: &str = "${";

crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs

@@ -0,0 +1,11 @@
+//! Recursive macro call presence over authored config values.
+
+use crate::config_presence::_helpers::macro_calls::macro_call;
+use crate::config_presence::_helpers::scan::scan;
+use crate::config_presence::models::Presence;
+use crate::types::AuthoredNode;
+
+/// Return whether `MACRO_CALL_PATTERN` matches any nested string, or defer to Python.
+pub fn contains_macro_call<N: AuthoredNode>(node: &N) -> Presence {
+    scan(node, &macro_call)
+}

crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs

@@ -0,0 +1,11 @@
+//! Recursive template presence over authored config values.
+
+use crate::config_presence::_helpers::scan::scan;
+use crate::config_presence::_helpers::template_tokens::template_token;
+use crate::config_presence::models::Presence;
+use crate::types::AuthoredNode;
+
+/// Return whether `contains_template_data` finds `${` in any nested string, or defer to Python.
+pub fn contains_template<N: AuthoredNode>(node: &N) -> Presence {
+    scan(node, &template_token)
+}

crates/sqlbuild-model-config/src/config_presence/main/mod.rs

@@ -0,0 +1,2 @@
+pub mod contains_macro_call;
+pub mod contains_template;

crates/sqlbuild-model-config/src/config_presence/mod.rs

@@ -0,0 +1,7 @@
+pub(crate) mod _helpers;
+pub mod constants;
+pub mod main;
+pub mod models;
+
+#[cfg(test)]
+mod tests;

crates/sqlbuild-model-config/src/config_presence/models.rs

@@ -0,0 +1,10 @@
+//! Whether authored config contains a construct, or that only Python can tell.
+
+/// The answer Python's recursive config scan gives, or a deferral to that scan.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Presence {
+    Present,
+    Absent,
+    /// A string needs Python's Unicode `\s` class to decide.
+    Deferred,
+}

crates/sqlbuild-model-config/src/config_presence/tests.rs

@@ -0,0 +1,4 @@
+#[path = "tests/test_config_presence.rs"]
+mod config_presence;
+#[path = "tests/test_types.rs"]
+mod test_types;

crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs

@@ -0,0 +1,76 @@
+use crate::config_presence::main::contains_macro_call::contains_macro_call;
+use crate::config_presence::main::contains_template::contains_template;
+use crate::config_presence::models::Presence;
+use crate::config_presence::tests::test_types::PresenceTestCase;
+use crate::tests::test_types::Value;
+
+#[test]
+fn given_config_values_when_scanning_then_presence_matches_python_or_defers() {
+    let test_cases = [
+        PresenceTestCase {
+            description: "plain scalars contain nothing",
+            value: Value::List(vec![
+                Value::Int(1),
+                Value::Bool(true),
+                Value::Null,
+                Value::Float,
+            ]),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Absent,
+        },
+        PresenceTestCase {
+            description: "nested mapping values are scanned, keys are not",
+            value: Value::Map(vec![
+                (Value::Str("${key}"), Value::Str("plain")),
+                (
+                    Value::Str("columns"),
+                    Value::Tuple(vec![Value::Str("@cents (amount)")]),
+                ),
+            ]),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Present,
+        },
+        PresenceTestCase {
+            description: "a template token anywhere in a string",
+            value: Value::Str("prefix_${target.schema}"),
+            expected_template: Presence::Present,
+            expected_macro_call: Presence::Absent,
+        },
+        PresenceTestCase {
+            description: "an at-name without a call or with an invalid start is no macro",
+            value: Value::Str("mail@example.com @1x( @ (x)"),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Absent,
+        },
+        PresenceTestCase {
+            description: "Python separators count as whitespace before the call",
+            value: Value::Str("@m\u{1f}\t("),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Present,
+        },
+        PresenceTestCase {
+            description: "non-ASCII after a macro name defers to Python's whitespace class",
+            value: Value::Str("@m\u{a0}("),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Deferred,
+        },
+        PresenceTestCase {
+            description: "a later call outweighs an earlier deferral",
+            value: Value::List(vec![Value::Str("@m\u{2003}("), Value::Str("@n(")]),
+            expected_template: Presence::Absent,
+            expected_macro_call: Presence::Present,
+        },
+    ];
+
+    for test_case in test_cases {
+        assert_eq!(
+            (
+                contains_template(&test_case.value),
+                contains_macro_call(&test_case.value)
+            ),
+            (test_case.expected_template, test_case.expected_macro_call),
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs

@@ -0,0 +1,10 @@
+use crate::config_presence::models::Presence;
+use crate::tests::test_types::Value;
+
+/// One authored config value and what each recursive scan finds in it.
+pub(super) struct PresenceTestCase {
+    pub(super) description: &'static str,
+    pub(super) value: Value,
+    pub(super) expected_template: Presence,
+    pub(super) expected_macro_call: Presence,
+}