From f17c89b6e7adfce1a80f1c8fb7725084261cb592 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:52:38 +0100 Subject: [PATCH 1/3] feat(native): add the native model config crate Parses MODEL header columns and audits, expands ${...} config templates and scans config for templates and macro calls with the Python compiler's exact rules, deferring to Python wherever it cannot reproduce them. --- Cargo.lock | 5 + Cargo.toml | 3 + crates/README.md | 10 +- crates/sqlbuild-model-config/Cargo.toml | 11 ++ .../config_presence/_helpers/macro_calls.rs | 42 +++++ .../src/config_presence/_helpers/mod.rs | 3 + .../src/config_presence/_helpers/scan.rs | 36 ++++ .../_helpers/template_tokens.rs | 13 ++ .../src/config_presence/constants.rs | 4 + .../main/contains_macro_call.rs | 11 ++ .../config_presence/main/contains_template.rs | 11 ++ .../src/config_presence/main/mod.rs | 2 + .../src/config_presence/mod.rs | 7 + .../src/config_presence/models.rs | 10 ++ .../src/config_presence/tests.rs | 4 + .../tests/test_config_presence.rs | 76 +++++++++ .../src/config_presence/tests/test_types.rs | 10 ++ .../src/header_metadata/_helpers/audits.rs | 108 ++++++++++++ .../src/header_metadata/_helpers/columns.rs | 86 ++++++++++ .../src/header_metadata/_helpers/mod.rs | 3 + .../src/header_metadata/_helpers/text.rs | 86 ++++++++++ .../src/header_metadata/constants.rs | 20 +++ .../src/header_metadata/main/mod.rs | 1 + .../main/parse_header_metadata.rs | 17 ++ .../src/header_metadata/mod.rs | 7 + .../src/header_metadata/models.rs | 44 +++++ .../src/header_metadata/tests.rs | 6 + .../src/header_metadata/tests/helpers.rs | 77 +++++++++ .../tests/test_parse_header_metadata.rs | 161 ++++++++++++++++++ .../src/header_metadata/tests/test_types.rs | 11 ++ crates/sqlbuild-model-config/src/lib.rs | 11 ++ .../src/templates/_helpers/evaluation.rs | 133 +++++++++++++++ .../src/templates/_helpers/expressions.rs | 77 +++++++++ .../src/templates/_helpers/mod.rs | 4 + .../src/templates/_helpers/pattern.rs | 27 +++ .../src/templates/_helpers/tokens.rs | 75 ++++++++ .../templates/main/expand_template_string.rs | 37 ++++ .../src/templates/main/mod.rs | 1 + .../src/templates/mod.rs | 7 + .../src/templates/models.rs | 60 +++++++ .../src/templates/tests.rs | 6 + .../src/templates/tests/helpers.rs | 12 ++ .../tests/test_expand_template_string.rs | 128 ++++++++++++++ .../src/templates/tests/test_types.rs | 81 +++++++++ .../src/templates/types.rs | 24 +++ crates/sqlbuild-model-config/src/tests.rs | 2 + .../src/tests/test_types.rs | 63 +++++++ crates/sqlbuild-model-config/src/types.rs | 34 ++++ crates/sqlbuild-python/Cargo.toml | 1 + fensu.toml | 1 + 50 files changed, 1665 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-model-config/Cargo.toml create mode 100644 crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/constants.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/main/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/models.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/tests.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/constants.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/mod.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/parse_header_metadata.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/mod.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/models.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/tests.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/tests/test_parse_header_metadata.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/lib.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/mod.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs create mode 100644 crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs create mode 100644 crates/sqlbuild-model-config/src/templates/main/mod.rs create mode 100644 crates/sqlbuild-model-config/src/templates/mod.rs create mode 100644 crates/sqlbuild-model-config/src/templates/models.rs create mode 100644 crates/sqlbuild-model-config/src/templates/tests.rs create mode 100644 crates/sqlbuild-model-config/src/templates/tests/helpers.rs create mode 100644 crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs create mode 100644 crates/sqlbuild-model-config/src/templates/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/templates/types.rs create mode 100644 crates/sqlbuild-model-config/src/tests.rs create mode 100644 crates/sqlbuild-model-config/src/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index ab2ed994fd..5aaef98134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -671,6 +671,10 @@ dependencies = [ "tempfile", ] +[[package]] +name = "sqlbuild-model-config" +version = "0.139.0" + [[package]] name = "sqlbuild-python" version = "0.139.0" @@ -683,6 +687,7 @@ dependencies = [ "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", + "sqlbuild-model-config", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", diff --git a/Cargo.toml b/Cargo.toml index 062b1353d6..92a3dbc5e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/sqlbuild-config", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", + "crates/sqlbuild-model-config", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -21,6 +22,7 @@ order = [ "sqlbuild-config", "sqlbuild-discovery", "sqlbuild-scopes", + "sqlbuild-model-config", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -70,6 +72,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } +sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } stacker = "0.1" diff --git a/crates/README.md b/crates/README.md index 9b52dca374..802a48b82d 100644 --- a/crates/README.md +++ b/crates/README.md @@ -11,6 +11,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. | | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -21,12 +22,13 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config - -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-scopes + -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core ``` -`sqlbuild-config` does not depend on the crates below it today, and `sqlbuild-scopes` uses only -`sqlbuild-sqltext`; their place in the order fixes which crates may use them. The JSON emitter +`sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and +`sqlbuild-scopes` uses only `sqlbuild-sqltext`; their place in the order fixes which crates may +use them. The JSON emitter lives in `sqlbuild-core` so that any layer can produce text that must equal Python's `json.dumps` output. diff --git a/crates/sqlbuild-model-config/Cargo.toml b/crates/sqlbuild-model-config/Cargo.toml new file mode 100644 index 0000000000..a4cf9e2917 --- /dev/null +++ b/crates/sqlbuild-model-config/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "sqlbuild-model-config" +version.workspace = true +description = "Native model configuration for SQLBuild: MODEL header metadata parsed with Python's exact rules." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs new file mode 100644 index 0000000000..2d8517fa20 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs @@ -0,0 +1,42 @@ +//! Python's `@[A-Za-z_][A-Za-z0-9_]*\s*\(` search over one string. + +use crate::config_presence::models::Presence; + +/// Search `text` for a macro call opening, deferring where Python's `\s` decides. +pub(crate) fn macro_call(text: &str) -> Presence { + let bytes = text.as_bytes(); + let mut deferred = false; + for (at, _) in text.match_indices('@') { + let mut index = at + 1; + if !bytes + .get(index) + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + { + continue; + } + while bytes + .get(index) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + index += 1; + } + while bytes.get(index).copied().is_some_and(is_python_ascii_space) { + index += 1; + } + match bytes.get(index) { + Some(b'(') => return Presence::Present, + Some(byte) if !byte.is_ascii() => deferred = true, + _ => {} + } + } + if deferred { + Presence::Deferred + } else { + Presence::Absent + } +} + +/// ASCII characters Python's `\s` matches in a `str` pattern. +fn is_python_ascii_space(byte: u8) -> bool { + matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') +} diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs new file mode 100644 index 0000000000..afb06b5ca1 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod macro_calls; +pub(crate) mod scan; +pub(crate) mod template_tokens; diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs new file mode 100644 index 0000000000..34669a21a2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs @@ -0,0 +1,36 @@ +//! The recursive config scan shared by every presence check. + +use crate::config_presence::models::Presence; +use crate::types::{AuthoredNode, NodeKind}; + +/// Scan mapping values, list and tuple items and strings like Python's recursive config scans. +pub(crate) fn scan( + node: &N, + text_presence: &impl Fn(&str) -> Presence, +) -> Presence { + match node.kind() { + NodeKind::Str => node.with_text(text_presence).unwrap_or(Presence::Deferred), + NodeKind::Map => any_present(node.entries().iter().map(|(_, value)| value), text_presence), + NodeKind::List | NodeKind::Tuple => any_present(node.items().iter(), text_presence), + _ => Presence::Absent, + } +} + +fn any_present<'node, N: AuthoredNode + 'node>( + nodes: impl Iterator, + text_presence: &impl Fn(&str) -> Presence, +) -> Presence { + let mut deferred = false; + for node in nodes { + match scan(node, text_presence) { + Presence::Present => return Presence::Present, + Presence::Deferred => deferred = true, + Presence::Absent => {} + } + } + if deferred { + Presence::Deferred + } else { + Presence::Absent + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs new file mode 100644 index 0000000000..dcf2a1d7f5 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs @@ -0,0 +1,13 @@ +//! Python's `TEMPLATE_OPEN_TOKEN in value` over one string. + +use crate::config_presence::constants::TEMPLATE_OPEN_TOKEN; +use crate::config_presence::models::Presence; + +/// Return whether `text` holds a template opening. +pub(crate) fn template_token(text: &str) -> Presence { + if text.contains(TEMPLATE_OPEN_TOKEN) { + Presence::Present + } else { + Presence::Absent + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/constants.rs b/crates/sqlbuild-model-config/src/config_presence/constants.rs new file mode 100644 index 0000000000..2d33d581a0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/constants.rs @@ -0,0 +1,4 @@ +//! Tokens the config presence scans look for. + +/// The opening of a `${...}` template. +pub const TEMPLATE_OPEN_TOKEN: &str = "${"; diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs new file mode 100644 index 0000000000..ba6326a391 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs @@ -0,0 +1,11 @@ +//! Recursive macro call presence over authored config values. + +use crate::config_presence::_helpers::macro_calls::macro_call; +use crate::config_presence::_helpers::scan::scan; +use crate::config_presence::models::Presence; +use crate::types::AuthoredNode; + +/// Return whether `MACRO_CALL_PATTERN` matches any nested string, or defer to Python. +pub fn contains_macro_call(node: &N) -> Presence { + scan(node, ¯o_call) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs new file mode 100644 index 0000000000..bacc702131 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs @@ -0,0 +1,11 @@ +//! Recursive template presence over authored config values. + +use crate::config_presence::_helpers::scan::scan; +use crate::config_presence::_helpers::template_tokens::template_token; +use crate::config_presence::models::Presence; +use crate::types::AuthoredNode; + +/// Return whether `contains_template_data` finds `${` in any nested string, or defer to Python. +pub fn contains_template(node: &N) -> Presence { + scan(node, &template_token) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs new file mode 100644 index 0000000000..36a864d560 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -0,0 +1,2 @@ +pub mod contains_macro_call; +pub mod contains_template; diff --git a/crates/sqlbuild-model-config/src/config_presence/mod.rs b/crates/sqlbuild-model-config/src/config_presence/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/config_presence/models.rs b/crates/sqlbuild-model-config/src/config_presence/models.rs new file mode 100644 index 0000000000..cecf2b46d0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/models.rs @@ -0,0 +1,10 @@ +//! Whether authored config contains a construct, or that only Python can tell. + +/// The answer Python's recursive config scan gives, or a deferral to that scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Presence { + Present, + Absent, + /// A string needs Python's Unicode `\s` class to decide. + Deferred, +} diff --git a/crates/sqlbuild-model-config/src/config_presence/tests.rs b/crates/sqlbuild-model-config/src/config_presence/tests.rs new file mode 100644 index 0000000000..965b2a21d9 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_config_presence.rs"] +mod config_presence; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs new file mode 100644 index 0000000000..5bf585a298 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -0,0 +1,76 @@ +use crate::config_presence::main::contains_macro_call::contains_macro_call; +use crate::config_presence::main::contains_template::contains_template; +use crate::config_presence::models::Presence; +use crate::config_presence::tests::test_types::PresenceTestCase; +use crate::tests::test_types::Value; + +#[test] +fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { + let test_cases = [ + PresenceTestCase { + description: "plain scalars contain nothing", + value: Value::List(vec![ + Value::Int(1), + Value::Bool(true), + Value::Null, + Value::Float, + ]), + expected_template: Presence::Absent, + expected_macro_call: Presence::Absent, + }, + PresenceTestCase { + description: "nested mapping values are scanned, keys are not", + value: Value::Map(vec![ + (Value::Str("${key}"), Value::Str("plain")), + ( + Value::Str("columns"), + Value::Tuple(vec![Value::Str("@cents (amount)")]), + ), + ]), + expected_template: Presence::Absent, + expected_macro_call: Presence::Present, + }, + PresenceTestCase { + description: "a template token anywhere in a string", + value: Value::Str("prefix_${target.schema}"), + expected_template: Presence::Present, + expected_macro_call: Presence::Absent, + }, + PresenceTestCase { + description: "an at-name without a call or with an invalid start is no macro", + value: Value::Str("mail@example.com @1x( @ (x)"), + expected_template: Presence::Absent, + expected_macro_call: Presence::Absent, + }, + PresenceTestCase { + description: "Python separators count as whitespace before the call", + value: Value::Str("@m\u{1f}\t("), + expected_template: Presence::Absent, + expected_macro_call: Presence::Present, + }, + PresenceTestCase { + description: "non-ASCII after a macro name defers to Python's whitespace class", + value: Value::Str("@m\u{a0}("), + expected_template: Presence::Absent, + expected_macro_call: Presence::Deferred, + }, + PresenceTestCase { + description: "a later call outweighs an earlier deferral", + value: Value::List(vec![Value::Str("@m\u{2003}("), Value::Str("@n(")]), + expected_template: Presence::Absent, + expected_macro_call: Presence::Present, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + contains_template(&test_case.value), + contains_macro_call(&test_case.value) + ), + (test_case.expected_template, test_case.expected_macro_call), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs new file mode 100644 index 0000000000..917d2a3746 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -0,0 +1,10 @@ +use crate::config_presence::models::Presence; +use crate::tests::test_types::Value; + +/// One authored config value and what each recursive scan finds in it. +pub(super) struct PresenceTestCase { + pub(super) description: &'static str, + pub(super) value: Value, + pub(super) expected_template: Presence, + pub(super) expected_macro_call: Presence, +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs new file mode 100644 index 0000000000..3431fa0d4f --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs @@ -0,0 +1,108 @@ +//! Audit lists and instances, as `parse_audit_instances` reads them with `null_as_empty`. + +use crate::header_metadata::_helpers::text::{ + entry, is_snake_case, non_blank_text, optional_bool, optional_count, optional_text, +}; +use crate::header_metadata::constants::{AUDIT_OPTION_KEYS, AUDIT_SEVERITIES}; +use crate::header_metadata::models::{HeaderMetadataDeferral, ParsedAudit}; +use crate::types::{AuthoredNode, NodeKind}; + +/// Parse an authored audit list; Python's `None` is an empty list. +pub(crate) fn audit_list( + node: &N, +) -> Result>, HeaderMetadataDeferral> { + match node.kind() { + NodeKind::Null => Ok(Vec::new()), + NodeKind::List => node.items().iter().map(audit_instance).collect(), + _ => Err(HeaderMetadataDeferral::Invalid), + } +} + +fn audit_instance(node: &N) -> Result, HeaderMetadataDeferral> { + match node.kind() { + NodeKind::Str => { + identity(node)?; + Ok(bare_audit(node.clone())) + } + NodeKind::Map => { + let entries = node.entries(); + let [(definition_name, arguments)] = entries.as_slice() else { + return Err(HeaderMetadataDeferral::Invalid); + }; + identity(definition_name)?; + match arguments.kind() { + NodeKind::Null => Ok(bare_audit(definition_name.clone())), + NodeKind::Map => configured_audit(definition_name, &arguments.entries()), + _ => Err(HeaderMetadataDeferral::Invalid), + } + } + _ => Err(HeaderMetadataDeferral::Invalid), + } +} + +fn configured_audit( + definition_name: &N, + options: &[(N, N)], +) -> Result, HeaderMetadataDeferral> { + let name = optional_text(entry(options, "name"))?; + if let Some(name) = &name { + identity(name)?; + } + let description = optional_text(entry(options, "description"))?; + let severity = optional_text(entry(options, "severity"))?; + if let Some(severity) = &severity { + let text = severity.text().ok_or(HeaderMetadataDeferral::Unsupported)?; + if !AUDIT_SEVERITIES.contains(&text.as_str()) { + return Err(HeaderMetadataDeferral::Invalid); + } + } + let run_scope = optional_text(entry(options, "run_scope"))?; + let always_run = optional_bool(entry(options, "always_run"))?; + if entry(options, "thresholds").is_some_and(|value| value.kind() != NodeKind::Null) { + return Err(HeaderMetadataDeferral::Unsupported); + } + let minimum_samples = optional_count(entry(options, "minimum_samples"))?; + let evidence_limit = optional_count(entry(options, "evidence_limit"))?; + let arguments: Vec<(N, N)> = options + .iter() + .filter(|(key, _)| !is_option_key(key)) + .cloned() + .collect(); + Ok(ParsedAudit { + definition_name: definition_name.clone(), + arguments, + name, + description, + severity, + run_scope, + always_run, + minimum_samples, + evidence_limit, + }) +} + +fn bare_audit(definition_name: N) -> ParsedAudit { + ParsedAudit { + definition_name, + arguments: Vec::new(), + name: None, + description: None, + severity: None, + run_scope: None, + always_run: None, + minimum_samples: None, + evidence_limit: None, + } +} + +fn identity(node: &N) -> Result<(), HeaderMetadataDeferral> { + if is_snake_case(&non_blank_text(node)?) { + Ok(()) + } else { + Err(HeaderMetadataDeferral::Invalid) + } +} + +fn is_option_key(key: &N) -> bool { + AUDIT_OPTION_KEYS.iter().any(|name| key.is_text(name)) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs new file mode 100644 index 0000000000..82f8fae2ab --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs @@ -0,0 +1,86 @@ +//! MODEL column mappings, as `parse_schema_columns` reads them with `migrate_from` allowed. + +use std::collections::HashSet; + +use crate::header_metadata::_helpers::audits::audit_list; +use crate::header_metadata::_helpers::text::{entry, non_blank_text, optional_bool, optional_text}; +use crate::header_metadata::constants::{MODEL_COLUMN_KEYS, NOT_NULL_AUDIT_NAME}; +use crate::header_metadata::models::{HeaderMetadataDeferral, ParsedColumn}; +use crate::types::{AuthoredNode, NodeKind}; + +/// Parse an authored columns mapping; Python's `None` declares no columns. +pub(crate) fn column_mapping( + node: &N, +) -> Result>, HeaderMetadataDeferral> { + match node.kind() { + NodeKind::Null => Ok(Vec::new()), + NodeKind::Map => { + let columns: Vec> = node + .entries() + .iter() + .map(|(name, metadata)| column(name, metadata)) + .collect::>()?; + if has_duplicate_names(&columns) { + Err(HeaderMetadataDeferral::Invalid) + } else { + Ok(columns) + } + } + _ => Err(HeaderMetadataDeferral::Invalid), + } +} + +/// Return whether two ASCII column names are equal under Python's `str.lower()`. +fn has_duplicate_names(columns: &[ParsedColumn]) -> bool { + let mut seen: HashSet = HashSet::new(); + !columns.iter().all(|column| { + column + .name + .text() + .is_some_and(|name| seen.insert(name.to_ascii_lowercase())) + }) +} + +fn column( + name: &N, + metadata: &N, +) -> Result, HeaderMetadataDeferral> { + let text = non_blank_text(name)?; + if !text.is_ascii() { + return Err(HeaderMetadataDeferral::Unsupported); + } + if metadata.kind() != NodeKind::Map { + return Err(HeaderMetadataDeferral::Invalid); + } + let entries = metadata.entries(); + if !entries.iter().all(|(key, _)| is_column_key(key)) { + return Err(HeaderMetadataDeferral::Invalid); + } + let nullable = optional_bool(entry(&entries, "nullable"))?; + let audits = match entry(&entries, "audits") { + Some(value) => audit_list(value)?, + None => Vec::new(), + }; + let declares_nullable = nullable + .as_ref() + .is_some_and(|value| value.kind() == NodeKind::Bool(true)); + if declares_nullable + && audits + .iter() + .any(|audit| audit.definition_name.is_text(NOT_NULL_AUDIT_NAME)) + { + return Err(HeaderMetadataDeferral::Invalid); + } + Ok(ParsedColumn { + name: name.clone(), + column_type: optional_text(entry(&entries, "type"))?, + nullable, + description: optional_text(entry(&entries, "description"))?, + audits, + migrate_from: optional_text(entry(&entries, "migrate_from"))?, + }) +} + +fn is_column_key(key: &N) -> bool { + MODEL_COLUMN_KEYS.iter().any(|name| key.is_text(name)) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs new file mode 100644 index 0000000000..6e200497a7 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod audits; +pub(crate) mod columns; +pub(crate) mod text; diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs new file mode 100644 index 0000000000..b1829e6728 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs @@ -0,0 +1,86 @@ +//! Text checks that agree with Python's `str` methods or defer. + +use crate::header_metadata::models::HeaderMetadataDeferral; +use crate::types::{AuthoredNode, NodeKind}; + +/// Return the text of a string whose `strip()` Python finds non-empty. +pub(crate) fn non_blank_text(node: &N) -> Result { + if node.kind() != NodeKind::Str { + return Err(HeaderMetadataDeferral::Invalid); + } + let text = node.text().ok_or(HeaderMetadataDeferral::Unsupported)?; + if text + .bytes() + .any(|byte| byte.is_ascii() && !is_python_space(byte)) + { + Ok(text) + } else if text.is_ascii() { + Err(HeaderMetadataDeferral::Invalid) + } else { + Err(HeaderMetadataDeferral::Unsupported) + } +} + +/// Return `None` for Python's `None`, or the node of a string `strip()` finds non-empty. +pub(crate) fn optional_text( + node: Option<&N>, +) -> Result, HeaderMetadataDeferral> { + match node { + None => Ok(None), + Some(value) if value.kind() == NodeKind::Null => Ok(None), + Some(value) => non_blank_text(value).map(|_| Some(value.clone())), + } +} + +/// Return `None` for Python's `None`, or the node of a boolean. +pub(crate) fn optional_bool( + node: Option<&N>, +) -> Result, HeaderMetadataDeferral> { + match node.map(AuthoredNode::kind) { + None | Some(NodeKind::Null) => Ok(None), + Some(NodeKind::Bool(_)) => Ok(node.cloned()), + Some(_) => Err(HeaderMetadataDeferral::Invalid), + } +} + +/// Return `None` for Python's `None`, or the node of a non-negative non-boolean integer. +pub(crate) fn optional_count( + node: Option<&N>, +) -> Result, HeaderMetadataDeferral> { + match node.map(AuthoredNode::kind) { + None | Some(NodeKind::Null) => Ok(None), + Some(NodeKind::Int { negative: false }) => Ok(node.cloned()), + Some(_) => Err(HeaderMetadataDeferral::Invalid), + } +} + +/// Return whether `name` fully matches Python's `^[a-z](?:[a-z0-9_]*[a-z0-9])?$` identity pattern. +pub(crate) fn is_snake_case(name: &str) -> bool { + let bytes = name.as_bytes(); + match (bytes.first(), bytes.last()) { + (Some(first), Some(last)) => { + first.is_ascii_lowercase() + && (last.is_ascii_lowercase() || last.is_ascii_digit()) + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_') + } + _ => false, + } +} + +/// Return the mapping entry whose key is the string `key`. +pub(crate) fn entry<'entries, N: AuthoredNode>( + entries: &'entries [(N, N)], + key: &str, +) -> Option<&'entries N> { + entries + .iter() + .find(|(name, _)| name.is_text(key)) + .map(|(_, value)| value) +} + +/// ASCII characters Python's `str.isspace` accepts, including the information separators. +fn is_python_space(byte: u8) -> bool { + matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/constants.rs b/crates/sqlbuild-model-config/src/header_metadata/constants.rs new file mode 100644 index 0000000000..75cc6cc2cb --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/constants.rs @@ -0,0 +1,20 @@ +//! Authored MODEL header metadata keys the native parser recognises. + +/// Keys one MODEL column mapping may declare. +pub const MODEL_COLUMN_KEYS: [&str; 5] = + ["type", "nullable", "description", "audits", "migrate_from"]; +/// Audit options that are not passed to the audit as arguments. +pub const AUDIT_OPTION_KEYS: [&str; 8] = [ + "name", + "description", + "severity", + "run_scope", + "always_run", + "thresholds", + "minimum_samples", + "evidence_limit", +]; +/// Audit severities a header may name. +pub const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; +/// The audit that contradicts `nullable = true`. +pub const NOT_NULL_AUDIT_NAME: &str = "not_null"; diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs new file mode 100644 index 0000000000..03b4f0af13 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs @@ -0,0 +1 @@ +pub mod parse_header_metadata; diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/parse_header_metadata.rs b/crates/sqlbuild-model-config/src/header_metadata/main/parse_header_metadata.rs new file mode 100644 index 0000000000..5ef7637674 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/parse_header_metadata.rs @@ -0,0 +1,17 @@ +//! Parse one MODEL header's columns and audits, or defer to Python. + +use crate::header_metadata::_helpers::audits::audit_list; +use crate::header_metadata::_helpers::columns::column_mapping; +use crate::header_metadata::models::{HeaderMetadata, HeaderMetadataDeferral}; +use crate::types::AuthoredNode; + +/// Parse the authored `columns` and `audits` values exactly as the Python compiler would. +pub fn parse_header_metadata( + columns: &N, + audits: &N, +) -> Result, HeaderMetadataDeferral> { + Ok(HeaderMetadata { + columns: column_mapping(columns)?, + audits: audit_list(audits)?, + }) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/header_metadata/models.rs b/crates/sqlbuild-model-config/src/header_metadata/models.rs new file mode 100644 index 0000000000..a0e5d6602d --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/models.rs @@ -0,0 +1,44 @@ +//! Parsed MODEL header metadata whose leaves are the caller's own authored values. + +/// One audit instance; every field holds the authored value Python would store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedAudit { + pub definition_name: N, + pub arguments: Vec<(N, N)>, + pub name: Option, + pub description: Option, + /// The authored severity text, which names one of the audit severities. + pub severity: Option, + pub run_scope: Option, + /// The authored boolean, or `None` for Python's default of `False`. + pub always_run: Option, + pub minimum_samples: Option, + pub evidence_limit: Option, +} + +/// One MODEL column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedColumn { + pub name: N, + pub column_type: Option, + pub nullable: Option, + pub description: Option, + pub audits: Vec>, + pub migrate_from: Option, +} + +/// The MODEL header's columns and model-level audits. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderMetadata { + pub columns: Vec>, + pub audits: Vec>, +} + +/// Why Python must parse a header's metadata instead. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HeaderMetadataDeferral { + /// Python's parse raises; it runs to raise its exact error. + Invalid, + /// Python parses it with rules the native parser does not reproduce. + Unsupported, +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests.rs b/crates/sqlbuild-model-config/src/header_metadata/tests.rs new file mode 100644 index 0000000000..4c3e7b55db --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_parse_header_metadata.rs"] +mod parse_header_metadata; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs new file mode 100644 index 0000000000..47c50e95a8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs @@ -0,0 +1,77 @@ +use crate::header_metadata::main::parse_header_metadata::parse_header_metadata; +use crate::header_metadata::models::{ + HeaderMetadata, HeaderMetadataDeferral, ParsedAudit, ParsedColumn, +}; +use crate::tests::test_types::Value; +use crate::types::AuthoredNode; + +/// A mapping from string keys. +pub(super) fn map(entries: &[(&'static str, Value)]) -> Value { + Value::Map( + entries + .iter() + .map(|(key, value)| (Value::Str(key), value.clone())) + .collect(), + ) +} + +/// The parse as one line per column and model audit, or why Python must parse it. +pub(super) fn summary( + columns: &Value, + audits: &Value, +) -> Result, HeaderMetadataDeferral> { + parse_header_metadata(columns, audits).map(|metadata| metadata_summary(&metadata)) +} + +fn metadata_summary(metadata: &HeaderMetadata) -> Vec { + metadata + .columns + .iter() + .map(column_summary) + .chain(metadata.audits.iter().map(audit_summary)) + .collect() +} + +fn column_summary(column: &ParsedColumn) -> String { + format!( + "{} {} {} {}", + text(Some(&column.name)), + text(column.column_type.as_ref()), + column + .nullable + .as_ref() + .map_or("-".to_owned(), |value| format!("{value:?}")), + column + .audits + .iter() + .map(audit_summary) + .collect::>() + .join(",") + ) +} + +fn audit_summary(audit: &ParsedAudit) -> String { + let arguments: Vec = audit + .arguments + .iter() + .map(|(key, _)| text(Some(key))) + .collect(); + format!( + "{}:{}{}", + text(Some(&audit.definition_name)), + arguments.join("+"), + audit + .severity + .as_ref() + .map_or(String::new(), |severity| format!( + "@{}", + text(Some(severity)) + )) + ) +} + +fn text(value: Option<&Value>) -> String { + value + .and_then(AuthoredNode::text) + .unwrap_or_else(|| "-".to_owned()) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_parse_header_metadata.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_parse_header_metadata.rs new file mode 100644 index 0000000000..9475ab725c --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_parse_header_metadata.rs @@ -0,0 +1,161 @@ +use crate::header_metadata::models::HeaderMetadataDeferral; +use crate::header_metadata::tests::helpers::{map, summary}; +use crate::header_metadata::tests::test_types::HeaderMetadataTestCase; +use crate::tests::test_types::Value; + +#[test] +fn given_header_metadata_when_parsing_then_happy_paths_parse_and_python_errors_defer() { + let column = |metadata: &[(&'static str, Value)]| map(&[("order_id", map(metadata))]); + let test_cases = [ + HeaderMetadataTestCase { + description: "absent columns and audits declare nothing", + columns: Value::Null, + audits: Value::Null, + expected_summary: Ok(&[]), + }, + HeaderMetadataTestCase { + description: "typed columns keep authored order, nullable and column audits", + columns: map(&[ + ( + "order_id", + map(&[ + ("type", Value::Str("INTEGER")), + ("nullable", Value::Bool(false)), + ( + "audits", + Value::List(vec![ + Value::Str("not_null"), + map(&[("unique", Value::Null)]), + ]), + ), + ]), + ), + ( + "Amount", + map(&[("description", Value::Str("Order amount"))]), + ), + ]), + audits: Value::List(vec![map(&[( + "row_count", + map(&[ + ("minimum", Value::Int(1)), + ("severity", Value::Str("warn")), + ("minimum_samples", Value::Int(0)), + ]), + )])]), + expected_summary: Ok(&[ + "order_id INTEGER Bool(false) not_null:,unique:", + "Amount - - ", + "row_count:minimum@warn", + ]), + }, + HeaderMetadataTestCase { + description: "an empty column mapping declares nothing", + columns: Value::Map(Vec::new()), + audits: Value::List(Vec::new()), + expected_summary: Ok(&[]), + }, + HeaderMetadataTestCase { + description: "case-insensitive duplicate columns defer", + columns: map(&[("id", map(&[])), ("ID", map(&[]))]), + audits: Value::Null, + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "unknown column keys defer", + columns: column(&[("format", Value::Str("x"))]), + audits: Value::Null, + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "nullable true with not_null defers", + columns: column(&[ + ("nullable", Value::Bool(true)), + ("audits", Value::List(vec![Value::Str("not_null")])), + ]), + audits: Value::Null, + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "blank or non-string text defers", + columns: column(&[("type", Value::Str(" "))]), + audits: Value::Null, + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "non-ASCII column names defer to Python's lower()", + columns: map(&[("caf\u{e9}", map(&[]))]), + audits: Value::Null, + expected_summary: Err(HeaderMetadataDeferral::Unsupported), + }, + HeaderMetadataTestCase { + description: "an audit name that is not snake_case defers", + columns: Value::Null, + audits: Value::List(vec![Value::Str("NotNull")]), + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "unknown severities, thresholds and negative counts defer", + columns: Value::Null, + audits: Value::List(vec![ + map(&[("a", map(&[("severity", Value::Str("fatal"))]))]), + map(&[("b", map(&[("thresholds", map(&[]))]))]), + ]), + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "thresholds are left to Python's parser", + columns: Value::Null, + audits: Value::List(vec![map(&[("b", map(&[("thresholds", map(&[]))]))])]), + expected_summary: Err(HeaderMetadataDeferral::Unsupported), + }, + HeaderMetadataTestCase { + description: "ASCII control characters are text to Python's strip", + columns: column(&[("description", Value::Str(" \u{1}"))]), + audits: Value::Null, + expected_summary: Ok(&["order_id - - "]), + }, + HeaderMetadataTestCase { + description: "a negative evidence limit defers", + columns: Value::Null, + audits: Value::List(vec![map(&[( + "a", + map(&[("evidence_limit", Value::Int(-1))]), + )])]), + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "a boolean count defers", + columns: Value::Null, + audits: Value::List(vec![map(&[( + "a", + map(&[("minimum_samples", Value::Bool(true))]), + )])]), + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "an audit list that is not a list defers", + columns: Value::Null, + audits: Value::Float, + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + HeaderMetadataTestCase { + description: "a multi-key audit mapping defers", + columns: Value::Null, + audits: Value::List(vec![map(&[("a", Value::Null), ("b", Value::Null)])]), + expected_summary: Err(HeaderMetadataDeferral::Invalid), + }, + ]; + + for test_case in test_cases { + assert_eq!( + summary(&test_case.columns, &test_case.audits), + test_case.expected_summary.map(|lines| lines + .iter() + .map(|line| (*line).to_owned()) + .collect::>()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs new file mode 100644 index 0000000000..55a9ffa600 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs @@ -0,0 +1,11 @@ +use crate::header_metadata::models::HeaderMetadataDeferral; +use crate::tests::test_types::Value; + +/// One header's `columns` and `audits` values and the parse the native parser must produce. +pub(super) struct HeaderMetadataTestCase { + pub(super) description: &'static str, + pub(super) columns: Value, + pub(super) audits: Value, + /// `name type nullable audit,audit` per column, then `audit:argument,...` per model audit. + pub(super) expected_summary: Result<&'static [&'static str], HeaderMetadataDeferral>, +} diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs new file mode 100644 index 0000000000..0e356fef98 --- /dev/null +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -0,0 +1,11 @@ +//! Native model configuration: MODEL header metadata, config scans and templates with the Python compiler's rules. + +#![forbid(unsafe_code)] + +pub mod config_presence; +pub mod header_metadata; +pub mod templates; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs new file mode 100644 index 0000000000..0f8a42afbb --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs @@ -0,0 +1,133 @@ +//! The Python template evaluator: references, `if`, `eq`, `ne` and `coalesce`. + +use crate::templates::models::{ + ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, +}; +use crate::templates::types::TemplateHost; + +const TRUE_LITERAL: &str = "true"; +const FALSE_LITERAL: &str = "false"; +const NULL_LITERAL: &str = "null"; +const FALSE_VALUES: [&str; 3] = ["", "0", "false"]; +const ENVIRONMENT_NAMESPACE: &str = "ENV"; +const IF_FUNCTION: &str = "if"; +const EQ_FUNCTION: &str = "eq"; +const NE_FUNCTION: &str = "ne"; +const COALESCE_FUNCTION: &str = "coalesce"; +const CONTEXT_NAMESPACE: &str = "CTX"; + +/// Evaluate one expression to the value Python's resolver returns. +pub(crate) fn evaluate( + host: &H, + expression: &Expression, + options: TemplateOptions, +) -> Result { + match expression { + Expression::Text(text) => host.text(text), + Expression::Reference(reference) => evaluate_reference(host, reference, options), + Expression::Function { name, arguments } => { + evaluate_function(host, name, arguments, options) + } + } +} + +/// Render a value as interpolated text, as `render_project_var_text` does for scalars. +pub(crate) fn render_text( + host: &H, + value: &H::Value, +) -> Result { + comparison_text(host, value) +} + +fn evaluate_reference( + host: &H, + reference: &str, + options: TemplateOptions, +) -> Result { + match reference { + TRUE_LITERAL => return host.boolean(true), + FALSE_LITERAL => return host.boolean(false), + NULL_LITERAL => return Ok(host.null()), + _ => {} + } + let Some((namespace, name)) = reference.split_once(':') else { + return host.variable(reference)?.ok_or(TemplateFailure::Missing); + }; + match namespace { + ENVIRONMENT_NAMESPACE => host.environment(name)?.ok_or(TemplateFailure::Missing), + CONTEXT_NAMESPACE if !options.allow_context => { + if options.preserve_context_tokens { + host.text(&format!("${{{reference}}}")) + } else { + Err(TemplateFailure::Invalid) + } + } + CONTEXT_NAMESPACE => match host.context(name)? { + ContextValue::Value(value) => Ok(value), + ContextValue::Unknown if options.preserve_unknown_context => { + host.text(&format!("${{{CONTEXT_NAMESPACE}:{name}}}")) + } + ContextValue::Unknown | ContextValue::Unavailable => Err(TemplateFailure::Missing), + }, + _ => Err(TemplateFailure::Invalid), + } +} + +fn evaluate_function( + host: &H, + name: &str, + arguments: &[Expression], + options: TemplateOptions, +) -> Result { + match (name, arguments) { + (IF_FUNCTION, [condition, when_true, when_false]) => { + let condition = evaluate(host, condition, options)?; + if truthiness(host, &condition)? { + evaluate(host, when_true, options) + } else { + evaluate(host, when_false, options) + } + } + (EQ_FUNCTION | NE_FUNCTION, [left, right]) => { + let left = evaluate(host, left, options)?; + let right = evaluate(host, right, options)?; + let equal = comparison_text(host, &left)? == comparison_text(host, &right)?; + host.boolean(equal == (name == EQ_FUNCTION)) + } + (COALESCE_FUNCTION, [.., last]) => { + for argument in arguments { + match evaluate(host, argument, options) { + Ok(value) if truthiness(host, &value)? => return Ok(value), + Ok(_) | Err(TemplateFailure::Missing) => {} + Err(failure) => return Err(failure), + } + } + evaluate(host, last, options) + } + _ => Err(TemplateFailure::Invalid), + } +} + +fn comparison_text(host: &H, value: &H::Value) -> Result { + match host.scalar(value).ok_or(TemplateFailure::Unsupported)? { + Scalar::Null => Ok(String::new()), + Scalar::Bool(flag) => Ok(if flag { TRUE_LITERAL } else { FALSE_LITERAL }.to_owned()), + Scalar::Text(text) => Ok(text), + } +} + +fn truthiness(host: &H, value: &H::Value) -> Result { + match host.scalar(value).ok_or(TemplateFailure::Unsupported)? { + Scalar::Null => Ok(false), + Scalar::Bool(flag) => Ok(flag), + Scalar::Text(text) if text.is_ascii() => { + let normalized = text + .trim_matches( + |character: char| matches!(character, '\t'..='\r' | '\u{1c}'..='\u{1f}' | ' '), + ) + .to_ascii_lowercase(); + Ok(!FALSE_VALUES.contains(&normalized.as_str())) + } + Scalar::Text(_) => Err(TemplateFailure::Unsupported), + } +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs b/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs new file mode 100644 index 0000000000..2ade14256b --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs @@ -0,0 +1,77 @@ +//! The Python template expression parser. + +use crate::templates::_helpers::tokens::{Token, tokenize}; +use crate::templates::models::{Expression, TemplateFailure}; + +/// Parse one `${...}` body into an expression. +pub(crate) fn parse_expression(body: &str) -> Result { + let tokens: Vec = tokenize(body)?; + let mut parser = Parser { tokens, index: 0 }; + let expression = parser.expression()?; + if parser.peek() == &Token::End { + Ok(expression) + } else { + Err(TemplateFailure::Invalid) + } +} + +struct Parser { + tokens: Vec, + index: usize, +} + +impl Parser { + fn peek(&self) -> &Token { + &self.tokens[self.index] + } + + fn advance(&mut self) -> Token { + let token = self.tokens[self.index].clone(); + self.index += 1; + token + } + + fn match_symbol(&mut self, symbol: char) -> bool { + if self.peek() == &Token::Symbol(symbol) { + self.index += 1; + true + } else { + false + } + } + + fn expression(&mut self) -> Result { + match self.peek() { + Token::Text(_) => match self.advance() { + Token::Text(text) => Ok(Expression::Text(text)), + _ => Err(TemplateFailure::Invalid), + }, + Token::Word(_) => { + let Token::Word(word) = self.advance() else { + return Err(TemplateFailure::Invalid); + }; + if self.match_symbol('(') { + self.function_call(word) + } else { + Ok(Expression::Reference(word)) + } + } + Token::Symbol(_) | Token::End => Err(TemplateFailure::Invalid), + } + } + + fn function_call(&mut self, name: String) -> Result { + let mut arguments: Vec = Vec::new(); + while self.peek() != &Token::Symbol(')') { + arguments.push(self.expression()?); + if !self.match_symbol(',') { + break; + } + } + if self.match_symbol(')') { + Ok(Expression::Function { name, arguments }) + } else { + Err(TemplateFailure::Invalid) + } + } +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs new file mode 100644 index 0000000000..458598d7c2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod evaluation; +pub(crate) mod expressions; +pub(crate) mod pattern; +pub(crate) mod tokens; diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs b/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs new file mode 100644 index 0000000000..8c21b517fd --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs @@ -0,0 +1,27 @@ +//! Python's `\$\{([^{}]+)\}` matches, found left to right without overlap. + +/// Return the byte spans of every template match, including the `${` and `}`. +pub(crate) fn template_spans(text: &str) -> Vec<(usize, usize)> { + let bytes = text.as_bytes(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut index = 0; + while index + 1 < bytes.len() { + if bytes[index] == b'$' && bytes[index + 1] == b'{' { + let body = index + 2; + let close = bytes[body..] + .iter() + .position(|byte| matches!(byte, b'{' | b'}')) + .map(|offset| body + offset); + if let Some(close) = close + && close > body + && bytes[close] == b'}' + { + spans.push((index, close + 1)); + index = close + 1; + continue; + } + } + index += 1; + } + spans +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs new file mode 100644 index 0000000000..11565d6ea2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs @@ -0,0 +1,75 @@ +//! The Python template tokenizer, deferring wherever Python's `str.isspace` would decide. + +use crate::templates::models::TemplateFailure; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Token { + Word(String), + Text(String), + Symbol(char), + End, +} + +const SYMBOLS: [char; 3] = ['(', ')', ',']; +const ESCAPE: char = '\\'; + +/// Split one template expression into tokens. +pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> { + let mut tokens: Vec = Vec::new(); + let mut characters = expression.chars().peekable(); + while let Some(&character) = characters.peek() { + if !character.is_ascii() { + return Err(TemplateFailure::Unsupported); + } + if is_python_space(character) { + characters.next(); + } else if SYMBOLS.contains(&character) { + tokens.push(Token::Symbol(character)); + characters.next(); + } else if is_quote(character) { + characters.next(); + tokens.push(Token::Text(quoted_text(&mut characters, character)?)); + } else { + let mut word = String::new(); + while let Some(&next) = characters.peek() { + if !next.is_ascii() { + return Err(TemplateFailure::Unsupported); + } + if is_python_space(next) || SYMBOLS.contains(&next) || is_quote(next) { + break; + } + word.push(next); + characters.next(); + } + tokens.push(Token::Word(word)); + } + } + tokens.push(Token::End); + Ok(tokens) +} + +fn quoted_text( + characters: &mut impl Iterator, + quote: char, +) -> Result { + let mut text = String::new(); + while let Some(character) = characters.next() { + if character == ESCAPE { + text.push(characters.next().ok_or(TemplateFailure::Invalid)?); + } else if character == quote { + return Ok(text); + } else { + text.push(character); + } + } + Err(TemplateFailure::Invalid) +} + +fn is_quote(character: char) -> bool { + matches!(character, '\'' | '"') +} + +/// ASCII characters Python's `str.isspace` accepts, including the information separators. +fn is_python_space(character: char) -> bool { + matches!(character, '\t'..='\r' | '\u{1c}'..='\u{1f}' | ' ') +} diff --git a/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs new file mode 100644 index 0000000000..ebd69d0f51 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs @@ -0,0 +1,37 @@ +//! Expand the `${...}` templates in one string exactly as the Python template resolver does. + +use crate::templates::_helpers::evaluation::{evaluate, render_text}; +use crate::templates::_helpers::expressions::parse_expression; +use crate::templates::_helpers::pattern::template_spans; +use crate::templates::models::{StringExpansion, TemplateFailure, TemplateOptions}; +use crate::templates::types::TemplateHost; + +/// Expand `text`: a whole-string template keeps its value, others are substituted as text. +pub fn expand_template_string( + host: &H, + text: &str, + options: TemplateOptions, +) -> Result, TemplateFailure> { + let spans: Vec<(usize, usize)> = template_spans(text); + if let [(start, end)] = spans.as_slice() + && *start == 0 + && *end == text.len() + { + let expression = parse_expression(&text[2..text.len() - 1])?; + return evaluate(host, &expression, options).map(StringExpansion::Value); + } + if spans.is_empty() { + return Ok(StringExpansion::Unchanged); + } + let mut expanded = String::with_capacity(text.len()); + let mut copied = 0; + for (start, end) in spans { + expanded.push_str(&text[copied..start]); + let expression = parse_expression(&text[start + 2..end - 1])?; + let value = evaluate(host, &expression, options)?; + expanded.push_str(&render_text(host, &value)?); + copied = end; + } + expanded.push_str(&text[copied..]); + Ok(StringExpansion::Text(expanded)) +} diff --git a/crates/sqlbuild-model-config/src/templates/main/mod.rs b/crates/sqlbuild-model-config/src/templates/main/mod.rs new file mode 100644 index 0000000000..d1b5730df4 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/main/mod.rs @@ -0,0 +1 @@ +pub mod expand_template_string; diff --git a/crates/sqlbuild-model-config/src/templates/mod.rs b/crates/sqlbuild-model-config/src/templates/mod.rs new file mode 100644 index 0000000000..91fb358a0f --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/templates/models.rs b/crates/sqlbuild-model-config/src/templates/models.rs new file mode 100644 index 0000000000..4d675be772 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/models.rs @@ -0,0 +1,60 @@ +//! Template expressions, scalar values and outcomes. + +/// A parsed `${...}` expression. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Expression { + Text(String), + Reference(String), + Function { + name: String, + arguments: Vec, + }, +} + +/// A value whose text Python's `str()` produces without calling user code. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Scalar { + Null, + Bool(bool), + /// An exact `str` or the decimal digits of an exact `int`. + Text(String), +} + +/// One context lookup. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ContextValue { + Unknown, + /// The key is known but has no value yet. + Unavailable, + Value(V), +} + +/// Why native expansion stops; Python then expands the value itself. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TemplateFailure { + /// A missing variable, environment variable or context value, which `coalesce` skips. + Missing, + /// Python raises another error for this template. + Invalid, + /// Python expands this template with rules the native expansion does not reproduce. + Unsupported, +} + +/// How evaluation treats `CTX:` references, mirroring the Python resolver's flags. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TemplateOptions { + pub allow_context: bool, + pub preserve_context_tokens: bool, + pub preserve_unknown_context: bool, +} + +/// One string's expansion. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StringExpansion { + /// The string holds no complete template and stays the same object. + Unchanged, + /// The whole string was one template, which evaluated to this value. + Value(V), + /// The templates inside the string were substituted. + Text(String), +} diff --git a/crates/sqlbuild-model-config/src/templates/tests.rs b/crates/sqlbuild-model-config/src/templates/tests.rs new file mode 100644 index 0000000000..0d59525ede --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_expand_template_string.rs"] +mod expand_template_string; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/templates/tests/helpers.rs b/crates/sqlbuild-model-config/src/templates/tests/helpers.rs new file mode 100644 index 0000000000..75f8fbd407 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/tests/helpers.rs @@ -0,0 +1,12 @@ +use crate::templates::models::{StringExpansion, TemplateFailure}; +use crate::templates::tests::test_types::Value; + +/// A substituted string. +pub(super) fn text(value: &str) -> Result, TemplateFailure> { + Ok(StringExpansion::Text(value.to_owned())) +} + +/// A whole-string template's value. +pub(super) fn value(value: Value) -> Result, TemplateFailure> { + Ok(StringExpansion::Value(value)) +} diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs new file mode 100644 index 0000000000..c50b3a8551 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs @@ -0,0 +1,128 @@ +use std::cell::RefCell; + +use crate::templates::main::expand_template_string::expand_template_string; +use crate::templates::models::{StringExpansion, TemplateFailure, TemplateOptions}; +use crate::templates::tests::helpers::{text, value}; +use crate::templates::tests::test_types::{TemplateTestCase, TestHost, Value}; + +#[test] +fn given_template_strings_when_expanding_then_values_and_reads_match_python() { + let test_cases = [ + TemplateTestCase { + description: "a whole-string variable keeps its value", + text: "${flag}", + expected: value(Value::Bool(true)), + expected_reads: &[], + }, + TemplateTestCase { + description: "embedded templates render as text", + text: "orders_${env}_${flag}_${null}", + expected: text("orders_prod_true_"), + expected_reads: &[], + }, + TemplateTestCase { + description: "coalesce skips a missing environment variable and re-reads the last", + text: "${coalesce(ENV:MISSING, ENV:SCHEMA_NAME, 'fallback')}", + expected: value(Value::Text("analytics".to_owned())), + expected_reads: &["ENV:MISSING", "ENV:SCHEMA_NAME"], + }, + TemplateTestCase { + description: "coalesce returns the last argument when none is truthy", + text: "${coalesce(ENV:MISSING, ' False ', ENV:EMPTY)}", + expected: value(Value::Text(String::new())), + expected_reads: &["ENV:MISSING", "ENV:EMPTY", "ENV:EMPTY"], + }, + TemplateTestCase { + description: "if, eq and ne compare rendered text", + text: "${if(eq(CTX:run.target, 'prod'), 'p', ne(flag, true))}", + expected: value(Value::Text("p".to_owned())), + expected_reads: &["CTX:run.target"], + }, + TemplateTestCase { + description: "escapes and nested quotes inside strings", + text: r#"${"a\"b)c"}"#, + expected: value(Value::Text("a\"b)c".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "an unknown context key is preserved", + text: "${CTX:model.alias}", + expected: value(Value::Text("${CTX:model.alias}".to_owned())), + expected_reads: &["CTX:model.alias"], + }, + TemplateTestCase { + description: "a string without a complete template is unchanged", + text: "${} and ${a{b}", + expected: Ok(StringExpansion::Unchanged), + expected_reads: &[], + }, + TemplateTestCase { + description: "a context key without a value is missing", + text: "${CTX:model.schema}", + expected: Err(TemplateFailure::Missing), + expected_reads: &["CTX:model.schema"], + }, + TemplateTestCase { + description: "parse errors are invalid", + text: "${coalesce(a,,b)}", + expected: Err(TemplateFailure::Invalid), + expected_reads: &[], + }, + TemplateTestCase { + description: "an unsupported function or namespace is invalid", + text: "${upper(env)}", + expected: Err(TemplateFailure::Invalid), + expected_reads: &[], + }, + TemplateTestCase { + description: "rendering an opaque value is left to Python", + text: "x_${opaque}", + expected: Err(TemplateFailure::Unsupported), + expected_reads: &[], + }, + TemplateTestCase { + description: "non-ASCII outside quotes is left to Python's whitespace rules", + text: "${coalesce(\u{a0}env)}", + expected: Err(TemplateFailure::Unsupported), + expected_reads: &[], + }, + TemplateTestCase { + description: "Python separators are whitespace between tokens", + text: "${coalesce(\u{1f}env\t)}", + expected: value(Value::Text("prod".to_owned())), + expected_reads: &[], + }, + ]; + + for test_case in test_cases { + let host = TestHost { + variables: vec![ + ("flag", Value::Bool(true)), + ("env", Value::Text("prod".to_owned())), + ("opaque", Value::Opaque), + ], + environment: vec![("SCHEMA_NAME", "analytics"), ("EMPTY", "")], + context: vec![("run.target", Some("prod")), ("model.schema", None)], + reads: RefCell::new(Vec::new()), + }; + let options = TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }; + let expansion = expand_template_string(&host, test_case.text, options); + assert_eq!( + (expansion, host.reads.into_inner()), + ( + test_case.expected, + test_case + .expected_reads + .iter() + .map(|read| (*read).to_owned()) + .collect::>() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_types.rs b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs new file mode 100644 index 0000000000..793c262d2a --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs @@ -0,0 +1,81 @@ +use std::cell::RefCell; + +use crate::templates::models::{ContextValue, Scalar, StringExpansion, TemplateFailure}; +use crate::templates::types::TemplateHost; + +/// A template value standing in for the Python objects the bindings evaluate. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum Value { + Null, + Bool(bool), + Text(String), + /// A value only Python can render, such as a float or a mapping. + Opaque, +} + +/// Fixed variables, environment and context, recording reads like the Python resolver. +pub(super) struct TestHost { + pub(super) variables: Vec<(&'static str, Value)>, + pub(super) environment: Vec<(&'static str, &'static str)>, + pub(super) context: Vec<(&'static str, Option<&'static str>)>, + pub(super) reads: RefCell>, +} + +impl TemplateHost for TestHost { + type Value = Value; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + Ok(self + .variables + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone())) + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.reads.borrow_mut().push(format!("ENV:{name}")); + Ok(self + .environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| Value::Text((*value).to_owned()))) + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.reads.borrow_mut().push(format!("CTX:{name}")); + Ok(match self.context.iter().find(|(key, _)| *key == name) { + None => ContextValue::Unknown, + Some((_, None)) => ContextValue::Unavailable, + Some((_, Some(value))) => ContextValue::Value(Value::Text((*value).to_owned())), + }) + } + + fn text(&self, text: &str) -> Result { + Ok(Value::Text(text.to_owned())) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(Value::Bool(flag)) + } + + fn null(&self) -> Value { + Value::Null + } + + fn scalar(&self, value: &Value) -> Option { + match value { + Value::Null => Some(Scalar::Null), + Value::Bool(flag) => Some(Scalar::Bool(*flag)), + Value::Text(text) => Some(Scalar::Text(text.clone())), + Value::Opaque => None, + } + } +} + +/// One template string, its expansion and the reads it records. +pub(super) struct TemplateTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected: Result, TemplateFailure>, + pub(super) expected_reads: &'static [&'static str], +} diff --git a/crates/sqlbuild-model-config/src/templates/types.rs b/crates/sqlbuild-model-config/src/templates/types.rs new file mode 100644 index 0000000000..1c3adb5926 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/types.rs @@ -0,0 +1,24 @@ +//! What template evaluation reads from its caller. + +use crate::templates::models::{ContextValue, Scalar, TemplateFailure}; + +/// The variables, environment and context one template expansion reads, owned by the caller. +pub trait TemplateHost { + /// One evaluated value; Python objects in the bindings. + type Value: Clone; + + /// Return a project variable, or `None` when it is not defined. + fn variable(&self, name: &str) -> Result, TemplateFailure>; + /// Record an environment read and return the variable, or `None` when it is not set. + fn environment(&self, name: &str) -> Result, TemplateFailure>; + /// Record a context read and return the context value. + fn context(&self, name: &str) -> Result, TemplateFailure>; + /// Return a text value. + fn text(&self, text: &str) -> Result; + /// Return a boolean value. + fn boolean(&self, flag: bool) -> Result; + /// Return the null value. + fn null(&self) -> Self::Value; + /// Return the scalar `value` is, or `None` when only Python can render or compare it. + fn scalar(&self, value: &Self::Value) -> Option; +} diff --git a/crates/sqlbuild-model-config/src/tests.rs b/crates/sqlbuild-model-config/src/tests.rs new file mode 100644 index 0000000000..3dd9cede02 --- /dev/null +++ b/crates/sqlbuild-model-config/src/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_types.rs"] +pub(crate) mod test_types; diff --git a/crates/sqlbuild-model-config/src/tests/test_types.rs b/crates/sqlbuild-model-config/src/tests/test_types.rs new file mode 100644 index 0000000000..b524b6ee45 --- /dev/null +++ b/crates/sqlbuild-model-config/src/tests/test_types.rs @@ -0,0 +1,63 @@ +use crate::types::{AuthoredNode, NodeKind}; + +/// An authored value tree standing in for the Python objects the bindings read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Value { + Null, + Bool(bool), + Int(i64), + Str(&'static str), + List(Vec), + Map(Vec<(Value, Value)>), + Tuple(Vec), + Float, +} + +impl AuthoredNode for Value { + fn kind(&self) -> NodeKind { + match self { + Self::Null => NodeKind::Null, + Self::Bool(flag) => NodeKind::Bool(*flag), + Self::Int(value) => NodeKind::Int { + negative: *value < 0, + }, + Self::Str(_) => NodeKind::Str, + Self::List(_) => NodeKind::List, + Self::Map(_) => NodeKind::Map, + Self::Tuple(_) => NodeKind::Tuple, + Self::Float => NodeKind::Other, + } + } + + fn text(&self) -> Option { + match self { + Self::Str(text) => Some((*text).to_owned()), + _ => None, + } + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + match self { + Self::Str(text) => Some(read(text)), + _ => None, + } + } + + fn is_text(&self, text: &str) -> bool { + matches!(self, Self::Str(value) if *value == text) + } + + fn items(&self) -> Vec { + match self { + Self::List(items) | Self::Tuple(items) => items.clone(), + _ => Vec::new(), + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + match self { + Self::Map(entries) => entries.clone(), + _ => Vec::new(), + } + } +} diff --git a/crates/sqlbuild-model-config/src/types.rs b/crates/sqlbuild-model-config/src/types.rs new file mode 100644 index 0000000000..a8f798c4bf --- /dev/null +++ b/crates/sqlbuild-model-config/src/types.rs @@ -0,0 +1,34 @@ +//! The shape of one authored header value as the parser inspects it. + +/// One authored value's kind; containers and strings are read through [`AuthoredNode`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodeKind { + Null, + Bool(bool), + /// An integer that is not a boolean, with its sign. + Int { + negative: bool, + }, + Str, + List, + Tuple, + Map, + /// Any other value: tuples, floats, markers and objects. + Other, +} + +/// Read access to an authored value tree owned by the caller, such as Python objects. +pub trait AuthoredNode: Clone { + /// Return this value's kind. + fn kind(&self) -> NodeKind; + /// Return the text of a string value, or `None` when it is not readable UTF-8 text. + fn text(&self) -> Option; + /// Return whether this is a string value equal to `text`. + fn is_text(&self, text: &str) -> bool; + /// Apply `read` to a string value's text, or return `None` when it is not readable UTF-8. + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option; + /// Return a list's or tuple's items in order. + fn items(&self) -> Vec; + /// Return a mapping's entries in insertion order. + fn entries(&self) -> Vec<(Self, Self)>; +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 84f69c98dc..7eab2dc618 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -24,6 +24,7 @@ sqlbuild-analysis.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/fensu.toml b/fensu.toml index 32db96582f..ac01351948 100644 --- a/fensu.toml +++ b/fensu.toml @@ -308,6 +308,7 @@ crate_names = [ "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", + "sqlbuild-model-config", "sqlbuild-python", "sqlbuild-rules", "sqlbuild-scopes", From 900b4ee347181a20f114aa84cd8104687da3061e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:52:43 +0100 Subject: [PATCH 2/3] feat(native): resolve model configuration natively behind the preview engine The model loop parses every model's header columns and audits in one native batch and expands config templates natively under native-preview. Python keeps every error: a native rejection re-runs Python, and a mismatch is raised if Python then succeeds. --- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/model_config/authored_nodes.rs | 79 ++ .../_helpers/model_config/config_presence.rs | 35 + .../_helpers/model_config/config_templates.rs | 206 +++++ .../_helpers/model_config/header_metadata.rs | 175 ++++ .../src/bindings/_helpers/model_config/mod.rs | 4 + .../src/bindings/main/register.rs | 7 +- src/sqlbuild/_native.pyi | 12 +- .../compile/_helpers/attachment/core.py | 33 +- .../_helpers/attachment/model_config.py | 84 +- .../_helpers/render/context_templates.py | 85 +- src/sqlbuild/compiler/compile/exceptions.py | 4 + src/sqlbuild/compiler/compile/models.py | 1 + src/sqlbuild/compiler/frontier/constants.py | 1 + src/sqlbuild/compiler/frontier/types.py | 1 + .../compiler/model_config/__init__.py | 1 + .../compiler/model_config/constants.py | 32 + .../compiler/model_config/main/__init__.py | 1 + .../main/_expand_native_config_templates.py | 34 + .../_native_config_contains_macro_call.py | 11 + .../main/_native_config_contains_template.py | 11 + .../main/_parse_native_header_metadata.py | 82 ++ src/sqlbuild/compiler/model_config/models.py | 51 ++ .../cli/commands/main/compile/_test_types.py | 13 + .../cli/commands/main/compile/helpers.py | 7 + .../main/compile/test_native_model_config.py | 129 +++ .../compiler/model_config/__init__.py | 0 .../compiler/model_config/_test_types.py | 61 ++ .../sqlbuild/compiler/model_config/helpers.py | 762 ++++++++++++++++++ .../model_config/test_native_model_config.py | 237 ++++++ 30 files changed, 2130 insertions(+), 30 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs create mode 100644 src/sqlbuild/compiler/model_config/__init__.py create mode 100644 src/sqlbuild/compiler/model_config/constants.py create mode 100644 src/sqlbuild/compiler/model_config/main/__init__.py create mode 100644 src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py create mode 100644 src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py create mode 100644 src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py create mode 100644 src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py create mode 100644 src/sqlbuild/compiler/model_config/models.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_config.py create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 9c489926c4..e5d5e03f58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod analysis; pub(crate) mod boundary; pub(crate) mod discovery; +pub(crate) mod model_config; pub(crate) mod rules; pub(crate) mod scopes; pub(crate) mod sqltext; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs new file mode 100644 index 0000000000..b8edbf6e2c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs @@ -0,0 +1,79 @@ +//! Python objects read in place as authored header values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, PyTupleMethods, +}; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +/// One Python object viewed as an authored value; parsed results keep the object itself. +#[derive(Clone, Debug)] +pub(crate) struct PyNode<'py>(pub(crate) Bound<'py, PyAny>); + +impl AuthoredNode for PyNode<'_> { + fn kind(&self) -> NodeKind { + let value = &self.0; + if value.is_none() { + NodeKind::Null + } else if let Ok(flag) = value.downcast::() { + NodeKind::Bool(flag.is_true()) + } else if value.is_instance_of::() { + value + .lt(0) + .map_or(NodeKind::Other, |negative| NodeKind::Int { negative }) + } else if value.is_instance_of::() { + NodeKind::Str + } else if value.is_instance_of::() { + NodeKind::List + } else if value.is_instance_of::() { + NodeKind::Tuple + } else if value.is_instance_of::() { + NodeKind::Map + } else { + NodeKind::Other + } + } + + fn text(&self) -> Option { + self.with_text(str::to_owned) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + if let Ok(value) = self.0.downcast::() + && let Ok(text) = value.to_str() + { + Some(read(text)) + } else { + None + } + } + + fn is_text(&self, text: &str) -> bool { + self.with_text(|value| value == text).unwrap_or(false) + } + + fn items(&self) -> Vec { + if let Ok(items) = self.0.downcast::() { + items.iter().map(PyNode).collect() + } else if let Ok(items) = self.0.downcast::() { + items.iter().map(PyNode).collect() + } else { + Vec::new() + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + if let Ok(entries) = self.0.downcast::() { + entries.iter().map(entry_nodes).collect() + } else { + Vec::new() + } + } +} + +fn entry_nodes<'py>( + (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> (PyNode<'py>, PyNode<'py>) { + (PyNode(key), PyNode(value)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs new file mode 100644 index 0000000000..8324088f1e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs @@ -0,0 +1,35 @@ +//! Recursive template and macro presence scans over authored Python config values. + +use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::config_presence::main::contains_macro_call::contains_macro_call; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::models::Presence; + +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; + +fn answer(presence: Presence) -> Option { + match presence { + Presence::Present => Some(true), + Presence::Absent => Some(false), + Presence::Deferred => None, + } +} + +/// Return whether any nested string holds `${`, or `None` where Python must scan. +#[pyfunction] +fn config_contains_template(value: Bound<'_, PyAny>) -> Option { + answer(contains_template(&PyNode(value))) +} + +/// Return whether any nested string holds a macro call, or `None` where Python must scan. +#[pyfunction] +fn config_contains_macro_call(value: Bound<'_, PyAny>) -> Option { + answer(contains_macro_call(&PyNode(value))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(config_contains_template, module)?)?; + module.add_function(wrap_pyfunction!(config_contains_macro_call, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs new file mode 100644 index 0000000000..9a2a60574d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -0,0 +1,206 @@ +//! `${...}` templates in authored Python config values, expanded natively. + +use std::cell::RefCell; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, PyTupleMethods, +}; +use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::models::{ + ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, +}; +use sqlbuild_model_config::templates::types::TemplateHost; + +const TEMPLATE_OPEN_TOKEN: &str = "${"; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; +const INVALID_OUTCOME: &str = "invalid"; +const UNSUPPORTED_OUTCOME: &str = "unsupported"; + +/// Variables, the process environment and context values, all read through Python. +struct PythonHost<'py> { + py: Python<'py>, + sources: TemplateSources<'py>, + reads: RefCell>, +} + +/// The variables, the `os.environ` mapping and the context values one expansion reads. +#[derive(FromPyObject)] +struct TemplateSources<'py>(Bound<'py, PyDict>, Bound<'py, PyAny>, Bound<'py, PyDict>); + +/// `allow_context`, `preserve_context_tokens` and `preserve_unknown_context`. +#[derive(FromPyObject)] +struct TemplateFlags(bool, bool, bool); + +/// A template failure, or a Python error raised while building the expanded containers. +enum Stop { + Failure(TemplateFailure), + Python(PyErr), +} + +impl From for Stop { + fn from(failure: TemplateFailure) -> Self { + Self::Failure(failure) + } +} + +fn unsupported(_: PyErr) -> TemplateFailure { + TemplateFailure::Unsupported +} + +impl<'py> TemplateHost for PythonHost<'py> { + type Value = Bound<'py, PyAny>; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + self.sources.0.get_item(name).map_err(unsupported) + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((ENVIRONMENT_READ, name.to_owned())); + let environment: &Bound<'py, PyAny> = &self.sources.1; + if environment.contains(name).map_err(unsupported)? { + environment.get_item(name).map(Some).map_err(unsupported) + } else { + Ok(None) + } + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((CONTEXT_READ, name.to_owned())); + match self.sources.2.get_item(name).map_err(unsupported)? { + None => Ok(ContextValue::Unknown), + Some(value) if value.is_none() => Ok(ContextValue::Unavailable), + Some(value) => Ok(ContextValue::Value(value)), + } + } + + fn text(&self, text: &str) -> Result { + Ok(PyString::new(self.py, text).into_any()) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(PyBool::new(self.py, flag).to_owned().into_any()) + } + + fn null(&self) -> Self::Value { + self.py.None().into_bound(self.py) + } + + fn scalar(&self, value: &Self::Value) -> Option { + if value.is_none() { + Some(Scalar::Null) + } else if let Ok(flag) = value.downcast_exact::() { + Some(Scalar::Bool(flag.is_true())) + } else if value.is_exact_instance_of::() + && let Ok(number) = value.extract::() + { + Some(Scalar::Text(number.to_string())) + } else if let Ok(text) = value.downcast_exact::() + && let Ok(text) = text.to_str() + { + Some(Scalar::Text(text.to_owned())) + } else { + None + } + } +} + +/// Return `(value, ordered reads)` as `expand_template_data` would, `invalid` or `unsupported`. +#[pyfunction] +fn expand_config_templates<'py>( + py: Python<'py>, + value: Bound<'py, PyAny>, + sources: TemplateSources<'py>, + flags: TemplateFlags, +) -> PyResult> { + let TemplateFlags(allow_context, preserve_context_tokens, preserve_unknown_context) = flags; + let options = TemplateOptions { + allow_context, + preserve_context_tokens, + preserve_unknown_context, + }; + let host = PythonHost { + py, + sources, + reads: RefCell::new(Vec::new()), + }; + match expanded(&host, &value, options) { + Ok(result) => Ok((result, host.reads.into_inner()) + .into_pyobject(py)? + .into_any() + .unbind()), + Err(Stop::Failure(TemplateFailure::Missing | TemplateFailure::Invalid)) => { + Ok(PyString::new(py, INVALID_OUTCOME).into_any().unbind()) + } + Err(Stop::Failure(TemplateFailure::Unsupported)) => { + Ok(PyString::new(py, UNSUPPORTED_OUTCOME).into_any().unbind()) + } + Err(Stop::Python(error)) => Err(error), + } +} + +fn expanded<'py>( + host: &PythonHost<'py>, + value: &Bound<'py, PyAny>, + options: TemplateOptions, +) -> Result, Stop> { + let py = host.py; + if let Ok(text) = value.downcast_exact::() { + let text = text.to_str().map_err(|_| TemplateFailure::Unsupported)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + return Ok(match expand_template_string(host, text, options)? { + StringExpansion::Unchanged => value.clone(), + StringExpansion::Value(result) => result, + StringExpansion::Text(result) => PyString::new(py, &result).into_any(), + }); + } + if let Ok(mapping) = value.downcast_exact::() { + let result = PyDict::new(py); + for (key, item) in mapping.iter() { + result + .set_item(key, expanded(host, &item, options)?) + .map_err(Stop::Python)?; + } + return Ok(result.into_any()); + } + if let Ok(items) = value.downcast_exact::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyList::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + if let Ok(items) = value.downcast_exact::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyTuple::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + if value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + return Err(Stop::Failure(TemplateFailure::Unsupported)); + } + Ok(value.clone()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_config_templates, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs new file mode 100644 index 0000000000..070a51dbe5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -0,0 +1,175 @@ +//! MODEL header columns and audits parsed natively into the Python contract objects. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{FromPyObject, IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_model_config::header_metadata::models::{ + HeaderMetadata, HeaderMetadataDeferral, ParsedAudit, ParsedColumn, +}; + +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; + +/// A model's `(columns, audits)` tuples, or the outcome name `invalid` or `unsupported`. +type ParsedMetadataRow = Py; + +const INVALID_OUTCOME: &str = "invalid"; +const UNSUPPORTED_OUTCOME: &str = "unsupported"; + +/// The Python classes the parsed metadata becomes, read from a Python mapping. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ContractClasses<'py> { + schema_column: Bound<'py, PyAny>, + schema_audit_instance: Bound<'py, PyAny>, + severities: Bound<'py, PyDict>, + allocate: Bound<'py, PyAny>, +} + +/// One model's authored `columns` and `audits` values and its column locations. +#[derive(FromPyObject)] +struct HeaderMetadataRequest<'py>(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyDict>); + +/// Return each model's `(columns, audits)` contract tuples, or why Python must parse it. +#[pyfunction] +fn parse_model_header_metadata<'py>( + py: Python<'py>, + requests: Vec>, + classes: ContractClasses<'py>, +) -> PyResult> { + let mut rows: Vec = Vec::with_capacity(requests.len()); + for HeaderMetadataRequest(columns, audits, locations) in requests { + rows.push( + match parse_header_metadata(&PyNode(columns), &PyNode(audits)) { + Ok(metadata) => metadata_row(py, &classes, &metadata, &locations)?, + Err(HeaderMetadataDeferral::Invalid) => { + PyString::new(py, INVALID_OUTCOME).into_any().unbind() + } + Err(HeaderMetadataDeferral::Unsupported) => { + PyString::new(py, UNSUPPORTED_OUTCOME).into_any().unbind() + } + }, + ); + } + Ok(rows) +} + +fn metadata_row<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + metadata: &HeaderMetadata>, + locations: &Bound<'py, PyDict>, +) -> PyResult { + let no_location: Bound<'py, PyAny> = py.None().into_bound(py); + let mut columns: Vec> = Vec::with_capacity(metadata.columns.len()); + for column in &metadata.columns { + columns.push(schema_column(py, classes, column, locations)?); + } + let mut audits: Vec> = Vec::with_capacity(metadata.audits.len()); + for audit in &metadata.audits { + audits.push(audit_instance(py, classes, audit, &no_location)?); + } + Ok((PyTuple::new(py, columns)?, PyTuple::new(py, audits)?) + .into_pyobject(py)? + .into_any() + .unbind()) +} + +/// Build a frozen contract dataclass the way `copy` does: allocate, then fill its `__dict__`. +fn contract_object<'py>( + allocate: &Bound<'py, PyAny>, + class: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let object = allocate.call1((class,))?; + let attributes = object.getattr("__dict__")?.downcast_into::()?; + for (name, value) in fields { + attributes.set_item(name, value)?; + } + Ok(object) +} + +fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { + value.map_or_else(|| py.None().into_bound(py), |node| node.0.clone()) +} + +fn schema_column<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + column: &ParsedColumn>, + locations: &Bound<'py, PyDict>, +) -> PyResult> { + let location = locations + .get_item(&column.name.0)? + .unwrap_or_else(|| py.None().into_bound(py)); + let audits = column + .audits + .iter() + .map(|audit| audit_instance(py, classes, audit, &location)) + .collect::>>()?; + contract_object( + &classes.allocate, + &classes.schema_column, + &[ + ("name", column.name.0.clone()), + ("type", optional(py, column.column_type.as_ref())), + ("nullable", optional(py, column.nullable.as_ref())), + ("description", optional(py, column.description.as_ref())), + ("meta", PyDict::new(py).into_any()), + ("audits", PyTuple::new(py, audits)?.into_any()), + ("location", location), + ("migrate_from", optional(py, column.migrate_from.as_ref())), + ], + ) +} + +fn audit_instance<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + audit: &ParsedAudit>, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let arguments = PyDict::new(py); + for (key, value) in &audit.arguments { + arguments.set_item(&key.0, &value.0)?; + } + let severity = match &audit.severity { + Some(severity) => classes + .severities + .get_item(&severity.0)? + .unwrap_or_else(|| py.None().into_bound(py)), + None => py.None().into_bound(py), + }; + let always_run = audit.always_run.as_ref().map_or_else( + || PyBool::new(py, false).to_owned().into_any(), + |node| node.0.clone(), + ); + contract_object( + &classes.allocate, + &classes.schema_audit_instance, + &[ + ("definition_name", audit.definition_name.0.clone()), + ("arguments", arguments.into_any()), + ("name", optional(py, audit.name.as_ref())), + ("description", optional(py, audit.description.as_ref())), + ("severity", severity), + ("run_scope", optional(py, audit.run_scope.as_ref())), + ("always_run", always_run), + ("thresholds", py.None().into_bound(py)), + ( + "minimum_samples", + optional(py, audit.minimum_samples.as_ref()), + ), + ( + "evidence_limit", + optional(py, audit.evidence_limit.as_ref()), + ), + ("location", location.clone()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_model_header_metadata, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs new file mode 100644 index 0000000000..d3bfe54b86 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod authored_nodes; +pub(crate) mod config_presence; +pub(crate) mod config_templates; +pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 924a4d638c..6b88d87354 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -3,7 +3,9 @@ use pyo3::pymodule; use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; -use crate::bindings::_helpers::{analysis, boundary, discovery, rules, scopes, sqltext}; +use crate::bindings::_helpers::{ + analysis, boundary, discovery, model_config, rules, scopes, sqltext, +}; use crate::bindings::models; #[pymodule] @@ -28,6 +30,9 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::project_files::register(module)?; scopes::scope_index::register(module)?; scopes::relationship_names::register(module)?; + model_config::header_metadata::register(module)?; + model_config::config_presence::register(module)?; + model_config::config_templates::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index acb25ca011..7db9e0cab4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,6 @@ """Private SQLBuild native engine bindings.""" -from typing import TypedDict +from typing import Any, TypedDict BUILD_IDENTITY: str @@ -127,6 +127,16 @@ def build_native_scope_index( ] ], ) -> NativeScopeIndex | None: ... +def parse_model_header_metadata( + requests: list[tuple[object, object, dict[str, Any]]], classes: dict[str, object] +) -> list[tuple[tuple[Any, ...], tuple[Any, ...]] | str]: ... +def config_contains_template(value: object) -> bool | None: ... +def config_contains_macro_call(value: object) -> bool | None: ... +def expand_config_templates( + value: object, + sources: tuple[dict[str, object], object, dict[str, str | None]], + flags: tuple[bool, bool, bool], +) -> tuple[object, list[tuple[str, str]]] | str: ... def scope_expected_model_names( sqls: list[str], syntax: dict[str, object] ) -> list[list[str] | None]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 6e3fb87045..9a64b6cca5 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -13,13 +13,13 @@ from sqlbuild.compiler.compile._helpers.analysis.validation import validate_sql_syntax from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - _contains_model_config_macro_cached, - _contains_template_data_cached, _model_sql_validation_gate, _resolve_model_schema, _validate_model_header_tags, build_layered_model_values, build_model_header_schema_entry, + contains_config_macro_calls, + contains_config_templates, find_matching_path_default, find_schema_model_match, strip_model_header_metadata_from_config, @@ -139,6 +139,12 @@ PythonHookEntry, SqlHookEntry, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.model_config.main._parse_native_header_metadata import ( + parse_native_header_metadata, +) +from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -268,6 +274,7 @@ class _ModelInputLoop: config_scan_cache: ModelConfigScanCache reusable_config_cache: _ReusableModelConfigCache declaration_cache: _VisibleModelDeclarationCache + native_header_metadata: dict[Path, NativeHeaderMetadata] @dataclass(frozen=True) @@ -507,6 +514,7 @@ def _build_model_inputs( strict=True, ) ) + native_model_config: bool = native_stage_enabled(NativeStage.MODEL_CONFIG) loop: _ModelInputLoop = _ModelInputLoop( discovered_inputs=discovered_inputs, context=context, @@ -514,13 +522,16 @@ def _build_model_inputs( sql_hook_definitions=sql_hook_definitions, legacy_schema_files=legacy_schema_files, model_header_column_cache=ModelHeaderColumnCache(), - config_scan_cache=ModelConfigScanCache(), + config_scan_cache=ModelConfigScanCache(native=native_model_config), reusable_config_cache=_ReusableModelConfigCache( defaults=discovered_inputs.project_config.defaults, path_defaults=discovered_inputs.project_config.path_defaults, target_config=context.target_config, ), declaration_cache=_VisibleModelDeclarationCache.build(context), + native_header_metadata=( + parse_native_header_metadata(model_files=render_files) if native_model_config else {} + ), ) model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile @@ -755,6 +766,7 @@ def _build_model_input( model_schema_description=model_schema.description if model_schema is not None else None, audit_factories=discovered_inputs.audit_factories, column_cache=model_header_column_cache, + native_metadata=loop.native_header_metadata.get(model_file.file_path), ) model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) header_schema_entry, enum_columns = resolve_enum_contract_columns( @@ -1204,6 +1216,7 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi request.materialization_defaults ) scan_cache: ModelConfigScanCache | None = request.scan_cache + native: bool = scan_cache is not None and scan_cache.native _validate_model_header_tags(model_header_values=model_header_values, model_name=model_name) layered_values: dict[str, object] = build_layered_model_values( @@ -1220,9 +1233,8 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi } for hook_key in raw_hook_values: del layered_values[hook_key] - has_authored_templates: bool = _contains_template_data_cached( - value=layered_values, - cache=scan_cache.template_presence if scan_cache is not None else None, + has_authored_templates: bool = contains_config_templates( + values=layered_values, scan_cache=scan_cache ) if has_authored_templates: early_resolved_values: dict[str, object] = resolve_early_model_templates( @@ -1230,12 +1242,14 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi effective_vars=effective_vars, effective_target_name=effective_target_name, run_id=run_id, + native=native, ) model_resolved_values: dict[str, object] = resolve_chained_model_context_templates( values=early_resolved_values, model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, + native=native, ) else: model_resolved_values = layered_values @@ -1266,6 +1280,7 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi run_id=run_id, include_target_values=False, ), + native=native, ) target_resolved_values: dict[str, object] = ( resolve_target_context_templates( @@ -1273,6 +1288,7 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, + native=native, ) if has_authored_templates or contains_template_data(model_resolved_values.get("database")) @@ -1292,10 +1308,7 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi and target_resolved_values.get("materialized") != MaterializationType.INCREMENTAL ): target_resolved_values.pop(MODEL_FULL_REFRESH_CONFIG_KEY, None) - if _contains_model_config_macro_cached( - values=target_resolved_values, - cache=scan_cache.macro_presence if scan_cache is not None else None, - ): + if contains_config_macro_calls(values=target_resolved_values, scan_cache=scan_cache): validate_model_config_has_no_macros(values=target_resolved_values) return CompileModelConfig( values=target_resolved_values, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index fc5f99faaf..f075087d2c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -26,12 +26,13 @@ MODEL_AUDIT_OVERRIDE_KEYS, MODEL_HEADER_METADATA_KEYS, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeModelConfigMismatchError from sqlbuild.compiler.compile.models import ( CachedModelHeaderColumns, CompileModelConfig, CompileModelInput, IdentityPresenceCache, + ModelConfigScanCache, ModelHeaderColumnCache, ) from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns @@ -41,6 +42,13 @@ DiscoveredSqlModelFile, ModelSchemaDeclaration, ) +from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( + native_config_contains_macro_call, +) +from sqlbuild.compiler.model_config.main._native_config_contains_template import ( + native_config_contains_template, +) +from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.compiler.path_defaults.main._select import select_path_default from sqlbuild.spec.contracts.models import ( DefaultsConfig, @@ -73,6 +81,36 @@ def _contains_template_data_cached(*, value: object, cache: IdentityPresenceCach return result +def contains_config_templates( + *, values: dict[str, object], scan_cache: ModelConfigScanCache | None +) -> bool: + """Return whether layered model config holds a `${...}` template, scanning natively if set.""" + + if scan_cache is not None and scan_cache.native: + native: bool | None = native_config_contains_template(values) + if native is not None: + return native + return _contains_template_data_cached( + value=values, cache=scan_cache.template_presence if scan_cache is not None else None + ) + + +def contains_config_macro_calls( + *, values: dict[str, object], scan_cache: ModelConfigScanCache | None +) -> bool: + """Return whether model config outside hooks holds a macro call, scanning natively if set.""" + + if scan_cache is not None and scan_cache.native: + native: bool | None = native_config_contains_macro_call( + [value for key, value in values.items() if key not in _MODEL_HOOK_KEYS] + ) + if native is not None: + return native + return _contains_model_config_macro_cached( + values=values, cache=scan_cache.macro_presence if scan_cache is not None else None + ) + + def _contains_model_config_macro_cached( *, values: dict[str, object], cache: IdentityPresenceCache | None ) -> bool: @@ -253,6 +291,7 @@ def build_model_header_schema_entry( model_schema_description: str | None = None, audit_factories: tuple[DiscoveredAuditFactory, ...] = (), column_cache: ModelHeaderColumnCache | None = None, + native_metadata: NativeHeaderMetadata | None = None, ) -> SchemaModelEntry | None: """Normalize model-owned MODEL(...) metadata into the existing schema entry shape.""" @@ -279,11 +318,23 @@ def build_model_header_schema_entry( error_class=CompileInputError, ) description: str | None = model_description or model_schema_description - local_columns: tuple[SchemaColumn, ...] = _parse_model_header_columns( - raw_columns=raw_columns, - file_path=file_path, - column_locations=column_locations or {}, - column_cache=column_cache, + native: NativeHeaderMetadata | None = ( + native_metadata + if native_metadata is not None + and native_metadata.applies_to( + raw_columns=raw_columns, raw_audits=raw_audits, column_locations=column_locations + ) + else None + ) + local_columns: tuple[SchemaColumn, ...] = ( + native.columns + if native is not None and not native.invalid + else _parse_model_header_columns( + raw_columns=raw_columns, + file_path=file_path, + column_locations=column_locations or {}, + column_cache=column_cache, + ) ) columns: tuple[SchemaColumn, ...] = _merge_model_schema_columns( model_name=model_name, @@ -296,13 +347,22 @@ def build_model_header_schema_entry( model_name=model_name, file_path=file_path, ) - audits: tuple[SchemaAuditInstance, ...] = parse_audit_instances( - raw_audits=raw_audits, - file_path=file_path, - label="model", - error_class=CompileInputError, - null_as_empty=True, + audits: tuple[SchemaAuditInstance, ...] = ( + native.audits + if native is not None and not native.invalid + else parse_audit_instances( + raw_audits=raw_audits, + file_path=file_path, + label="model", + error_class=CompileInputError, + null_as_empty=True, + ) ) + if native is not None and native.invalid: + raise NativeModelConfigMismatchError( + f"{file_path}: native model config rejected MODEL columns or audits that the Python " + "model config accepts; run with SQLBUILD_COMPILER_ENGINE=python" + ) generated_audits: tuple[SchemaAuditInstance, ...] = parse_model_header_audit_factories( raw_audit_factories=raw_audit_factories, file_path=file_path, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index 0cd7368083..5d9f7a896c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -7,8 +7,14 @@ from uuid import uuid4 from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data -from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, PRESERVE_TARGET_VALUE +from sqlbuild.compiler.compile.exceptions import NativeModelConfigMismatchError from sqlbuild.compiler.compile.types import CompileContextKey +from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ, INVALID_OUTCOME +from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( + expand_native_config_templates, +) +from sqlbuild.compiler.model_config.models import NativeTemplateExpansion, TemplateResolutionFlags from sqlbuild.spec.contracts.models import TargetConfig @@ -18,12 +24,13 @@ def resolve_early_model_templates( effective_vars: dict[str, object], effective_target_name: str | None, run_id: str, + native: bool = False, ) -> dict[str, object]: """Resolve `${name}`, `${ENV:...}`, and early `run.*` model templates.""" return cast( dict[str, object], - expand_template_data( + expand_config_templates( value=values, variables=effective_vars, context_values=build_run_context_values( @@ -34,6 +41,7 @@ def resolve_early_model_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=True, + native=native, ), ) @@ -44,12 +52,13 @@ def resolve_model_context_templates( model_name: str, effective_target_name: str | None, run_id: str, + native: bool = False, ) -> dict[str, object]: """Resolve model-bound `CTX` values once logical model identity is known.""" return cast( dict[str, object], - expand_template_data( + expand_config_templates( value=values, variables={}, context_values=build_model_context_values( @@ -63,6 +72,7 @@ def resolve_model_context_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=True, + native=native, ), ) @@ -73,6 +83,7 @@ def resolve_chained_model_context_templates( model_name: str, effective_target_name: str | None, run_id: str, + native: bool = False, ) -> dict[str, object]: """Resolve twice so ${CTX:model.*} values may chain exactly one level without looping.""" @@ -81,12 +92,14 @@ def resolve_chained_model_context_templates( model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, + native=native, ) return resolve_model_context_templates( values=first_pass_values, model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, + native=native, ) @@ -96,12 +109,13 @@ def resolve_target_context_templates( model_name: str, effective_target_name: str | None, run_id: str, + native: bool = False, ) -> dict[str, object]: """Resolve late `target.*` values after environment overrides finalize naming.""" return cast( dict[str, object], - expand_template_data( + expand_config_templates( value=values, variables={}, context_values=build_model_context_values( @@ -115,6 +129,7 @@ def resolve_target_context_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, + native=native, ), ) @@ -180,6 +195,7 @@ def apply_environment_database_schema_overrides( effective_vars: dict[str, object], target_config: TargetConfig | None, model_context_values: dict[str, str | None], + native: bool = False, ) -> dict[str, object]: """Return values with environment database/schema overrides applied.""" @@ -188,7 +204,7 @@ def apply_environment_database_schema_overrides( overridden: dict[str, object] = dict(values) if target_config.database is not None and target_config.database != PRESERVE_TARGET_VALUE: - overridden["database"] = expand_template_data( + overridden["database"] = expand_config_templates( value=target_config.database, variables=effective_vars, context_values=model_context_values, @@ -196,9 +212,10 @@ def apply_environment_database_schema_overrides( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, + native=native, ) if target_config.schema is not None and target_config.schema != PRESERVE_TARGET_VALUE: - overridden["schema"] = expand_template_data( + overridden["schema"] = expand_config_templates( value=target_config.schema, variables=effective_vars, context_values=model_context_values, @@ -206,6 +223,7 @@ def apply_environment_database_schema_overrides( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, + native=native, ) return overridden @@ -218,3 +236,58 @@ def resolve_run_id(*, selected_run_id: str | None) -> str: timestamp_prefix: str = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ") unique_suffix: str = uuid4().hex[:12] return f"{timestamp_prefix}_{unique_suffix}" + + +def expand_config_templates( + *, + value: object, + variables: dict[str, object], + context_values: dict[str, str | None], + context_label: str, + allow_context: bool, + preserve_context_tokens: bool, + preserve_unknown_context: bool, + native: bool, +) -> object: + """Expand like `expand_template_data`, natively when set; native rejections re-run Python.""" + + outcome: NativeTemplateExpansion | str | None = ( + expand_native_config_templates( + value=value, + variables=variables, + context_values=context_values, + flags=TemplateResolutionFlags( + allow_context=allow_context, + preserve_context_tokens=preserve_context_tokens, + preserve_unknown_context=preserve_unknown_context, + ), + ) + if native + else None + ) + if isinstance(outcome, NativeTemplateExpansion): + _record_template_reads(outcome.reads) + return outcome.value + expanded: object = expand_template_data( + value=value, + variables=variables, + context_values=context_values, + context_label=context_label, + allow_context=allow_context, + preserve_context_tokens=preserve_context_tokens, + preserve_unknown_context=preserve_unknown_context, + ) + if outcome == INVALID_OUTCOME: + raise NativeModelConfigMismatchError( + f"Native model config rejected {context_label} templates that the Python model " + "config expands; run with SQLBUILD_COMPILER_ENGINE=python" + ) + return expanded + + +def _record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: + for kind, name in reads: + if kind == ENVIRONMENT_READ: + COMPILE_INPUT_READS.environment_read(name) + else: + COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 5d849a9b26..f681db9f46 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -29,3 +29,7 @@ class CompactAnalysisInputError(CompileInputError): class AnalysisCacheEntryError(ValueError): """Raised when a persisted model analysis cache entry is invalid.""" + + +class NativeModelConfigMismatchError(RuntimeError): + """Raised when native model config rejects a value the Python model config accepts.""" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 17de6fa5f4..88b2dd7b71 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1564,6 +1564,7 @@ class ModelConfigScanCache: template_presence: IdentityPresenceCache = field(default_factory=IdentityPresenceCache) macro_presence: IdentityPresenceCache = field(default_factory=IdentityPresenceCache) + native: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index cfc1e9fd6d..d0e5cd5007 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -18,6 +18,7 @@ NATIVE_STAGE_TIERS: dict[NativeStage, NativeStageTier] = { NativeStage.DISCOVERY: NativeStageTier.SHIPPED, NativeStage.DECLARATION_SCOPES: NativeStageTier.PREVIEW, + NativeStage.MODEL_CONFIG: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index dbeb1fe7d3..2acb0641f6 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -16,6 +16,7 @@ class NativeStage(StrEnum): DISCOVERY = "discovery" DECLARATION_SCOPES = "declaration_scopes" + MODEL_CONFIG = "model_config" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/model_config/__init__.py b/src/sqlbuild/compiler/model_config/__init__.py new file mode 100644 index 0000000000..be257e2ac0 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/__init__.py @@ -0,0 +1 @@ +"""Native model configuration for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/model_config/constants.py b/src/sqlbuild/compiler/model_config/constants.py new file mode 100644 index 0000000000..c770c0fc2a --- /dev/null +++ b/src/sqlbuild/compiler/model_config/constants.py @@ -0,0 +1,32 @@ +"""Contract dataclass shapes the native header metadata parser builds.""" + +from __future__ import annotations + +SCHEMA_COLUMN_FIELDS: tuple[str, ...] = ( + "name", + "type", + "nullable", + "description", + "meta", + "audits", + "location", + "migrate_from", +) +SCHEMA_AUDIT_INSTANCE_FIELDS: tuple[str, ...] = ( + "definition_name", + "arguments", + "name", + "description", + "severity", + "run_scope", + "always_run", + "thresholds", + "minimum_samples", + "evidence_limit", + "location", +) +INVALID_OUTCOME: str = "invalid" +UNSUPPORTED_OUTCOME: str = "unsupported" +ENVIRONMENT_READ: str = "env" +COLUMNS_HEADER_KEY: str = "columns" +AUDITS_HEADER_KEY: str = "audits" diff --git a/src/sqlbuild/compiler/model_config/main/__init__.py b/src/sqlbuild/compiler/model_config/main/__init__.py new file mode 100644 index 0000000000..8665a5bb71 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/__init__.py @@ -0,0 +1 @@ +"""Native model configuration entry points.""" diff --git a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py new file mode 100644 index 0000000000..2b657cc936 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py @@ -0,0 +1,34 @@ +"""Expand `${...}` templates in model config natively for the preview compiler engine.""" + +from __future__ import annotations + +import os + +import sqlbuild._native as _native +from sqlbuild.compiler.model_config.constants import UNSUPPORTED_OUTCOME +from sqlbuild.compiler.model_config.models import ( + NativeTemplateExpansion, + TemplateResolutionFlags, +) + + +def expand_native_config_templates( + *, + value: object, + variables: dict[str, object], + context_values: dict[str, str | None], + flags: TemplateResolutionFlags, +) -> NativeTemplateExpansion | str: + """Return the expansion, or `invalid` (Python raises) or `unsupported` (Python must run).""" + + try: + outcome: tuple[object, list[tuple[str, str]]] | str = _native.expand_config_templates( + value, + (variables, os.environ, context_values), + (flags.allow_context, flags.preserve_context_tokens, flags.preserve_unknown_context), + ) + except TypeError: + return UNSUPPORTED_OUTCOME + if isinstance(outcome, str): + return outcome + return NativeTemplateExpansion(value=outcome[0], reads=tuple(outcome[1])) diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py b/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py new file mode 100644 index 0000000000..0a4fc7eed4 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py @@ -0,0 +1,11 @@ +"""Macro call presence over authored model config, answered natively.""" + +from __future__ import annotations + +import sqlbuild._native as _native + + +def native_config_contains_macro_call(value: object) -> bool | None: + """Return whether any nested string holds a macro call, or None where Python must scan.""" + + return _native.config_contains_macro_call(value) diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py b/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py new file mode 100644 index 0000000000..897e41ee03 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py @@ -0,0 +1,11 @@ +"""Template presence over authored model config, answered natively.""" + +from __future__ import annotations + +import sqlbuild._native as _native + + +def native_config_contains_template(value: object) -> bool | None: + """Return whether any nested string holds `${`, or None where Python must scan.""" + + return _native.config_contains_template(value) diff --git a/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py b/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py new file mode 100644 index 0000000000..626043f964 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py @@ -0,0 +1,82 @@ +"""Parse MODEL header columns and audits natively for the preview compiler engine.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import fields +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.auditing.types import AuditSeverity +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.model_config.constants import ( + AUDITS_HEADER_KEY, + COLUMNS_HEADER_KEY, + INVALID_OUTCOME, + SCHEMA_AUDIT_INSTANCE_FIELDS, + SCHEMA_COLUMN_FIELDS, + UNSUPPORTED_OUTCOME, +) +from sqlbuild.compiler.model_config.models import NativeHeaderMetadata +from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn + + +def parse_native_header_metadata( + *, model_files: Sequence[DiscoveredSqlModelFile] +) -> dict[Path, NativeHeaderMetadata]: + """Return native header metadata by model file; models Python must parse are absent.""" + + if not model_files or not _contract_shapes_match(): + return {} + try: + parsed: list[tuple[tuple[SchemaColumn, ...], tuple[SchemaAuditInstance, ...]] | str] = ( + _native.parse_model_header_metadata( + [ + ( + model_file.header_values.get(COLUMNS_HEADER_KEY), + model_file.header_values.get(AUDITS_HEADER_KEY), + model_file.header_column_locations, + ) + for model_file in model_files + ], + { + "schema_column": SchemaColumn, + "schema_audit_instance": SchemaAuditInstance, + "severities": {severity.value: severity for severity in AuditSeverity}, + "allocate": object.__new__, + }, + ) + ) + except (TypeError, ValueError): + return {} + return { + model_file.file_path: _native_metadata(model_file=model_file, metadata=metadata) + for model_file, metadata in zip(model_files, parsed, strict=True) + if metadata != UNSUPPORTED_OUTCOME + } + + +def _native_metadata( + *, + model_file: DiscoveredSqlModelFile, + metadata: tuple[tuple[SchemaColumn, ...], tuple[SchemaAuditInstance, ...]] | str, +) -> NativeHeaderMetadata: + columns, audits = ((), ()) if isinstance(metadata, str) else metadata + return NativeHeaderMetadata( + raw_columns=model_file.header_values.get(COLUMNS_HEADER_KEY), + raw_audits=model_file.header_values.get(AUDITS_HEADER_KEY), + column_locations=model_file.header_column_locations, + columns=columns, + audits=audits, + invalid=metadata == INVALID_OUTCOME, + ) + + +def _contract_shapes_match() -> bool: + return ( + tuple(field.name for field in fields(SchemaColumn)) == SCHEMA_COLUMN_FIELDS + and tuple(field.name for field in fields(SchemaAuditInstance)) + == SCHEMA_AUDIT_INSTANCE_FIELDS + and not hasattr(SchemaColumn, "__post_init__") + and not hasattr(SchemaAuditInstance, "__post_init__") + ) diff --git a/src/sqlbuild/compiler/model_config/models.py b/src/sqlbuild/compiler/model_config/models.py new file mode 100644 index 0000000000..4e53ab4450 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/models.py @@ -0,0 +1,51 @@ +"""Natively parsed MODEL header metadata and the authored values it was parsed from.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn, SourceLocation + + +@dataclass(frozen=True, slots=True) +class NativeHeaderMetadata: + """One model's header columns and audits; `invalid` means Python must parse them and raise.""" + + raw_columns: object | None + raw_audits: object | None + column_locations: dict[str, SourceLocation] + columns: tuple[SchemaColumn, ...] + audits: tuple[SchemaAuditInstance, ...] + invalid: bool = False + + def applies_to( + self, + *, + raw_columns: object | None, + raw_audits: object | None, + column_locations: dict[str, SourceLocation] | None, + ) -> bool: + """Return whether this parse was made from exactly these authored objects.""" + + return ( + raw_columns is self.raw_columns + and raw_audits is self.raw_audits + and column_locations is self.column_locations + ) + + +@dataclass(frozen=True, slots=True) +class NativeTemplateExpansion: + """A natively expanded config value and the environment and context names it read, in order.""" + + value: object + reads: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True, slots=True) +class TemplateResolutionFlags: + """How one template expansion treats `CTX:` references.""" + + allow_context: bool + preserve_context_tokens: bool + preserve_unknown_context: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index fe4db75a86..5d40696aa0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -840,3 +840,16 @@ class CompilerEngineTierStoreTestCase: expected_reused: tuple[bool, ...] expected_first_engine_stores: tuple[str, ...] expected_second_engine_stores: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeModelConfigParityTestCase: + """Authored model config compiled by two engines that must report and write the same.""" + + description: str + project_files: dict[str, str] + project_config_replacements: tuple[tuple[str, str], ...] + environment: dict[str, str] + engines: tuple[str, str] + expected_exit_codes: tuple[int, int] + expected_report_text: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f389e87a07..a784a02d06 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2966,6 +2966,7 @@ def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, i '\n[rules]\nselect = ["XSQBR"]\n\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n' ) _FUTURE_MTIME_OFFSET_NS: int = 5_000_000_000 +_PHASE_DURATION: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") _PROGRESS_LINE: re.Pattern[str] = re.compile(r".+ (START|OK \(\d+\.\d+s\))") _COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') _COMPILE_REUSE_EXTRA_PROJECT_FILES: dict[str, str] = { @@ -3093,6 +3094,12 @@ def compile_reuse_hit_count(*, report_path: Path) -> int: return cast(dict[str, int], payload["compile_timings"])["project_reuse_hits"] +def stderr_without_durations(*, stderr: str) -> str: + """Return compile stderr without its phase durations.""" + + return _PHASE_DURATION.sub("", stderr) + + def compile_notes(*, stderr: str) -> list[str]: """Return the note lines a compile printed to stderr.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_config.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_config.py new file mode 100644 index 0000000000..dcab92987b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_config.py @@ -0,0 +1,129 @@ +"""Native model config in the preview engine compiles exactly as the Python engine does.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + NativeModelConfigParityTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + prepare_compile_reuse_project, + replace_project_text, + report_without_engine, + run_reuse_compile, + stderr_without_durations, + write_project_file, +) + +_ORDER_QUALITY_MODEL: str = """MODEL ( + materialized table, + description "Order quality checks.", + schema "${coalesce(ENV:SQB_ORDER_QUALITY_SCHEMA, 'quality')}_${CTX:run.target}", + tags ["${if(eq(CTX:run.target, 'dev'), 'development', 'release')}"], + audits [ + expression_is_true ( + name "quantity_is_positive", + expression "quantity > 0", + severity error, + ), + ], + columns ( + order_id ( + type INTEGER, + nullable false, + description "Order key", + audits [not_null (severity error, name order_id_present), unique], + ), + status ( + type VARCHAR, + audits [ + accepted_values ( + values ["placed", "preparing", "ready", "completed", "cancelled"], + severity warn, + description "Known statuses", + always_run true, + ), + ], + ), + Quantity (type INTEGER, description "Waffles ordered"), + ), +); + +SELECT order_id, status, quantity AS Quantity +FROM __ref("stg_orders") +""" +_INVALID_SEVERITY_MODEL: str = """MODEL ( + materialized table, + columns ( + order_id (audits [not_null (severity fatal)]), + ), +); + +SELECT order_id FROM __ref("stg_orders") +""" +_TARGET_SCHEMA: tuple[str, str] = ( + 'schema = "dev"', + "schema = \"${coalesce(ENV:SQB_TARGET_SCHEMA, ENV:SQB_MISSING_SCHEMA, 'dev')}\"", +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeModelConfigParityTestCase( + description="header_metadata_and_templates", + project_files={"models/marts/order_quality.sql": _ORDER_QUALITY_MODEL}, + project_config_replacements=(_TARGET_SCHEMA,), + environment={"SQB_TARGET_SCHEMA": "analytics"}, + engines=("python", "native-preview"), + expected_exit_codes=(0, 0), + expected_report_text="order_quality", + ), + NativeModelConfigParityTestCase( + description="python_error_for_an_invalid_audit_severity", + project_files={"models/marts/order_quality.sql": _INVALID_SEVERITY_MODEL}, + project_config_replacements=(), + environment={}, + engines=("python", "native-preview"), + expected_exit_codes=(1, 1), + expected_report_text="severity", + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_model_config_when_compiling_with_preview_then_output_matches_python( + test_case: NativeModelConfigParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + for relative_path, contents in test_case.project_files.items(): + write_project_file(prepared_project, relative_path, contents) + for old, new in test_case.project_config_replacements: + replace_project_text(prepared_project, "sqlbuild_project.toml", old, new) + runs: list[CompileReuseRun] = [ + run_reuse_compile( + project_dir=copy_compile_project( + source=prepared_project, destination=tmp_path / engine + ), + env=test_case.environment, + global_args=("--compiler-engine", engine), + ) + for engine in test_case.engines + ] + + assert tuple(run.returncode for run in runs) == test_case.expected_exit_codes + assert report_without_engine(runs[1]) == report_without_engine(runs[0]) + assert stderr_without_durations(stderr=runs[1].stderr) == stderr_without_durations( + stderr=runs[0].stderr + ) + assert runs[1].compiled == runs[0].compiled + assert test_case.expected_report_text in runs[0].report + runs[0].stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/__init__.py b/tests/integration/src/sqlbuild/compiler/model_config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py new file mode 100644 index 0000000000..0c76e9dc78 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -0,0 +1,61 @@ +"""Test case types for the native model configuration parity tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class HeaderMetadataParityTestCase: + """Seeded MODEL header metadata parsed natively and in Python.""" + + description: str + seed: int + count: int + expected_minimum_parsed: int + expected_minimum_rejected: int + expected_minimum_unsupported: int + + +@dataclass(frozen=True) +class ConfigPresenceParityTestCase: + """Seeded config values scanned for templates and macro calls natively and in Python.""" + + description: str + seed: int + count: int + expected_minimum_present: int + expected_minimum_deferred: int + + +@dataclass(frozen=True) +class ModelConfigTierTestCase: + """One compiler engine and how often its model loop calls each native model config entry.""" + + description: str + engine: str + expected_native_calls: dict[str, int] + + +@dataclass(frozen=True) +class ConfigTemplateParityTestCase: + """Seeded config values whose templates expand natively and in Python.""" + + description: str + seed: int + count: int + allow_context: bool + preserve_context_tokens: bool + preserve_unknown_context: bool + expected_minimum_expanded: int + expected_minimum_rejected: int + expected_minimum_unsupported: int + + +@dataclass(frozen=True) +class NativeRejectionTestCase: + """A native entry that rejects valid config, which the Python re-run must expose.""" + + description: str + native_entry: str + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py new file mode 100644 index 0000000000..1e6153f2e4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -0,0 +1,762 @@ +"""Seeded MODEL header values and engine-parity outcomes for the native model config tests.""" + +from __future__ import annotations + +import os +import random +from collections.abc import Callable +from dataclasses import dataclass, fields +from itertools import compress +from pathlib import Path +from typing import cast + +import pytest + +import sqlbuild._native as _native +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances +from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates +from sqlbuild.compiler.compile._helpers.render.templating import ( + contains_template_data, + expand_template_data, +) +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, MACRO_CALL_PATTERN +from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeModelConfigMismatchError +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs +from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.model_config.constants import INVALID_OUTCOME, UNSUPPORTED_OUTCOME +from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( + native_config_contains_macro_call, +) +from sqlbuild.compiler.model_config.main._native_config_contains_template import ( + native_config_contains_template, +) +from sqlbuild.compiler.model_config.main._parse_native_header_metadata import ( + parse_native_header_metadata, +) +from sqlbuild.compiler.model_config.models import NativeHeaderMetadata +from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( + resolve_effective_collection_rendering, +) +from sqlbuild.spec.contracts.models import SourceLocation +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_TEXTS: tuple[object, ...] = ( + "INTEGER", + "Order amount", + "caf\u00e9 total", + "", + " ", + "\u00a0", + "${target.schema}", + "@cents(amount)", + "@cents\u2003(amount)", + "@cents\x1f(amount)", + "mail@example.com", + "\x1c", + 1, + True, + None, + 1.5, + ("tuple",), +) +_AUDIT_NAMES: tuple[str, ...] = ( + "not_null", + "unique", + "accepted_values", + "NotNull", + "row_count_", + "", + "_private", + "a", +) +_COLUMN_NAMES: tuple[str, ...] = ( + "order_id", + "Order_ID", + "amount", + "caf\u00e9", + "status", + " ", + "\u00a0", + "id", +) +_COLUMN_KEYS: tuple[str, ...] = ("type", "nullable", "description", "audits", "migrate_from") +_AUDIT_OPTIONS: tuple[tuple[str, tuple[object, ...]], ...] = ( + ("name", ("custom_name", "Bad Name", "", None, 1)), + ("description", ("Checks the rows", "", None)), + ("severity", ("warn", "error", "fatal", None, 1)), + ("run_scope", ("final", "", None)), + ("always_run", (True, False, None, "yes")), + ("thresholds", (None, {"warn": {"above": 1}})), + ("minimum_samples", (0, 5, -1, True, None, 2**70)), + ("evidence_limit", (0, 10, -3, None)), + ("values", (["PLACED", "SHIPPED"], None, {"nested": [1, 2]})), + ("minimum", (1, 2.5)), +) + + +_RAISES: str = "Python raises" +NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ( + "parse_model_header_metadata", + "expand_config_templates", + "config_contains_template", + "config_contains_macro_call", +) +MODEL_CONFIG_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = "orders.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n\n' + '[targets.dev]\nconnection = "local"\n' + "schema = \"${coalesce(ENV:SQB_MISSING_SCHEMA, 'dev')}_${CTX:model.name}\"\n" + ), + "models/orders.sql": ( + 'MODEL (\n materialized table,\n description "Orders.",\n' + " tags [\"${if(eq(CTX:run.target, 'dev'), 'development', 'release')}\"],\n" + " columns (\n" + " order_id (type INTEGER, nullable false, audits [not_null (severity error), unique]),\n" + ' status (audits [accepted_values (values ["placed", "shipped"])]),\n' + " ),\n);\n\nSELECT 1 AS order_id, 'placed' AS status\n" + ), + "models/customers.sql": ( + "MODEL (\n materialized view,\n columns (customer_id (type INTEGER)),\n);\n\n" + "SELECT 1 AS customer_id\n" + ), +} + + +@dataclass(frozen=True) +class HeaderMetadataParity: + """How the native header metadata parse compared with Python's over one corpus.""" + + mismatches: list[tuple[object, object, object]] + parsed: int + rejected: int + unsupported: int + + +@dataclass(frozen=True) +class ConfigTemplateParity: + """How native template expansion compared with Python's over one corpus.""" + + mismatches: list[tuple[object, object, object]] + expanded: int + rejected: int + unsupported: int + + +@dataclass(frozen=True) +class TemplateFlags: + """The resolver flags one expansion runs with.""" + + allow_context: bool + preserve_context_tokens: bool + preserve_unknown_context: bool + + +@dataclass(frozen=True) +class ConfigPresenceParity: + """How the native presence scans compared with Python's over one corpus.""" + + mismatches: list[tuple[object, object, object]] + present: int + deferred: int + + +def generated_header_metadata(*, rng: random.Random, count: int) -> list[tuple[object, object]]: + """Return seeded `(columns, audits)` header values mixing valid and invalid shapes.""" + + return [(_columns(rng=rng), _audit_list(rng=rng)) for _ in range(count)] + + +def generated_config_values(*, rng: random.Random, count: int) -> list[object]: + """Return seeded nested config values holding templates, macro calls and plain text.""" + + return [_config_value(rng=rng, depth=0) for _ in range(count)] + + +def model_config_engine_outcome( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[dict[str, int], str]: + """Build compile inputs under `engine`; count native entry calls and describe model config.""" + + for relative_path, contents in MODEL_CONFIG_PROJECT.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + calls: dict[str, int] = dict.fromkeys(NATIVE_MODEL_CONFIG_ENTRIES, 0) + for name in NATIVE_MODEL_CONFIG_ENTRIES: + monkeypatch.setattr(_native, name, _counted(calls=calls, name=name)) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + adapter: DuckDbAdapter = DuckDbAdapter() + inputs: CompileProjectInputs = build_compile_inputs( + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="20261007T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + return calls, repr( + [(model.config, model.schema_entry, model.enum_columns) for model in inputs.model_inputs] + ) + + +def native_rejection_error( + *, project_dir: Path, native_entry: str, monkeypatch: pytest.MonkeyPatch +) -> str: + """Build model inputs in preview while `native_entry` rejects everything; return the error.""" + + monkeypatch.setattr(_native, native_entry, _REJECTIONS[native_entry]) + try: + _ = model_config_engine_outcome( + project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch + ) + except NativeModelConfigMismatchError as error: + return str(error) + return "no error" + + +def _reject_header_metadata(requests: list[object], classes: object) -> list[str]: + del classes + return [INVALID_OUTCOME for _ in requests] + + +def _reject_templates(*args: object) -> str: + del args + return INVALID_OUTCOME + + +_REJECTIONS: dict[str, Callable[..., object]] = { + "parse_model_header_metadata": _reject_header_metadata, + "expand_config_templates": _reject_templates, +} + + +def _counted(*, calls: dict[str, int], name: str) -> Callable[..., object]: + entry: Callable[..., object] = getattr(_native, name) + + def counted(*args: object) -> object: + calls[name] += 1 + return entry(*args) + + return counted + + +def generated_template_values(*, rng: random.Random, count: int) -> list[object]: + """Return seeded config values holding valid, invalid and unusual `${...}` templates.""" + + return [_template_value(rng=rng, depth=0) for _ in range(count)] + + +def config_template_parity(*, values: list[object], flags: TemplateFlags) -> ConfigTemplateParity: + """Compare native expansion, its reads and its rejections with Python's expansion.""" + + answered: list[object] = list( + compress( + values, + [ + _native_classification(value=value, flags=flags) != UNSUPPORTED_OUTCOME + for value in values + ], + ) + ) + outcomes: list[tuple[object, object, object]] = [ + (value, _python_expansion(value=value, flags=flags), _native_expansion(value, flags)) + for value in answered + ] + return ConfigTemplateParity( + mismatches=mismatches( + inputs=[value for value, _, _ in outcomes], + expected=[expected for _, expected, _ in outcomes], + actual=[actual for _, _, actual in outcomes], + ), + expanded=sum(actual != _RAISES for _, _, actual in outcomes), + rejected=sum(actual == _RAISES for _, _, actual in outcomes), + unsupported=len(values) - len(outcomes), + ) + + +def _python_expansion(*, value: object, flags: TemplateFlags) -> object: + with COMPILE_INPUT_READS.recording() as reads: + try: + result: object = expand_template_data( + value=value, + variables=TEMPLATE_VARIABLES, + context_values=TEMPLATE_CONTEXT, + context_label="model config", + allow_context=flags.allow_context, + preserve_context_tokens=flags.preserve_context_tokens, + preserve_unknown_context=flags.preserve_unknown_context, + ) + except CompileInputError: + return _RAISES + return (_expansion_shape(result), reads.environment_names, reads.read_run_id) + + +def _native_classification(*, value: object, flags: TemplateFlags) -> object: + return _native.expand_config_templates( + value, + (TEMPLATE_VARIABLES, os.environ, TEMPLATE_CONTEXT), + (flags.allow_context, flags.preserve_context_tokens, flags.preserve_unknown_context), + ) + + +def _native_expansion(value: object, flags: TemplateFlags) -> object: + with COMPILE_INPUT_READS.recording() as reads: + try: + result: object = expand_config_templates( + value=value, + variables=TEMPLATE_VARIABLES, + context_values=TEMPLATE_CONTEXT, + context_label="model config", + allow_context=flags.allow_context, + preserve_context_tokens=flags.preserve_context_tokens, + preserve_unknown_context=flags.preserve_unknown_context, + native=True, + ) + except CompileInputError: + return _RAISES + except NativeModelConfigMismatchError as error: + return str(error) + return (_expansion_shape(result), reads.environment_names, reads.read_run_id) + + +def _expansion_shape(value: object) -> object: + return (repr(value), _type_tree(value)) + + +def _type_tree(value: object) -> object: + children: tuple[object, ...] = _CHILDREN.get(type(value), _no_children)(value) + return (type(value).__name__, tuple(_type_tree(item) for item in children)) + + +def _no_children(value: object) -> tuple[object, ...]: + del value + return () + + +def _mapping_children(value: object) -> tuple[object, ...]: + return tuple(cast(dict[object, object], value).values()) + + +def _sequence_children(value: object) -> tuple[object, ...]: + return tuple(cast(tuple[object, ...], value)) + + +_CHILDREN: dict[type, Callable[[object], tuple[object, ...]]] = { + dict: _mapping_children, + list: _sequence_children, + tuple: _sequence_children, +} + + +def header_metadata_parity(*, headers: list[tuple[object, object]]) -> HeaderMetadataParity: + """Compare the native parse or rejection with Python's wherever native answers.""" + + model_files: list[DiscoveredSqlModelFile] = [ + _model_file(index=index, columns=columns, audits=audits) + for index, (columns, audits) in enumerate(headers) + ] + native: dict[Path, NativeHeaderMetadata] = parse_native_header_metadata(model_files=model_files) + parsed: list[NativeHeaderMetadata | None] = [ + native.get(model_file.file_path) for model_file in model_files + ] + python: list[object] = [_python_metadata(model_file=model_file) for model_file in model_files] + compared: list[tuple[DiscoveredSqlModelFile, object, NativeHeaderMetadata | None]] = list( + compress( + zip(model_files, python, parsed, strict=True), + [metadata is not None for metadata in parsed], + ) + ) + return HeaderMetadataParity( + mismatches=mismatches( + inputs=[model_file.header_values for model_file, _, _ in compared], + expected=[_shape(value) for _, value, _ in compared], + actual=[_native_shape(metadata) for _, _, metadata in compared], + ), + parsed=sum(not metadata.invalid for metadata in native.values()), + rejected=sum(metadata.invalid for metadata in native.values()), + unsupported=len(model_files) - len(compared), + ) + + +def config_presence_parity(*, values: list[object]) -> ConfigPresenceParity: + """Compare the native template and macro scans with Python's wherever they answer.""" + + native: list[tuple[bool | None, bool | None]] = [ + (native_config_contains_template(value), native_config_contains_macro_call(value)) + for value in values + ] + python: list[tuple[bool, bool]] = [ + (contains_template_data(value), _python_contains_macro(value)) for value in values + ] + answered: list[tuple[object, tuple[bool, bool], tuple[bool | None, bool | None]]] = list( + compress( + zip(values, python, native, strict=True), + [None not in answer for answer in native], + ) + ) + return ConfigPresenceParity( + mismatches=mismatches( + inputs=[value for value, _, _ in answered], + expected=[expected for _, expected, _ in answered], + actual=[actual for _, _, actual in answered], + ), + present=sum(any(expected) for _, expected, _ in answered), + deferred=len(values) - len(answered), + ) + + +def _python_contains_macro(value: object) -> bool: + return _MACRO_SCANS.get(type(value), _absent)(value) + + +def _absent(value: object) -> bool: + del value + return False + + +def _string_contains_macro(value: object) -> bool: + return MACRO_CALL_PATTERN.search(str(value)) is not None + + +def _mapping_contains_macro(value: object) -> bool: + return any(_python_contains_macro(item) for item in cast(dict[object, object], value).values()) + + +def _sequence_contains_macro(value: object) -> bool: + return any(_python_contains_macro(item) for item in cast(tuple[object, ...], value)) + + +_MACRO_SCANS: dict[type, Callable[[object], bool]] = { + str: _string_contains_macro, + dict: _mapping_contains_macro, + list: _sequence_contains_macro, + tuple: _sequence_contains_macro, +} + + +def _python_metadata(*, model_file: DiscoveredSqlModelFile) -> object: + try: + return ( + parse_schema_columns( + raw_columns=model_file.header_values.get("columns"), + file_path=model_file.relative_path, + label="model", + error_class=CompileInputError, + column_locations=model_file.header_column_locations or {}, + allow_migrate_from=True, + ), + parse_audit_instances( + raw_audits=model_file.header_values.get("audits"), + file_path=model_file.relative_path, + label="model", + error_class=CompileInputError, + null_as_empty=True, + ), + ) + except Exception as error: + return f"{type(error).__name__}: {error}" + + +def _shape(value: object) -> object: + return _SHAPES.get(type(value), _raises)(value) + + +def _raises(value: object) -> object: + del value + return _RAISES + + +def _metadata_shape(value: object) -> object: + return (repr(value), _attribute_orders(value), _locations(value)) + + +def _native_shape(metadata: NativeHeaderMetadata | None) -> object: + parsed: NativeHeaderMetadata = cast(NativeHeaderMetadata, metadata) + return _NATIVE_SHAPES[parsed.invalid](parsed) + + +def _parsed_shape(metadata: NativeHeaderMetadata) -> object: + return _shape((metadata.columns, metadata.audits)) + + +def _rejected_shape(metadata: NativeHeaderMetadata) -> object: + del metadata + return _RAISES + + +_NATIVE_SHAPES: dict[bool, Callable[[NativeHeaderMetadata], object]] = { + False: _parsed_shape, + True: _rejected_shape, +} + + +_SHAPES: dict[type, Callable[[object], object]] = {tuple: _metadata_shape} + + +def _attribute_orders(value: object) -> object: + return _item_orders(_flattened(value)) + + +def _item_orders(items: tuple[object, ...]) -> object: + return tuple( + (type(item).__name__, tuple(vars(item)), _item_orders(getattr(item, "audits", ()))) + for item in items + ) + + +def _locations(value: object) -> object: + return tuple(_item_locations(item) for item in _flattened(value)) + + +def _item_locations(item: object) -> object: + audit_locations: tuple[object, ...] = tuple( + audit.location for audit in getattr(item, "audits", ()) + ) + return (getattr(item, "location", None), audit_locations) + + +def _flattened(value: object) -> tuple[object, ...]: + columns, audits = cast(tuple[tuple[object, ...], tuple[object, ...]], value) + return (*columns, *audits) + + +def _model_file(*, index: int, columns: object, audits: object) -> DiscoveredSqlModelFile: + relative_path: Path = Path("models") / f"model_{index}.sql" + names: list[str] = [*filter(_is_text, getattr(columns, "keys", tuple)())] + locations: dict[str, SourceLocation] = { + name: SourceLocation(path=relative_path, line=line + 2, column=3) + for line, name in enumerate(names[: len(names) * (index % 2)]) + } + model_file: DiscoveredSqlModelFile = object.__new__(DiscoveredSqlModelFile) + values: dict[str, object] = {field.name: None for field in fields(DiscoveredSqlModelFile)} + values.update( + file_path=Path("/project") / relative_path, + relative_path=relative_path, + header_values={"columns": columns, "audits": audits}, + header_column_locations=locations, + ) + for name, value in values.items(): + object.__setattr__(model_file, name, value) + return model_file + + +def _is_text(value: object) -> bool: + return isinstance(value, str) + + +def _columns(*, rng: random.Random) -> object: + return rng.choices( + (None, rng.choice(([], "columns", 1, ("order_id",))), _column_mapping(rng=rng)), + weights=(8, 4, 88), + )[0] + + +def _column_mapping(*, rng: random.Random) -> dict[object, object]: + return { + rng.choices((rng.choice(_COLUMN_NAMES), rng.choice((1, None))), weights=(9, 1))[0]: ( + _column(rng=rng) + ) + for _ in range(rng.randint(0, 5)) + } + + +def _column(*, rng: random.Random) -> object: + keys: list[str] = rng.sample(_COLUMN_KEYS, k=rng.randint(0, len(_COLUMN_KEYS))) + column: dict[object, object] = {key: _COLUMN_VALUES[key](rng) for key in keys} + extra_keys: tuple[object, ...] = rng.choices( + ((), (rng.choice(("format", 1)),)), weights=(19, 1) + )[0] + column.update(dict.fromkeys(extra_keys, "x")) + return rng.choices((column, rng.choice((None, "INTEGER", []))), weights=(19, 1))[0] + + +def _text(rng: random.Random) -> object: + return rng.choice(_TEXTS) + + +def _nullable(rng: random.Random) -> object: + return rng.choice((True, False, None, "yes")) + + +def _audits(rng: random.Random) -> object: + return _audit_list(rng=rng) + + +_COLUMN_VALUES: dict[str, Callable[[random.Random], object]] = { + "type": _text, + "nullable": _nullable, + "description": _text, + "audits": _audits, + "migrate_from": _text, +} + + +def _audit_list(*, rng: random.Random) -> object: + return rng.choices( + ( + None, + rng.choice(("not_null", {"unique": None}, ("unique",))), + [_audit(rng=rng) for _ in range(rng.randint(0, 3))], + ), + weights=(30, 4, 66), + )[0] + + +def _audit(*, rng: random.Random) -> object: + name: str = rng.choice(_AUDIT_NAMES) + options: list[tuple[str, tuple[object, ...]]] = rng.sample(_AUDIT_OPTIONS, k=rng.randint(0, 4)) + return rng.choices( + ( + name, + {name: None}, + rng.choice(({name: None, "unique": None}, {1: None}, {name: "x"}, 3)), + {name: {key: rng.choice(choices) for key, choices in options}}, + ), + weights=(35, 10, 5, 50), + )[0] + + +def _template_value(*, rng: random.Random, depth: int) -> object: + factories: tuple[Callable[[], object], ...] = ( + lambda: _template_text(rng=rng), + lambda: rng.choice(_TEXTS), + lambda: [_template_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))], + lambda: tuple(_template_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 2))), + lambda: { + rng.choice(("schema", "database", "${key}", 1)): _template_value( + rng=rng, depth=depth + 1 + ) + for _ in range(rng.randint(0, 3)) + }, + ) + return rng.choices(factories, weights=(60 + 1000 * (depth > 1), 10, 10, 5, 15))[0]() + + +def _template_text(*, rng: random.Random) -> str: + parts: list[str] = [ + rng.choices( + (f"${{{_template_expression(rng=rng, depth=0)}}}", rng.choice(_TEMPLATE_NOISE)), + weights=(4, 1), + )[0] + for _ in range(rng.randint(1, 3)) + ] + text: str = "".join(parts) + noise: str = rng.choices((rng.choice(_TEMPLATE_NOISE), ""), weights=(15, 85))[0] + position: int = rng.randrange(len(text) + 1) + return text[:position] + noise + text[position:] + + +def _template_expression(*, rng: random.Random, depth: int) -> str: + factories: tuple[Callable[[], str], ...] = ( + lambda: rng.choice(_TEMPLATE_REFERENCES), + lambda: rng.choice(_TEMPLATE_LITERALS), + lambda: "{}({})".format( + rng.choice(_TEMPLATE_FUNCTIONS), + rng.choice((",", ", ", " ,\t", "\x1f,")).join( + _template_expression(rng=rng, depth=depth + 1) + for _ in range(rng.choice((0, 1, 2, 2, 3, 3, 4))) + ), + ), + ) + return rng.choices(factories, weights=(5, 3, 4 - 3 * (depth > 2)))[0]() + + +TEMPLATE_VARIABLES: dict[str, object] = { + "flag": True, + "zero": 0, + "count": 12, + "big": 2**70, + "env": "prod", + "spaced": " FALSE ", + "ratio": 1.5, + "items": [1, "a"], + "mapping": {"a": 1}, + "none": None, + "unicode": "caf\u00e9", +} +TEMPLATE_CONTEXT: dict[str, str | None] = { + "run.id": "20261007T000000Z_abc", + "run.target": "prod", + "model.name": "orders", + "model.schema": None, +} +TEMPLATE_ENVIRONMENT: dict[str, str] = { + "SQB_TEMPLATE_SCHEMA": "analytics", + "SQB_TEMPLATE_EMPTY": "", + "SQB_TEMPLATE_ZERO": "0", +} +_TEMPLATE_REFERENCES: tuple[str, ...] = ( + *TEMPLATE_VARIABLES, + "missing", + "ENV:SQB_TEMPLATE_SCHEMA", + "ENV:SQB_TEMPLATE_EMPTY", + "ENV:SQB_TEMPLATE_ZERO", + "ENV:SQB_TEMPLATE_MISSING", + "CTX:run.id", + "CTX:run.target", + "CTX:model.name", + "CTX:model.schema", + "CTX:model.alias", + "OTHER:name", + "true", + "false", + "null", +) +_TEMPLATE_LITERALS: tuple[str, ...] = ( + "'prod'", + '"0"', + "''", + "' False '", + "'a\\'b'", + "'caf\u00e9'", + "'unterminated", + "'trailing\\", +) +_TEMPLATE_FUNCTIONS: tuple[str, ...] = ("if", "eq", "ne", "coalesce", "upper") +_TEMPLATE_NOISE: tuple[str, ...] = ( + "orders_", + "${", + "}", + "{", + "$", + "${}", + "(", + ")", + ",", + "\u00a0", + "\u2003", + "\x1c", + " ", + "caf\u00e9", +) + + +def _config_value(*, rng: random.Random, depth: int) -> object: + factories: tuple[Callable[[], object], ...] = ( + lambda: rng.choice(_TEXTS), + lambda: [_config_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))], + lambda: tuple(_config_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))), + lambda: { + rng.choice(("schema", "${key}", "tags", 1)): _config_value(rng=rng, depth=depth + 1) + for _ in range(rng.randint(0, 3)) + }, + ) + return rng.choices(factories, weights=(50 + 1000 * (depth > 2), 20, 10, 20))[0]() diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py new file mode 100644 index 0000000000..509a1ebdba --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -0,0 +1,237 @@ +"""The native model config stage matches Python exactly or defers to it.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + ConfigPresenceParityTestCase, + ConfigTemplateParityTestCase, + HeaderMetadataParityTestCase, + ModelConfigTierTestCase, + NativeRejectionTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( + TEMPLATE_ENVIRONMENT, + ConfigPresenceParity, + ConfigTemplateParity, + HeaderMetadataParity, + TemplateFlags, + config_presence_parity, + config_template_parity, + generated_config_values, + generated_header_metadata, + generated_template_values, + header_metadata_parity, + model_config_engine_outcome, + native_rejection_error, +) + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderMetadataParityTestCase( + description="seeded columns and audits, valid and invalid", + seed=20261007, + count=4000, + expected_minimum_parsed=400, + expected_minimum_rejected=2000, + expected_minimum_unsupported=100, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_header_metadata_when_parsing_then_native_matches_python_or_defers( + test_case: HeaderMetadataParityTestCase, +) -> None: + parity: HeaderMetadataParity = header_metadata_parity( + headers=generated_header_metadata(rng=random.Random(test_case.seed), count=test_case.count) + ) + + assert ( + parity.mismatches, + parity.parsed >= test_case.expected_minimum_parsed, + parity.rejected >= test_case.expected_minimum_rejected, + parity.unsupported >= test_case.expected_minimum_unsupported, + ) == ([], True, True, True), (parity.parsed, parity.rejected, parity.unsupported) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigPresenceParityTestCase( + description="seeded nested config with templates and macro calls", + seed=20261008, + count=4000, + expected_minimum_present=400, + expected_minimum_deferred=20, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_config_when_scanning_presence_then_native_matches_python_or_defers( + test_case: ConfigPresenceParityTestCase, +) -> None: + parity: ConfigPresenceParity = config_presence_parity( + values=generated_config_values(rng=random.Random(test_case.seed), count=test_case.count) + ) + + assert ( + parity.mismatches, + parity.present >= test_case.expected_minimum_present, + parity.deferred >= test_case.expected_minimum_deferred, + ) == ([], True, True), (parity.present, parity.deferred) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigTemplateParityTestCase( + description="model config resolution preserving unknown context", + seed=20261009, + count=4000, + allow_context=True, + preserve_context_tokens=False, + preserve_unknown_context=True, + expected_minimum_expanded=800, + expected_minimum_rejected=400, + expected_minimum_unsupported=100, + ), + ConfigTemplateParityTestCase( + description="target resolution rejecting unknown context", + seed=20261010, + count=4000, + allow_context=True, + preserve_context_tokens=False, + preserve_unknown_context=False, + expected_minimum_expanded=800, + expected_minimum_rejected=400, + expected_minimum_unsupported=100, + ), + ConfigTemplateParityTestCase( + description="context disallowed but preserved", + seed=20261011, + count=4000, + allow_context=False, + preserve_context_tokens=True, + preserve_unknown_context=False, + expected_minimum_expanded=800, + expected_minimum_rejected=400, + expected_minimum_unsupported=100, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_templates_when_expanding_then_native_matches_python_or_defers( + test_case: ConfigTemplateParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + for name, value in TEMPLATE_ENVIRONMENT.items(): + monkeypatch.setenv(name, value) + + parity: ConfigTemplateParity = config_template_parity( + values=generated_template_values(rng=random.Random(test_case.seed), count=test_case.count), + flags=TemplateFlags( + allow_context=test_case.allow_context, + preserve_context_tokens=test_case.preserve_context_tokens, + preserve_unknown_context=test_case.preserve_unknown_context, + ), + ) + + assert ( + parity.mismatches, + parity.expanded >= test_case.expected_minimum_expanded, + parity.rejected >= test_case.expected_minimum_rejected, + parity.unsupported >= test_case.expected_minimum_unsupported, + ) == ([], True, True, True), (parity.expanded, parity.rejected, parity.unsupported) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelConfigTierTestCase( + description="python oracle", + engine="python", + expected_native_calls={ + "parse_model_header_metadata": 0, + "expand_config_templates": 0, + "config_contains_template": 0, + "config_contains_macro_call": 0, + }, + ), + ModelConfigTierTestCase( + description="shipped native stages only", + engine="native", + expected_native_calls={ + "parse_model_header_metadata": 0, + "expand_config_templates": 0, + "config_contains_template": 0, + "config_contains_macro_call": 0, + }, + ), + ModelConfigTierTestCase( + description="native preview", + engine="native-preview", + expected_native_calls={ + "parse_model_header_metadata": 1, + "expand_config_templates": 6, + "config_contains_template": 2, + "config_contains_macro_call": 2, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_tier_when_building_model_inputs_then_native_config_runs_only_in_preview( + test_case: ModelConfigTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls, config = model_config_engine_outcome( + project_dir=tmp_path / test_case.engine, engine=test_case.engine, monkeypatch=monkeypatch + ) + _, python_config = model_config_engine_outcome( + project_dir=tmp_path / "python", engine="python", monkeypatch=monkeypatch + ) + + assert (calls, config.replace(test_case.engine, "python")) == ( + test_case.expected_native_calls, + python_config, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeRejectionTestCase( + description="header metadata", + native_entry="parse_model_header_metadata", + expected_message="native model config rejected MODEL columns or audits", + ), + NativeRejectionTestCase( + description="templates", + native_entry="expand_config_templates", + expected_message="templates that the Python model config expands", + ), + ], + ids=lambda case: case.description, +) +def test_given_native_rejects_valid_config_when_python_accepts_then_a_mismatch_is_raised( + test_case: NativeRejectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + message: str = native_rejection_error( + project_dir=tmp_path, native_entry=test_case.native_entry, monkeypatch=monkeypatch + ) + + assert ( + test_case.expected_message in message, + "SQLBUILD_COMPILER_ENGINE=python" in message, + ) == ( + True, + True, + ), message + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 826a5946bd687b417c4c3a4ecf2447a1840b7ff9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:52:43 +0100 Subject: [PATCH 3/3] feat(native): cover model configuration in the compiler differential Seeds now generate templated targets and model headers with typed, audited columns, render coverage requires them, and the failure corpus adds model config errors. --- .../_helpers/corpus/render_failure_cases.py | 56 +++++++++++++- .../_helpers/coverage/render.py | 22 ++++++ .../classes/discovery_features.py | 10 ++- .../classes/project_builder.py | 20 ++++- .../classes/render_features.py | 34 +++++++++ scripts/compiler_differential/constants.py | 5 ++ .../compiler_differential/_test_types.py | 13 ++++ .../scripts/compiler_differential/helpers.py | 25 +++++++ .../test_compiler_differential.py | 70 +++++++++++------ .../sqlbuild/compiler/pipeline/_test_types.py | 3 + .../src/sqlbuild/compiler/pipeline/helpers.py | 75 +++++++++++++++++-- .../test_dataflow_analysis_schedule.py | 28 ++++++- 12 files changed, 325 insertions(+), 36 deletions(-) diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index 55db1432e5..cfe2c9ebcd 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -10,6 +10,7 @@ FAILURE_BASE_MART, FAILURE_BASE_STAGING, FAILURE_MART_PATH, + GENERATOR_MISSING_ENV_VAR, ) from scripts.compiler_differential.models import FailureCase @@ -59,7 +60,13 @@ def _staging_columns(*, columns: str, files: dict[str, str] | None = None) -> di def render_failure_cases() -> tuple[FailureCase, ...]: """Return the render-time failure cases in a stable order.""" - return (*_macro_cases(), *_interpolation_cases(), *_reference_cases(), *_attachment_cases()) + return ( + *_macro_cases(), + *_interpolation_cases(), + *_reference_cases(), + *_attachment_cases(), + *_model_config_cases(), + ) def _macro_cases() -> tuple[FailureCase, ...]: @@ -352,3 +359,50 @@ def _attachment_cases() -> tuple[FailureCase, ...]: }, ), ) + + +def _model_config_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="model-column-audit-unknown-severity", + expected_code="P001", + expected_message="severity", + files=_staging_header(" columns (amount (audits [not_null (severity fatal)])),"), + ), + failure_case( + name="model-duplicate-column", + expected_code="P001", + expected_message="duplicate", + files=_staging_header(" columns (amount (type DOUBLE), AMOUNT (type DOUBLE)),"), + ), + failure_case( + name="model-blank-column-type", + expected_code="P001", + expected_message="non-empty string", + files=_staging_header(' columns (amount (type " ")),'), + ), + failure_case( + name="model-nullable-column-with-not-null", + expected_code="P002", + expected_message="not_null", + files=_staging_header(" columns (amount (nullable true, audits [not_null])),"), + ), + failure_case( + name="model-template-missing-environment", + expected_code="P001", + expected_message="missing ENV variable", + files=_staging_header(f' schema "${{ENV:{GENERATOR_MISSING_ENV_VAR}}}",'), + ), + failure_case( + name="model-template-unknown-function", + expected_code="P001", + expected_message="unsupported template function", + files=_staging_header(" schema \"${upper('orders')}\","), + ), + failure_case( + name="model-template-unterminated-string", + expected_code="P001", + expected_message="unterminated", + files=_staging_header(' schema "${coalesce(\'orders)}",'), + ), + ) diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 828659d4a4..772a64666b 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -60,6 +60,8 @@ } _REFERENCE_COLLECTIONS: tuple[str, ...] = ("model_inputs", "sql_function_inputs", "audit_inputs") _RELATION_REFERENCE_KINDS: frozenset[str] = frozenset({"ref", "source", "seed"}) +_TEMPLATE_OPEN: str = "${" +_TARGET_NAMESPACE_KEYS: tuple[str, ...] = ("database", "schema") def required_render_kinds() -> tuple[str, ...]: @@ -111,6 +113,7 @@ def rendered_input_kinds(capture_text: str) -> frozenset[str]: kinds.update(_hook_kinds(capture)) kinds.update(_reference_kinds(capture)) kinds.update(_attachment_kinds(capture)) + kinds.update(_model_config_kinds(capture)) return frozenset(kinds) @@ -419,3 +422,22 @@ def _attachment_kinds(capture: dict[str, object]) -> set[str]: if config.get("matched_path_default") is not None: kinds.add("path_default") return kinds + + +def _model_config_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + target: dict[str, object] = as_json_object(capture.get("effective_target")) or {} + if any(_TEMPLATE_OPEN in str(target.get(key)) for key in _TARGET_NAMESPACE_KEYS): + kinds.add("target_namespace_template") + for model in records(capture.get("model_inputs")): + model_file: dict[str, object] = as_json_object(model.get("model_file")) or {} + header: dict[str, object] = as_json_object(model_file.get("header_values")) or {} + if any(_TEMPLATE_OPEN in json.dumps(value) for value in header.values()): + kinds.add("model_header_template") + schema_entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + for column in records(schema_entry.get("columns")): + if column.get("type") is not None and any( + audit.get("severity") is not None for audit in records(column.get("audits")) + ): + kinds.add("model_column_audit_options") + return kinds diff --git a/scripts/compiler_differential/classes/discovery_features.py b/scripts/compiler_differential/classes/discovery_features.py index d0a30ddd73..4874620df5 100644 --- a/scripts/compiler_differential/classes/discovery_features.py +++ b/scripts/compiler_differential/classes/discovery_features.py @@ -15,6 +15,7 @@ GENERATOR_FEATURE_BLOCKS, GENERATOR_FEATURE_FOLDER, GENERATOR_FEATURE_STRIDE, + GENERATOR_MISSING_ENV_VAR, GENERATOR_OPTIONAL_FEATURE_SHARE, GENERATOR_PROJECT_ADAPTER, GENERATOR_RARE_FEATURE_BLOCKS, @@ -491,7 +492,14 @@ def _local_config(self) -> None: def _target_override(self) -> None: self._features.add("target_override") self.config_lines.extend( - ["", "[targets.ci]", 'schema = "ci_checks"', "", "[targets.ci.vars]", 'region = "west"'] + [ + "", + "[targets.ci]", + f"schema = \"${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, 'ci_checks')}}\"", + "", + "[targets.ci.vars]", + 'region = "west"', + ] ) self._files.setdefault("sqlbuild_local.toml", "") self._files["sqlbuild_local.toml"] = ( diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index 43138485c0..03b776ddbf 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -24,6 +24,7 @@ GENERATOR_INVALID_SHARE, GENERATOR_LAYERS, GENERATOR_MACRO_KIND, + GENERATOR_MISSING_ENV_VAR, GENERATOR_NON_ASCII_DESCRIPTION_SHARE, GENERATOR_NON_ASCII_LABELS, GENERATOR_RARE_FEATURE_BLOCKS, @@ -110,6 +111,7 @@ def build(self) -> GeneratedProject: domains=domains, adapter=extras.adapter, extra_lines=[*extras.config_lines, *rendering.config_lines], + templated_target=rendering.templated_target, ) if invalid_kind is not None: self._inject_invalid(kind=invalid_kind) @@ -125,6 +127,11 @@ def build(self) -> GeneratedProject: succeeding_commands=extras.succeeding_commands, ) + def _target_schema(self, *, schema: str, templated: bool) -> str: + if not templated: + return schema + return f"${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, '{schema}')}}" + def _chance(self, share: float) -> bool: return self._random.random() < share @@ -132,7 +139,12 @@ def _label(self) -> str: return self._random.choice(GENERATOR_NON_ASCII_LABELS) def _write_config( - self, *, domains: list[str], adapter: str | None, extra_lines: list[str] + self, + *, + domains: list[str], + adapter: str | None, + extra_lines: list[str], + templated_target: bool, ) -> None: lines: list[str] = [ f'name = "generated_{self._seed}"', @@ -148,7 +160,11 @@ def _write_config( f'site_label = "{self._label()}"', "", "[targets.dev]", - f'schema = "{self._random.choice(("analytics", "main"))}"', + 'schema = "{}"'.format( + self._target_schema( + schema=self._random.choice(("analytics", "main")), templated=templated_target + ) + ), "", "[settings]", f'default_audit_severity = "{self._random.choice(("warn", "error"))}"', diff --git a/scripts/compiler_differential/classes/render_features.py b/scripts/compiler_differential/classes/render_features.py index fa791e7ed0..8f574e4ca0 100644 --- a/scripts/compiler_differential/classes/render_features.py +++ b/scripts/compiler_differential/classes/render_features.py @@ -7,6 +7,7 @@ from scripts.compiler_differential.constants import ( GENERATOR_CHANNEL_ENV_VAR, + GENERATOR_MISSING_ENV_VAR, GENERATOR_RENDER_FOLDER, GENERATOR_RENDER_TEST_FOLDER, ) @@ -35,6 +36,7 @@ def __init__( self._sources: list[str] = sources self._label: Callable[[], str] = label self.config_lines: list[str] = [] + self.templated_target: bool = False def write(self) -> None: """Write the selected rendering blocks in their canonical order.""" @@ -49,6 +51,7 @@ def write(self) -> None: "macros_across_resources": self._macros_across_resources, "enum_column_contract": self._enum_column_contract, "resource_audits": self._resource_audits, + "model_config": self._model_config, } for block in self._blocks: writer: Callable[[], None] | None = writers.get(block) @@ -415,3 +418,34 @@ def _resource_audits(self) -> None: "GROUP BY c.label\n" ), ) + + def _model_config(self) -> None: + self._features.add("model_config") + self.templated_target = True + base: ModelPlan = self._base() + self._files[self._owned(role="audits/generic", name="configured_floor.sql")] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE amount < @minimum\n' + ) + self._model( + name=f"configured_{base.name}", + description=f"Configured copy of {base.name} (réglé)", + header=( + " materialized table,\n" + f' schema "${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, ' + f"'configured')}}_${{CTX:run.target}}\",\n" + " tags [\"${if(eq(CTX:run.target, 'dev'), 'development', 'release')}\"],\n" + " audits [\n" + f" configured_floor (minimum {self._random.randint(-40, -2)}, severity warn,\n" + ' name configured_amount_floor, description "Amounts stay above the floor"),\n' + " ],\n" + " columns (\n" + ' id (type INTEGER, nullable false, description "Row key",\n' + " audits [not_null (severity error, name configured_id_present),\n" + " unique (always_run true)]),\n" + ' amount (type DOUBLE, description "Montant réglé"),\n' + ' status (audits [accepted_values (values ["placed", "shipped"],\n' + ' severity warn, description "Known statuses")]),\n' + " ),\n" + ), + body=f'SELECT id, amount, status\nFROM __ref("{base.name}")\n', + ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 05f3f8cdac..acbcc32f7e 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -191,6 +191,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "macros_across_resources", "enum_column_contract", "resource_audits", + "model_config", ) GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( "incremental_append", @@ -231,6 +232,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_RENDER_FOLDER: str = "models/rendering" GENERATOR_RENDER_TEST_FOLDER: str = "tests/unit/rendering" GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" +GENERATOR_MISSING_ENV_VAR: str = "SQB_DIFFERENTIAL_UNSET" GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" GENERATOR_CRLF: str = "\r\n" @@ -451,6 +453,9 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "source_column_audit", "seed_column_audit", "path_default", + "model_header_template", + "target_namespace_template", + "model_column_audit_options", ) RENDER_INDIRECT_KINDS: dict[str, str] = { kind: "the used macro's own function source reads it; the capture does not record context reads" diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index 2056886b3d..9c434d9283 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -17,6 +17,19 @@ class HarnessRunTestCase: expected_absent: tuple[str, ...] +@dataclass(frozen=True) +class SharedAnalysisSeedTestCase: + """A generated seed that must compile cleanly and reuse shared analyses.""" + + description: str + seed: int + expected_lines: tuple[str, ...] + expected_capture_sides: tuple[str, ...] + expected_compile_exit_code: int + expected_minimum_shareable_members: int + expected_minimum_shared_reuse: int + + @dataclass(frozen=True) class ProjectExpectationTestCase: """A broken project run with one --expect value and the verdict the harness must reach.""" diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index 059fbbedba..61cb287cc4 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -137,3 +137,28 @@ def perturbation_arguments(directory: Path, *, source: str = _SITECUSTOMIZE) -> "--engine-env", f"native-preview:PYTHONPATH={write_native_perturbation(directory, source=source)}", ) + + +SHARED_ANALYSIS_SEED: int = 34 +COMPILED_PROJECT_CAPTURE: str = "0-compile/003-compiled_project.json" + + +def shared_analysis_seed_arguments(*, work_dir: Path, seed: int) -> list[str]: + """Harness arguments comparing one generated seed with itself, capturing every stage.""" + + return [ + "--jobs", + "1", + "--work-dir", + str(work_dir), + "--corpus", + "seeds", + "--seed-start", + str(seed), + "--seeds", + "1", + "--engines", + "python", + "python", + "--stage-captures", + ] diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index ba415b93f9..3faf8d875d 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import shutil from pathlib import Path import pytest @@ -12,16 +13,25 @@ CoverageFailureTestCase, HarnessRunTestCase, ProjectExpectationTestCase, + SharedAnalysisSeedTestCase, ) from tests.e2e.scripts.compiler_differential.helpers import ( + COMPILED_PROJECT_CAPTURE, DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, RENDER_PERTURBATION, + SHARED_ANALYSIS_SEED, harness_arguments, perturbation_arguments, + shared_analysis_seed_arguments, write_broken_ref_project, write_failure_case_project, ) +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + CompiledProjectRun, + compiled_project_capture, + use_wave_analysis, +) @pytest.mark.parametrize( @@ -271,42 +281,56 @@ def test_given_required_coverage_missing_when_comparing_then_harness_fails_and_s @pytest.mark.parametrize( "test_case", [ - HarnessRunTestCase( + SharedAnalysisSeedTestCase( description="shared_analysis_seed_against_itself", - extra_arguments=( - "--corpus", - "seeds", - "--seed-start", - "29", - "--seeds", - "1", - "--engines", - "python", - "python", - "--stage-captures", - ), - expected_exit_code=0, + seed=SHARED_ANALYSIS_SEED, expected_lines=( - "OK seed/29", + f"OK seed/{SHARED_ANALYSIS_SEED}", "Compiler differential passed: 1 projects identical (python vs python)", ), - expected_patterns=(), - expected_absent=("DIFF",), + expected_capture_sides=("left-python-captures", "right-python-captures"), + expected_compile_exit_code=0, + expected_minimum_shareable_members=1, + expected_minimum_shared_reuse=1, ) ], ids=lambda case: case.description, ) def test_given_python_engine_twice_when_capturing_stages_then_captures_are_identical( - test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: SharedAnalysisSeedTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + work_dir: Path = tmp_path / "work" exit_code: int = run_compiler_differential( - ["--jobs", "1", "--work-dir", str(tmp_path / "work"), *test_case.extra_arguments] + shared_analysis_seed_arguments(work_dir=work_dir, seed=test_case.seed) ) - output: str = capsys.readouterr().out - assert exit_code == test_case.expected_exit_code, output - assert all(line in output for line in test_case.expected_lines), output - assert not any(text in output for text in test_case.expected_absent), output + seed_dir: Path = work_dir / f"seed__{test_case.seed}" + project_dir: Path = Path(shutil.copytree(seed_dir / "source", tmp_path / "project")) + with monkeypatch.context() as waves_patch: + use_wave_analysis(waves_patch) + compiled: CompiledProjectRun = compiled_project_capture( + project_dir=project_dir, + capture_dir=tmp_path / "captures", + capsys=capsys, + monkeypatch=waves_patch, + ) + + assert (exit_code, all(line in output for line in test_case.expected_lines)) == (0, True), ( + output + ) + assert "DIFF" not in output, output + assert all( + (seed_dir / side / COMPILED_PROJECT_CAPTURE).is_file() + for side in test_case.expected_capture_sides + ), output + assert ( + compiled.exit_code, + compiled.shareable_members >= test_case.expected_minimum_shareable_members, + compiled.shared_reuse >= test_case.expected_minimum_shared_reuse, + ) == (test_case.expected_compile_exit_code, True, True), compiled @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 0e6963f61c..f34443dbe1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -445,6 +445,9 @@ class DataflowCaptureCase: description: str write_project: Callable[[Path], object] schedules: tuple[DataflowScheduleCase, ...] + expected_exit_code: int + expected_minimum_shareable_members: int + expected_minimum_shared_reuse: int expected_difference: str | None = None diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 52cea2d482..21b584ac37 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -11,7 +11,7 @@ from contextlib import suppress from pathlib import Path from types import ModuleType -from typing import Any, cast +from typing import Any, NamedTuple, cast import orjson import pytest @@ -58,6 +58,23 @@ SharedBindingQueryCase, ) + +class CompiledProjectRun(NamedTuple): + """One cold compile's exit code, compiled-project capture and shared-analysis reuse.""" + + exit_code: int + capture: Path + shareable_members: int + deduplicated_queries: int + shared_memo_hits: int + + @property + def shared_reuse(self) -> int: + """Return how many member analyses were served by another member's analysis.""" + + return self.deduplicated_queries + self.shared_memo_hits + + _REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] _NORMALIZATION_MODULES: tuple[ModuleType, ...] = (normalize_batch, binding_sharing, compact) _ANALYSIS_CACHE_METRICS: tuple[str, ...] = ( @@ -314,14 +331,62 @@ def compiled_project_capture( capture_dir: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, -) -> Path: - """Cold-compile once with stage captures; return the compiled-project capture file.""" +) -> CompiledProjectRun: + """Cold-compile once with stage captures, counting how often shared analyses were reused.""" + keys: list[object] = [] + deduplicated: list[int] = [] + memo_hits: list[int] = [] with monkeypatch.context() as capture_patch: capture_patch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(capture_dir)) - _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + record_shared_analysis_reuse( + monkeypatch=capture_patch, keys=keys, deduplicated=deduplicated, memo_hits=memo_hits + ) + exit_code: int = main( + ["--project-dir", str(project_dir), "compile", "--json", "--no-cache"] + ) _ = capsys.readouterr() - return next(capture_dir.glob(f"*-{CompilerStage.COMPILED_PROJECT.value}.json")) + return CompiledProjectRun( + exit_code=exit_code, + capture=next(capture_dir.glob(f"*-{CompilerStage.COMPILED_PROJECT.value}.json")), + shareable_members=len(keys) - len(set(keys)), + deduplicated_queries=sum(deduplicated), + shared_memo_hits=sum(memo_hits), + ) + + +def record_shared_analysis_reuse( + *, + monkeypatch: pytest.MonkeyPatch, + keys: list[object], + deduplicated: list[int], + memo_hits: list[int], +) -> None: + """Record shared-result keys, each batch's deduplicated queries and its memo hits.""" + + member_keys: Callable[..., tuple[object, ...]] = compact.shared_result_keys + + def counted_keys(**kwargs: Any) -> tuple[object, ...]: + computed: tuple[object, ...] = member_keys(**kwargs) + keys.extend(filter(None, computed)) + return computed + + prepare: Callable[..., CompactBatchPreparation] = compact._prepare_compact_analysis_batch + reuse: Callable[..., Mapping[int, object]] = compact.reused_shared_results + + def counted_prepare(**kwargs: Any) -> CompactBatchPreparation: + preparation: CompactBatchPreparation = prepare(**kwargs) + deduplicated.append(len(preparation.cleaned_sql) - len(preparation.queries)) + return preparation + + def counted_reuse(**kwargs: Any) -> Mapping[int, object]: + reused: Mapping[int, object] = reuse(**kwargs) + memo_hits.append(len(reused)) + return reused + + monkeypatch.setattr(compact, "_prepare_compact_analysis_batch", counted_prepare) + monkeypatch.setattr(compact, "reused_shared_results", counted_reuse) + monkeypatch.setattr(compact, "shared_result_keys", counted_keys) def reshape_models(*, project_dir: Path, names: tuple[str, ...], indexes: tuple[int, ...]) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_dataflow_analysis_schedule.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_dataflow_analysis_schedule.py index 6b951d7fa0..79314aadea 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_dataflow_analysis_schedule.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_dataflow_analysis_schedule.py @@ -24,6 +24,7 @@ DataflowStartFailureCase, ) from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + CompiledProjectRun, analysis_pool_thread_ids, compile_outcome, compiled_project_capture, @@ -39,7 +40,7 @@ ) _PROJECT: RandomDagProject = RandomDagProject(seed=97, model_count=40, errors=True) -_SHARED_ANALYSIS_SEED: int = 29 +_SHARED_ANALYSIS_SEED: int = 34 _CAPTURE_SCHEDULES: tuple[DataflowScheduleCase, ...] = ( DataflowScheduleCase("one worker with single models", 1, 1, 0.0), DataflowScheduleCase("four workers with small batches", 4, 2, 0.004), @@ -100,11 +101,17 @@ def test_given_perturbed_schedule_when_compiling_repeatedly_then_output_is_ident "random dag", random_dag_writer(_PROJECT), _CAPTURE_SCHEDULES, + expected_exit_code=1, + expected_minimum_shareable_members=0, + expected_minimum_shared_reuse=0, ), DataflowCaptureCase( "differential seed with shared analyses", generate_project(seed=_SHARED_ANALYSIS_SEED).write, _CAPTURE_SCHEDULES, + expected_exit_code=0, + expected_minimum_shareable_members=1, + expected_minimum_shared_reuse=1, ), ], ids=lambda case: case.description, @@ -119,7 +126,7 @@ def test_given_perturbed_schedule_when_capturing_compiled_project_then_capture_i _ = test_case.write_project(project_dir) with monkeypatch.context() as waves_patch: use_wave_analysis(waves_patch) - reference: Path = compiled_project_capture( + reference: CompiledProjectRun = compiled_project_capture( project_dir=project_dir, capture_dir=tmp_path / "reference", capsys=capsys, @@ -128,15 +135,28 @@ def test_given_perturbed_schedule_when_capturing_compiled_project_then_capture_i for index, schedule in enumerate(test_case.schedules): with monkeypatch.context() as schedule_patch: perturb_dataflow_schedule(monkeypatch=schedule_patch, case=schedule, seed=index) - actual: Path = compiled_project_capture( + actual: CompiledProjectRun = compiled_project_capture( project_dir=project_dir, capture_dir=tmp_path / f"schedule-{index}", capsys=capsys, monkeypatch=monkeypatch, ) - difference: Divergence | None = first_capture_difference(left=reference, right=actual) + difference: Divergence | None = first_capture_difference( + left=reference.capture, right=actual.capture + ) assert getattr(difference, "location", None) == test_case.expected_difference, schedule + assert ( + actual.exit_code, + actual.shareable_members >= test_case.expected_minimum_shareable_members, + actual.shared_reuse + >= test_case.expected_minimum_shared_reuse * (schedule.workers == 1), + ) == (test_case.expected_exit_code, True, True), (schedule, actual) + assert ( + reference.exit_code, + reference.shareable_members >= test_case.expected_minimum_shareable_members, + reference.shared_reuse >= test_case.expected_minimum_shared_reuse, + ) == (test_case.expected_exit_code, True, True), reference @pytest.mark.parametrize(