diff --git a/Cargo.lock b/Cargo.lock index 03a1b44287..affb7f6b71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -706,6 +706,13 @@ dependencies = [ "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-attachments" +version = "0.139.0" +dependencies = [ + "sqlbuild-sqltext", +] + [[package]] name = "sqlbuild-cache" version = "0.139.0" @@ -756,6 +763,7 @@ dependencies = [ "pyo3", "serde_json", "sqlbuild-analysis", + "sqlbuild-attachments", "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", diff --git a/Cargo.toml b/Cargo.toml index 2f610cd372..69b74e5e53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", "crates/sqlbuild-render", + "crates/sqlbuild-attachments", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -27,6 +28,7 @@ order = [ "sqlbuild-scopes", "sqlbuild-model-config", "sqlbuild-render", + "sqlbuild-attachments", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -75,6 +77,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-attachments = { path = "crates/sqlbuild-attachments" } sqlbuild-cache = { path = "crates/sqlbuild-cache" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } diff --git a/crates/README.md b/crates/README.md index 40d973f153..6cb8ef83ac 100644 --- a/crates/README.md +++ b/crates/README.md @@ -14,6 +14,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | | `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-attachments` | Compile attachments for tests, audits, sources, functions, scenarios and seeds: attached generic audit argument merge, raw and quoted argument rendering, and severity and run scope resolution. Anything it cannot reproduce exactly defers to Python, which also raises every attachment error. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -24,9 +25,9 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-model-config -> sqlbuild-render - -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache - -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-render + -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config + -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-core ``` `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and diff --git a/crates/sqlbuild-attachments/Cargo.toml b/crates/sqlbuild-attachments/Cargo.toml new file mode 100644 index 0000000000..5cd5e398a9 --- /dev/null +++ b/crates/sqlbuild-attachments/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sqlbuild-attachments" +version.workspace = true +description = "Native compile attachments for SQLBuild: tests, audits, sources, functions, scenarios and seeds rendered with Python's exact rules." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs b/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs new file mode 100644 index 0000000000..64996b43bb --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod parameters; +pub(crate) mod policies; diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs b/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs new file mode 100644 index 0000000000..fdc1b0feb6 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs @@ -0,0 +1,139 @@ +//! Python's raw and quoted SQL argument patterns and argument value rendering. + +use crate::audits::models::ArgumentValue; + +/// One `@name` or `@'name'` occurrence, in byte offsets. +pub(crate) struct ParameterMatch<'sql> { + pub(crate) name: &'sql str, + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) quoted: bool, +} + +/// Every parameter in start order, or None where Python's Unicode whitespace could decide. +pub(crate) fn parameter_matches(sql: &str) -> Option>> { + let bytes: &[u8] = sql.as_bytes(); + let mut matches: Vec> = Vec::new(); + let mut quoted_end: usize = 0; + let mut raw_end: usize = 0; + for (index, byte) in bytes.iter().enumerate() { + if *byte != b'@' + || index + .checked_sub(1) + .is_some_and(|previous| bytes[previous] == b'@') + { + continue; + } + if bytes.get(index + 1) == Some(&b'\'') { + if index < quoted_end { + continue; + } + if let Some(end) = identifier_end(bytes, index + 2) + && bytes.get(end) == Some(&b'\'') + { + matches.push(ParameterMatch { + name: &sql[index + 2..end], + start: index, + end: end + 1, + quoted: true, + }); + quoted_end = end + 1; + } + continue; + } + if index < raw_end { + continue; + } + let Some(end) = identifier_end(bytes, index + 1) else { + continue; + }; + if !opens_call(bytes, end)? { + matches.push(ParameterMatch { + name: &sql[index + 1..end], + start: index, + end, + quoted: false, + }); + raw_end = end; + } + } + Some(matches) +} + +/// Python's `render_sql_argument_value` for one value. +pub(crate) fn render_value(value: &ArgumentValue, quoted: bool) -> String { + match value { + ArgumentValue::List(items) => items + .iter() + .map(|item| render_value(item, quoted)) + .collect::>() + .join(", "), + ArgumentValue::Boolean(true) => "TRUE".to_owned(), + ArgumentValue::Boolean(false) => "FALSE".to_owned(), + ArgumentValue::Null => "NULL".to_owned(), + ArgumentValue::Number(text) => text.clone(), + ArgumentValue::Text(text) if quoted => format!("'{}'", text.replace('\'', "''")), + ArgumentValue::Text(text) => text.clone(), + } +} + +/// The end of `[A-Za-z_][A-Za-z0-9_]*` at `start`. +fn identifier_end(bytes: &[u8], start: usize) -> Option { + let first: u8 = *bytes.get(start)?; + if !(first.is_ascii_alphabetic() || first == b'_') { + return None; + } + let mut end: usize = start + 1; + while bytes + .get(end) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + end += 1; + } + Some(end) +} + +/// Python's `(?!\s*\()` after a name; non-ASCII whitespace candidates defer. +fn opens_call(bytes: &[u8], mut index: usize) -> Option { + while let Some(byte) = bytes.get(index) { + if !byte.is_ascii() { + return None; + } + if !matches!( + byte, + b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c | 0x1c..=0x1f + ) { + return Some(*byte == b'('); + } + index += 1; + } + Some(false) +} + +/// Python's `render_parameterized_sql`, or None where Python raises or decides. +pub(crate) fn render_parameterized_sql( + sql: &str, + arguments: &[(String, ArgumentValue)], + reject_unused: bool, +) -> Option { + let matches = parameter_matches(sql)?; + if reject_unused { + for (name, _) in arguments { + if !matches.iter().any(|item| item.name == name.as_str()) { + return None; + } + } + } + let mut rendered: String = String::with_capacity(sql.len()); + let mut previous_end: usize = 0; + for item in &matches { + let (_, value) = arguments + .iter() + .find(|(name, _)| name.as_str() == item.name)?; + rendered.push_str(&sql[previous_end..item.start]); + rendered.push_str(&render_value(value, item.quoted)); + previous_end = item.end; + } + rendered.push_str(&sql[previous_end..]); + Some(rendered) +} diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs new file mode 100644 index 0000000000..b951b8aef0 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs @@ -0,0 +1,33 @@ +//! Python's `resolve_audit_severity` and `resolve_audit_run_scope`. + +use crate::audits::models::PolicySource; + +const SEVERITIES: [&str; 2] = ["warn", "error"]; +const RUN_SCOPES: [&str; 2] = ["final", "delta_and_final"]; + +/// The severity text; unknown values defer so Python raises. +pub(crate) fn resolve_severity( + instance: Option<&str>, + default: Option<&str>, +) -> Option<&'static str> { + match (instance, default) { + (Some(value), _) | (None, Some(value)) => known(&SEVERITIES, value), + (None, None) => Some("error"), + } +} + +/// Where the run scope comes from; unknown values defer so Python raises. +pub(crate) fn resolve_run_scope( + instance: Option<&str>, + default: Option<&str>, +) -> Option { + match (instance, default) { + (Some(value), _) => known(&RUN_SCOPES, value).map(|_| PolicySource::Instance), + (None, Some(value)) => known(&RUN_SCOPES, value).map(|_| PolicySource::Default), + (None, None) => Some(PolicySource::Fallback), + } +} + +fn known(values: &[&'static str], value: &str) -> Option<&'static str> { + values.iter().copied().find(|candidate| *candidate == value) +} diff --git a/crates/sqlbuild-attachments/src/audits/main/mod.rs b/crates/sqlbuild-attachments/src/audits/main/mod.rs new file mode 100644 index 0000000000..24d931eb6d --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/mod.rs @@ -0,0 +1 @@ +pub mod render_attached_audit; diff --git a/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs b/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs new file mode 100644 index 0000000000..8de9074e92 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs @@ -0,0 +1,58 @@ +//! Python's attached generic audit rendering: argument merge, SQL, severity and run scope. + +use crate::audits::_helpers::parameters::render_parameterized_sql; +use crate::audits::_helpers::policies::{resolve_run_scope, resolve_severity}; +use crate::audits::models::{ArgumentValue, AuditAttachment, PolicySource, RenderedAudit}; + +/// Render one attachment as Python does, or None where Python raises or decides. +#[must_use] +pub fn render_attached_audit(attachment: &AuditAttachment) -> Option { + if attachment.measurement { + if attachment.instance_severity.is_some() || !attachment.has_thresholds { + return None; + } + } else if attachment.has_thresholds || attachment.has_minimum_samples { + return None; + } + let arguments: Vec<(String, ArgumentValue)> = merged_arguments(attachment)?; + let sql_body: String = render_parameterized_sql(&attachment.sql_body, &arguments, false)?; + let evidence_sql: Option = match &attachment.evidence_sql { + Some(evidence) => Some(render_parameterized_sql(evidence, &arguments, false)?), + None => None, + }; + let severity: &'static str = if attachment.has_thresholds { + if attachment.threshold_error { + "error" + } else { + "warn" + } + } else { + resolve_severity( + attachment.instance_severity.as_deref(), + attachment.default_severity.as_deref(), + )? + }; + let run_scope_source: PolicySource = resolve_run_scope( + attachment.instance_run_scope.as_deref(), + attachment.default_run_scope.as_deref(), + )?; + Some(RenderedAudit { + sql_body, + evidence_sql, + severity, + run_scope_source, + }) +} + +/// Python's `merge_audit_arguments`; values that differ only as Python objects defer. +fn merged_arguments(attachment: &AuditAttachment) -> Option> { + let mut merged: Vec<(String, ArgumentValue)> = attachment.implicit_arguments.clone(); + for (name, value) in &attachment.explicit_arguments { + match merged.iter_mut().find(|(existing, _)| existing == name) { + Some((_, existing)) if existing == value => {} + Some(_) => return None, + None => merged.push((name.clone(), value.clone())), + } + } + Some(merged) +} diff --git a/crates/sqlbuild-attachments/src/audits/mod.rs b/crates/sqlbuild-attachments/src/audits/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/audits/models.rs b/crates/sqlbuild-attachments/src/audits/models.rs new file mode 100644 index 0000000000..d14d4abb39 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/models.rs @@ -0,0 +1,46 @@ +//! Attached audit inputs and the rendering Python's attachment produces from them. + +/// One audit argument value as Python holds it; numbers keep Python's own `str()` text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArgumentValue { + Null, + Boolean(bool), + Number(String), + Text(String), + List(Vec), +} + +/// Where a resolved run scope came from, so Python keeps its own value object. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PolicySource { + Instance, + Default, + Fallback, +} + +/// One generic audit attachment: definition SQL, arguments and authored policies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuditAttachment { + pub sql_body: String, + pub evidence_sql: Option, + pub implicit_arguments: Vec<(String, ArgumentValue)>, + pub explicit_arguments: Vec<(String, ArgumentValue)>, + pub measurement: bool, + pub has_thresholds: bool, + pub has_minimum_samples: bool, + pub threshold_error: bool, + pub instance_severity: Option, + pub default_severity: Option, + pub instance_run_scope: Option, + pub default_run_scope: Option, +} + +/// The rendered SQL and the policies one attachment resolves to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenderedAudit { + pub sql_body: String, + pub evidence_sql: Option, + /// `warn` or `error`. + pub severity: &'static str, + pub run_scope_source: PolicySource, +} diff --git a/crates/sqlbuild-attachments/src/audits/tests.rs b/crates/sqlbuild-attachments/src/audits/tests.rs new file mode 100644 index 0000000000..8c3f865211 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_render_attached_audit.rs"] +mod render_attached_audit; +#[path = "tests/test_render_parameterized_sql.rs"] +mod render_parameterized_sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/audits/tests/helpers.rs b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs new file mode 100644 index 0000000000..2e18592014 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs @@ -0,0 +1,62 @@ +use crate::audits::_helpers::parameters::render_parameterized_sql; +use crate::audits::main::render_attached_audit::render_attached_audit; +use crate::audits::models::{ArgumentValue, AuditAttachment, RenderedAudit}; + +/// Arguments shared by the parameter rendering cases. +pub(super) fn order_arguments() -> Vec<(String, ArgumentValue)> { + vec![ + ( + "column".to_owned(), + ArgumentValue::Text("status".to_owned()), + ), + ( + "values".to_owned(), + ArgumentValue::List(vec![ + ArgumentValue::Text("it's".to_owned()), + ArgumentValue::Number("2".to_owned()), + ArgumentValue::Null, + ArgumentValue::Boolean(true), + ]), + ), + ] +} + +pub(super) fn rendered_sql(sql: &str, reject_unused: bool) -> Option { + render_parameterized_sql(sql, &order_arguments(), reject_unused) +} + +/// A violations audit over `@column` with the given explicit arguments and policies. +pub(super) fn attachment( + explicit: Vec<(String, ArgumentValue)>, + severity: Option<&str>, + run_scope: Option<&str>, +) -> AuditAttachment { + AuditAttachment { + sql_body: "SELECT * FROM t WHERE @column NOT IN (@'values')".to_owned(), + evidence_sql: Some("SELECT @column FROM t".to_owned()), + implicit_arguments: vec![( + "column".to_owned(), + ArgumentValue::Text("status".to_owned()), + )], + explicit_arguments: explicit, + measurement: false, + has_thresholds: false, + has_minimum_samples: false, + threshold_error: false, + instance_severity: severity.map(str::to_owned), + default_severity: None, + instance_run_scope: run_scope.map(str::to_owned), + default_run_scope: None, + } +} + +pub(super) fn rendering_text(attachment: &AuditAttachment) -> Option { + let rendered: RenderedAudit = render_attached_audit(attachment)?; + Some(format!( + "{} | {} | {} | {:?}", + rendered.sql_body, + rendered.evidence_sql.unwrap_or_default(), + rendered.severity, + rendered.run_scope_source + )) +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs new file mode 100644 index 0000000000..d4adaa5cfe --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs @@ -0,0 +1,58 @@ +use crate::audits::models::ArgumentValue; +use crate::audits::tests::helpers::{attachment, rendering_text}; +use crate::audits::tests::test_types::AttachedAuditTestCase; + +#[test] +fn given_attached_audits_when_rendering_then_python_rendering_is_returned() { + let values: (String, ArgumentValue) = ( + "values".to_owned(), + ArgumentValue::List(vec![ArgumentValue::Number("1".to_owned())]), + ); + let test_cases = [ + AttachedAuditTestCase { + description: "explicit arguments merge after implicit ones with default policies", + attachment: attachment(vec![values.clone()], None, None), + expected_rendering: Some( + "SELECT * FROM t WHERE status NOT IN (1) | SELECT status FROM t | error | Fallback", + ), + }, + AttachedAuditTestCase { + description: "instance severity and run scope", + attachment: attachment(vec![values.clone()], Some("warn"), Some("final")), + expected_rendering: Some( + "SELECT * FROM t WHERE status NOT IN (1) | SELECT status FROM t | warn | Instance", + ), + }, + AttachedAuditTestCase { + description: "an explicit argument overriding an implicit one defers", + attachment: attachment( + vec![( + "column".to_owned(), + ArgumentValue::Text("amount".to_owned()), + )], + None, + None, + ), + expected_rendering: None, + }, + AttachedAuditTestCase { + description: "an unknown severity defers so Python raises", + attachment: attachment(vec![values.clone()], Some("fatal"), None), + expected_rendering: None, + }, + AttachedAuditTestCase { + description: "an unknown run scope defers so Python raises", + attachment: attachment(vec![values], None, Some("delta")), + expected_rendering: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + rendering_text(&test_case.attachment).as_deref(), + test_case.expected_rendering, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs new file mode 100644 index 0000000000..ee0822a9e4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs @@ -0,0 +1,49 @@ +use crate::audits::tests::helpers::rendered_sql; +use crate::audits::tests::test_types::ParameterizedSqlTestCase; + +#[test] +fn given_parameterized_sql_when_rendering_then_python_text_is_returned() { + let test_cases = [ + ParameterizedSqlTestCase { + description: "raw and quoted parameters: lists joined, only text quoted, quotes doubled", + sql: "WHERE @column IN (@'values') AND x = @values", + reject_unused: false, + expected_sql: Some( + "WHERE status IN ('it''s', 2, NULL, TRUE) AND x = it's, 2, NULL, TRUE", + ), + }, + ParameterizedSqlTestCase { + description: "macro calls, doubled @ and runtime placeholders are left alone", + sql: "@column (x) @@column @@@column @column(y)", + reject_unused: false, + expected_sql: Some("@column (x) @@column @@@column @column(y)"), + }, + ParameterizedSqlTestCase { + description: "a missing argument defers so Python raises", + sql: "WHERE @missing", + reject_unused: false, + expected_sql: None, + }, + ParameterizedSqlTestCase { + description: "an unused argument defers when unused arguments are rejected", + sql: "WHERE @column", + reject_unused: true, + expected_sql: None, + }, + ParameterizedSqlTestCase { + description: "a non-ASCII character after a name defers to Python's whitespace", + sql: "WHERE @column\u{a0}(", + reject_unused: false, + expected_sql: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + rendered_sql(test_case.sql, test_case.reject_unused).as_deref(), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_types.rs b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs new file mode 100644 index 0000000000..fad31e8d91 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::audits::models::AuditAttachment; + +pub(super) struct ParameterizedSqlTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) reject_unused: bool, + pub(super) expected_sql: Option<&'static str>, +} + +pub(super) struct AttachedAuditTestCase { + pub(super) description: &'static str, + pub(super) attachment: AuditAttachment, + /// `(sql body, evidence, severity, run scope source)` spelled as text, or None to defer. + pub(super) expected_rendering: Option<&'static str>, +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs new file mode 100644 index 0000000000..49f9fa71a8 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod scan; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs new file mode 100644 index 0000000000..a925840de3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs @@ -0,0 +1,76 @@ +//! Python's cursor intrinsic scan over quotes, comments and identifier boundaries. + +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; + +const INTRINSIC_NAMES: [&[u8]; 2] = [b"__cursor_start", b"__cursor_end"]; +/// Only dollar quotes: Python's own quote scan does not double backticks, so quotes run here. +const DOLLAR_POLICY: QuotePolicy = QuotePolicy { + backtick_identifiers: false, + single_quote_backslash_escapes: false, + double_quote_backslash_escapes: false, + dollar_quotes: true, +}; + +/// The next scan offset after `index`, or `Err` where Python finds an intrinsic or raises. +pub(crate) fn step(bytes: &[u8], index: usize) -> Result { + if let Some(end) = comment_end(bytes, index).map_err(|_| ())? { + return Ok(end); + } + match bytes[index] { + quote @ (b'\'' | b'"' | b'`') => return quoted_end(bytes, index, quote), + b'$' => { + return Ok(non_code_end(bytes, index, DOLLAR_POLICY) + .map_err(|_| ())? + .unwrap_or(index + 1)); + } + _ => {} + } + for name in INTRINSIC_NAMES { + if bytes[index..].starts_with(name) { + let end: usize = index + name.len(); + let before: Option = index.checked_sub(1).map(|previous| bytes[previous]); + if !continues_identifier(before)? && !continues_identifier(bytes.get(end).copied())? { + return Err(()); + } + return Ok(end); + } + } + Ok(index + 1) +} + +/// Python's quoted text end: `'` and `"` double to escape, backticks do not. +fn quoted_end(bytes: &[u8], start: usize, quote: u8) -> Result { + let mut index: usize = start + 1; + loop { + index += bytes[index..] + .iter() + .position(|byte| *byte == quote) + .ok_or(())?; + if quote != b'`' && bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + 1); + } +} + +/// Python's `isalnum() or _`; non-ASCII neighbours defer to Python's Unicode rules. +fn continues_identifier(byte: Option) -> Result { + match byte { + None => Ok(false), + Some(byte) if !byte.is_ascii() => Err(()), + Some(byte) => Ok(byte.is_ascii_alphanumeric() || byte == b'_'), + } +} + +/// Python's `any(name in sql for name in _INTRINSIC_NAMES)`. +pub(crate) fn contains_intrinsic_name(bytes: &[u8]) -> bool { + for name in INTRINSIC_NAMES { + if bytes.windows(name.len()).any(|window| window == name) { + return true; + } + } + false +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs new file mode 100644 index 0000000000..b375e99da5 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs @@ -0,0 +1,27 @@ +//! Python's `reject_cursor_intrinsics` for SQL that it accepts. + +use crate::cursor_intrinsics::_helpers::scan::{contains_intrinsic_name, step}; +use crate::cursor_intrinsics::models::IntrinsicCheck; + +/// `Free` where Python accepts `sql`; anything Python could reject or judge by Unicode defers. +#[must_use] +pub fn intrinsic_free(sql: &str, reserved_markers: &[String]) -> IntrinsicCheck { + if reserved_markers + .iter() + .any(|marker| sql.contains(marker.as_str())) + { + return IntrinsicCheck::Deferred; + } + let bytes: &[u8] = sql.as_bytes(); + if !contains_intrinsic_name(bytes) { + return IntrinsicCheck::Free; + } + let mut index: usize = 0; + while index < bytes.len() { + let Ok(next) = step(bytes, index) else { + return IntrinsicCheck::Deferred; + }; + index = next; + } + IntrinsicCheck::Free +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs new file mode 100644 index 0000000000..78a625860d --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs @@ -0,0 +1 @@ +pub mod intrinsic_free; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs new file mode 100644 index 0000000000..496c2f0225 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs @@ -0,0 +1,8 @@ +//! Outcome of the native cursor intrinsic check. + +/// Whether SQL is free of cursor intrinsics, or Python must decide (and usually raise). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IntrinsicCheck { + Free, + Deferred, +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests.rs new file mode 100644 index 0000000000..e5bc1da50a --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_intrinsic_free.rs"] +mod intrinsic_free; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs new file mode 100644 index 0000000000..5d398e4442 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs @@ -0,0 +1,49 @@ +use crate::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use crate::cursor_intrinsics::models::IntrinsicCheck; +use crate::cursor_intrinsics::tests::test_types::IntrinsicCheckTestCase; + +#[test] +fn given_sql_when_checking_cursor_intrinsics_then_python_acceptance_is_returned() { + let test_cases = [ + IntrinsicCheckTestCase { + description: "SQL without intrinsic names is free", + sql: "SELECT 'unclosed", + expected_check: IntrinsicCheck::Free, + }, + IntrinsicCheckTestCase { + description: "names inside quotes, comments, dollar quotes and identifiers are free", + sql: "SELECT '__cursor_start()', `a``__cursor_end`, x__cursor_end -- __cursor_end\n\ + /* __cursor_start */ $tag$__cursor_end()$tag$, __cursor_start_at", + expected_check: IntrinsicCheck::Free, + }, + IntrinsicCheckTestCase { + description: "an intrinsic call defers so Python raises", + sql: "WHERE ts >= __cursor_start()", + expected_check: IntrinsicCheck::Deferred, + }, + IntrinsicCheckTestCase { + description: "an unclosed quote after an intrinsic name defers", + sql: "SELECT x__cursor_start, 'open", + expected_check: IntrinsicCheck::Deferred, + }, + IntrinsicCheckTestCase { + description: "a non-ASCII neighbour defers to Python's Unicode identifier rules", + sql: "SELECT é__cursor_end", + expected_check: IntrinsicCheck::Deferred, + }, + IntrinsicCheckTestCase { + description: "a reserved marker defers", + sql: "SELECT __reserved_marker__", + expected_check: IntrinsicCheck::Deferred, + }, + ]; + + for test_case in test_cases { + assert_eq!( + intrinsic_free(test_case.sql, &["__reserved_marker__".to_owned()]), + test_case.expected_check, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_types.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_types.rs new file mode 100644 index 0000000000..0a0d151f4e --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::cursor_intrinsics::models::IntrinsicCheck; + +pub(super) struct IntrinsicCheckTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_check: IntrinsicCheck, +} diff --git a/crates/sqlbuild-attachments/src/lib.rs b/crates/sqlbuild-attachments/src/lib.rs new file mode 100644 index 0000000000..61217c24d9 --- /dev/null +++ b/crates/sqlbuild-attachments/src/lib.rs @@ -0,0 +1,7 @@ +//! Native compile attachments: what tests, audits, sources, functions, scenarios and seeds attach. + +#![forbid(unsafe_code)] + +pub mod audits; +pub mod cursor_intrinsics; +pub mod seeds; diff --git a/crates/sqlbuild-attachments/src/seeds/main/mod.rs b/crates/sqlbuild-attachments/src/seeds/main/mod.rs new file mode 100644 index 0000000000..97cfdfe052 --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/main/mod.rs @@ -0,0 +1 @@ +pub mod pair_seed_declarations; diff --git a/crates/sqlbuild-attachments/src/seeds/main/pair_seed_declarations.rs b/crates/sqlbuild-attachments/src/seeds/main/pair_seed_declarations.rs new file mode 100644 index 0000000000..c9cce065ac --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/main/pair_seed_declarations.rs @@ -0,0 +1,20 @@ +//! Python's `build_seed_inputs` pairing of seed declarations with seed CSV files. + +use std::collections::HashMap; + +/// Each declaration's file index by stem (a later stem wins), or the first unmatched declaration. +pub fn pair_seed_declarations( + declarations: &[String], + stems: &[String], +) -> Result, usize> { + let files: HashMap<&str, usize> = stems + .iter() + .enumerate() + .map(|(index, stem)| (stem.as_str(), index)) + .collect(); + let mut pairs: Vec = Vec::with_capacity(declarations.len()); + for (index, name) in declarations.iter().enumerate() { + pairs.push(*files.get(name.as_str()).ok_or(index)?); + } + Ok(pairs) +} diff --git a/crates/sqlbuild-attachments/src/seeds/mod.rs b/crates/sqlbuild-attachments/src/seeds/mod.rs new file mode 100644 index 0000000000..e29b572755 --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/mod.rs @@ -0,0 +1,4 @@ +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/seeds/tests.rs b/crates/sqlbuild-attachments/src/seeds/tests.rs new file mode 100644 index 0000000000..debcba33a3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_pair_seed_declarations.rs"] +mod pair_seed_declarations; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/seeds/tests/test_pair_seed_declarations.rs b/crates/sqlbuild-attachments/src/seeds/tests/test_pair_seed_declarations.rs new file mode 100644 index 0000000000..b2e1838f4b --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/tests/test_pair_seed_declarations.rs @@ -0,0 +1,45 @@ +use crate::seeds::main::pair_seed_declarations::pair_seed_declarations; +use crate::seeds::tests::test_types::SeedPairingTestCase; + +#[test] +fn given_seed_declarations_when_pairing_then_python_pairs_are_returned() { + let test_cases = [ + SeedPairingTestCase { + description: "declarations pair with files by stem in declaration order", + declarations: &["products", "orders"], + stems: &["orders", "products", "customers"], + expected_pairs: Ok(vec![1, 0]), + }, + SeedPairingTestCase { + description: "a later file with the same stem wins", + declarations: &["orders"], + stems: &["orders", "orders"], + expected_pairs: Ok(vec![1]), + }, + SeedPairingTestCase { + description: "the first declaration without a file is reported", + declarations: &["orders", "missing", "absent"], + stems: &["orders"], + expected_pairs: Err(1), + }, + ]; + + for test_case in test_cases { + let declarations: Vec = test_case + .declarations + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let stems: Vec = test_case + .stems + .iter() + .map(|stem| (*stem).to_owned()) + .collect(); + assert_eq!( + pair_seed_declarations(&declarations, &stems), + test_case.expected_pairs, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/seeds/tests/test_types.rs b/crates/sqlbuild-attachments/src/seeds/tests/test_types.rs new file mode 100644 index 0000000000..408e85d833 --- /dev/null +++ b/crates/sqlbuild-attachments/src/seeds/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct SeedPairingTestCase { + pub(super) description: &'static str, + pub(super) declarations: &'static [&'static str], + pub(super) stems: &'static [&'static str], + pub(super) expected_pairs: Result, usize>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 840752e5fa..7f29826022 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -21,6 +21,7 @@ mimalloc.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs new file mode 100644 index 0000000000..196d0b0f10 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs @@ -0,0 +1,57 @@ +//! Python audit and hook argument values read as plain argument values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString}; +use sqlbuild_attachments::audits::models::ArgumentValue; + +/// Read a `dict[str, object]` in insertion order, or None for values Python must judge. +pub(crate) fn argument_pairs( + arguments: &Bound<'_, PyDict>, +) -> PyResult>> { + let mut pairs: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.len()); + for (key, value) in arguments.iter() { + if !key.is_exact_instance_of::() { + return Ok(None); + } + let Some(argument) = argument_value(&value)? else { + return Ok(None); + }; + pairs.push((key.extract()?, argument)); + } + Ok(Some(pairs)) +} + +/// One value: None, bool, int and finite float (by `str()`), str, or a list of those. +fn argument_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(ArgumentValue::Null)); + } + if value.is_instance_of::() { + return Ok(Some(ArgumentValue::Boolean(value.extract()?))); + } + if value.is_instance_of::() { + return Ok(Some(ArgumentValue::Number(value.str()?.extract()?))); + } + if value.is_instance_of::() { + let number: f64 = value.extract()?; + if !number.is_finite() { + return Ok(None); + } + return Ok(Some(ArgumentValue::Number(value.str()?.extract()?))); + } + if value.is_instance_of::() { + return Ok(Some(ArgumentValue::Text(value.extract()?))); + } + if !value.is_exact_instance_of::() { + return Ok(None); + } + let list: &Bound<'_, PyList> = value.downcast_exact::()?; + let mut items: Vec = Vec::with_capacity(list.len()); + for item in list.iter() { + let Some(argument) = argument_value(&item)? else { + return Ok(None); + }; + items.push(argument); + } + Ok(Some(ArgumentValue::List(items))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs new file mode 100644 index 0000000000..d16db09cbf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs @@ -0,0 +1,77 @@ +//! Attached generic audit rendering for the preview compile attachments. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::audits::main::render_attached_audit::render_attached_audit; +use sqlbuild_attachments::audits::models::{AuditAttachment, PolicySource, RenderedAudit}; + +use crate::bindings::_helpers::attachments::argument_values::argument_pairs; +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// `(sql body, evidence sql, severity, run scope source)`. +type RenderedRow = (String, Option, &'static str, &'static str); + +/// Authored policies of one attachment: mode and threshold flags, severities and run scopes. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct AuditPolicies { + measurement: bool, + has_thresholds: bool, + has_minimum_samples: bool, + threshold_error: bool, + instance_severity: Option, + default_severity: Option, + instance_run_scope: Option, + default_run_scope: Option, +} + +/// Render `(sql body, evidence)` with `(implicit, explicit)` arguments, or `None` for Python. +#[pyfunction] +fn render_attached_generic_audit( + sql: (String, Option), + arguments: (Bound<'_, PyDict>, Bound<'_, PyDict>), + policies: AuditPolicies, +) -> PyResult> { + compiler_guard(|| { + let (Some(implicit), Some(explicit)) = + (argument_pairs(&arguments.0)?, argument_pairs(&arguments.1)?) + else { + return Ok(None); + }; + let attachment: AuditAttachment = AuditAttachment { + sql_body: sql.0, + evidence_sql: sql.1, + implicit_arguments: implicit, + explicit_arguments: explicit, + measurement: policies.measurement, + has_thresholds: policies.has_thresholds, + has_minimum_samples: policies.has_minimum_samples, + threshold_error: policies.threshold_error, + instance_severity: policies.instance_severity, + default_severity: policies.default_severity, + instance_run_scope: policies.instance_run_scope, + default_run_scope: policies.default_run_scope, + }; + Ok(render_attached_audit(&attachment).map(rendered_row)) + }) +} + +fn rendered_row(rendered: RenderedAudit) -> RenderedRow { + let source: &'static str = match rendered.run_scope_source { + PolicySource::Instance => "instance", + PolicySource::Default => "default", + PolicySource::Fallback => "fallback", + }; + ( + rendered.sql_body, + rendered.evidence_sql, + rendered.severity, + source, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_attached_generic_audit, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs new file mode 100644 index 0000000000..dd7199762c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -0,0 +1,19 @@ +//! Cursor intrinsic rejection for SQL outside cursor-based incremental models. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// Return whether Python accepts `sql` as free of cursor intrinsics; `False` defers to Python. +#[pyfunction] +fn sql_free_of_cursor_intrinsics(sql: &str, reserved_markers: Vec) -> PyResult { + compiler_guard(|| Ok(intrinsic_free(sql, &reserved_markers) == IntrinsicCheck::Free)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_free_of_cursor_intrinsics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs new file mode 100644 index 0000000000..94501a40c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod argument_values; +pub(crate) mod audit_rendering; +pub(crate) mod cursor_intrinsics; +pub(crate) mod registration; +pub(crate) mod seed_pairing; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs new file mode 100644 index 0000000000..840225b1d2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs @@ -0,0 +1,12 @@ +//! Register the native compile attachment functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::attachments::{audit_rendering, cursor_intrinsics, seed_pairing}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + audit_rendering::register(module)?; + seed_pairing::register(module)?; + cursor_intrinsics::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs new file mode 100644 index 0000000000..f9dcbb3fe6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs @@ -0,0 +1,26 @@ +//! Seed declaration pairing for the preview compile attachments. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// Pair declarations with seed file stems: `(file indices, None)` or `(None, missing index)`. +#[pyfunction] +fn pair_seed_files( + declarations: Vec, + stems: Vec, +) -> PyResult<(Option>, Option)> { + compiler_guard(|| { + Ok(match pair_seed_declarations(&declarations, &stems) { + Ok(pairs) => (Some(pairs), None), + Err(missing) => (None, Some(missing)), + }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(pair_seed_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 7863a42fdd..4ad9b8950e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod analysis; +pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; pub(crate) mod discovery; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 64ff38a3ba..cde73bc8c0 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,7 +4,7 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, + analysis, attachments, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, }; use crate::bindings::models; @@ -31,6 +31,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { model_config::registration::register(module)?; render::macro_calls::register(module)?; cache::native_store::register(module)?; + attachments::registration::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/fensu.toml b/fensu.toml index 7520b517e9..8f64aec491 100644 --- a/fensu.toml +++ b/fensu.toml @@ -305,6 +305,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ "sqlbuild-analysis", + "sqlbuild-attachments", "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 772a64666b..8b048215c5 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -254,6 +254,14 @@ def _declaration_kinds( def _interpolation_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() + for source in records(capture.get("source_inputs")): + source_file: dict[str, object] = as_json_object(source.get("source_file")) or {} + if _TEMPLATE_OPEN in json.dumps(source_file.get("source_entries")): + kinds.add("source_template") + for function in records(capture.get("sql_function_inputs")): + function_file: dict[str, object] = as_json_object(function.get("function_file")) or {} + if _TEMPLATE_OPEN in json.dumps(function_file.get("header_values")): + kinds.add("function_header_template") for model in records(capture.get("model_inputs")): model_file: dict[str, object] = as_json_object(model.get("model_file")) or {} authored: str = str(model_file.get("query_sql", "")) @@ -429,6 +437,14 @@ def _model_config_kinds(capture: dict[str, object]) -> set[str]: target: dict[str, object] = as_json_object(capture.get("effective_target")) or {} if any(_TEMPLATE_OPEN in str(target.get(key)) for key in _TARGET_NAMESPACE_KEYS): kinds.add("target_namespace_template") + for source in records(capture.get("source_inputs")): + source_file: dict[str, object] = as_json_object(source.get("source_file")) or {} + if _TEMPLATE_OPEN in json.dumps(source_file.get("source_entries")): + kinds.add("source_template") + for function in records(capture.get("sql_function_inputs")): + function_file: dict[str, object] = as_json_object(function.get("function_file")) or {} + if _TEMPLATE_OPEN in json.dumps(function_file.get("header_values")): + kinds.add("function_header_template") for model in records(capture.get("model_inputs")): model_file: dict[str, object] = as_json_object(model.get("model_file")) or {} header: dict[str, object] = as_json_object(model_file.get("header_values")) or {} diff --git a/scripts/compiler_differential/classes/render_features.py b/scripts/compiler_differential/classes/render_features.py index 8f574e4ca0..8ab0026dfb 100644 --- a/scripts/compiler_differential/classes/render_features.py +++ b/scripts/compiler_differential/classes/render_features.py @@ -449,3 +449,29 @@ def _model_config(self) -> None: ), body=f'SELECT id, amount, status\nFROM __ref("{base.name}")\n', ) + self._files["sources/configured.yml"] = ( + "sources:\n" + " - name: configured_events\n" + f' description: "Configured feed ${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, ' + "'events')}\"\n" + ' expression: "(SELECT 1 AS id, CAST(2.5 AS DOUBLE) AS amount)"\n' + " meta:\n" + f" owner: \"${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, 'orders team')}}\"\n" + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + f" description: \"Key ${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, 'id')}}\"\n" + " audits:\n" + " - accepted_values:\n" + f" values: [\"${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, '1')}}\"]\n" + " - name: amount\n" + " type: DOUBLE\n" + ) + self._files["functions/sql/configured_label.sql"] = ( + "FUNCTION (\n" + ' description "Label a configured status.",\n' + f" schema \"${{coalesce(ENV:{GENERATOR_MISSING_ENV_VAR}, 'udfs')}}\",\n" + " arguments (raw_status STRING),\n" + " returns STRING,\n);\n\n" + "UPPER(raw_status)\n" + ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index de07f6da70..7c8278ecf1 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -461,6 +461,8 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "model_header_template", "target_namespace_template", "model_column_audit_options", + "source_template", + "function_header_template", ) RENDER_INDIRECT_KINDS: dict[str, str] = { kind: "the used macro's own function source reads it; the capture does not record context reads" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 83909e19fc..47e6dab755 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -175,6 +175,15 @@ class SqlReferenceScanner: def scope_expected_model_names( sqls: list[str], syntax: dict[str, object] ) -> list[list[str] | None]: ... +def sql_free_of_cursor_intrinsics(sql: str, reserved_markers: list[str]) -> bool: ... +def pair_seed_files( + declarations: list[str], stems: list[str] +) -> tuple[list[int], None] | tuple[None, int]: ... +def render_attached_generic_audit( + sql: tuple[str, str | None], + arguments: tuple[dict[str, object], dict[str, object]], + policies: dict[str, object], +) -> tuple[str, str | None, str, str] | None: ... def scan_sql_declaration_references( sqls: list[str], ) -> list[list[tuple[int, str, str | None, int, int]] | None]: ... diff --git a/src/sqlbuild/compiler/attachments/__init__.py b/src/sqlbuild/compiler/attachments/__init__.py new file mode 100644 index 0000000000..cc22a420ba --- /dev/null +++ b/src/sqlbuild/compiler/attachments/__init__.py @@ -0,0 +1 @@ +"""Native compile attachments for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/attachments/constants.py b/src/sqlbuild/compiler/attachments/constants.py new file mode 100644 index 0000000000..053a93caab --- /dev/null +++ b/src/sqlbuild/compiler/attachments/constants.py @@ -0,0 +1,4 @@ +"""Values the native compile attachments return.""" + +INSTANCE_POLICY: str = "instance" +DEFAULT_POLICY: str = "default" diff --git a/src/sqlbuild/compiler/attachments/main/__init__.py b/src/sqlbuild/compiler/attachments/main/__init__.py new file mode 100644 index 0000000000..35e81c6936 --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/__init__.py @@ -0,0 +1 @@ +"""Native compile attachment entry points.""" diff --git a/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py b/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py new file mode 100644 index 0000000000..822fdac33b --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py @@ -0,0 +1,13 @@ +"""Pair seed declarations with seed CSV files natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native + + +def pair_native_seed_files( + *, declaration_names: list[str], file_stems: list[str] +) -> tuple[list[int], None] | tuple[None, int]: + """Return each declaration's seed file index, or the index of the first one without a file.""" + + return _native.pair_seed_files(declaration_names, file_stems) diff --git a/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py b/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py new file mode 100644 index 0000000000..e8b84975fd --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py @@ -0,0 +1,34 @@ +"""Render one attached generic audit natively for the preview compiler engine.""" + +from __future__ import annotations + +from dataclasses import asdict + +import sqlbuild._native as _native +from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit + + +def render_native_attached_audit( + *, + sql_body: str, + evidence_sql: str | None, + implicit_arguments: dict[str, object], + explicit_arguments: dict[str, object], + policies: NativeAuditPolicies, +) -> NativeRenderedAudit | None: + """Return the rendering, or None where Python must render the audit and raise its errors.""" + + if not all( + value is None or isinstance(value, str) + for value in ( + policies.instance_severity, + policies.default_severity, + policies.instance_run_scope, + policies.default_run_scope, + ) + ) or not isinstance(explicit_arguments, dict): + return None + rendered: tuple[str, str | None, str, str] | None = _native.render_attached_generic_audit( + (sql_body, evidence_sql), (implicit_arguments, explicit_arguments), asdict(policies) + ) + return None if rendered is None else NativeRenderedAudit(*rendered) diff --git a/src/sqlbuild/compiler/attachments/models.py b/src/sqlbuild/compiler/attachments/models.py new file mode 100644 index 0000000000..5df6644dca --- /dev/null +++ b/src/sqlbuild/compiler/attachments/models.py @@ -0,0 +1,29 @@ +"""Plain values exchanged with the native compile attachments.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class NativeAuditPolicies: + """One attachment's evaluation mode, threshold facts, severities and run scopes.""" + + measurement: bool + has_thresholds: bool + has_minimum_samples: bool + threshold_error: bool + instance_severity: str | None + default_severity: str | None + instance_run_scope: str | None + default_run_scope: str | None + + +@dataclass(frozen=True, slots=True) +class NativeRenderedAudit: + """Rendered audit SQL, the severity value and where the run scope comes from.""" + + sql_body: str + evidence_sql: str | None + severity: str + run_scope_source: str diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 05db949a19..0ace8f95ed 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -7,6 +7,11 @@ from functools import partial from pathlib import Path +from sqlbuild.compiler.attachments.constants import DEFAULT_POLICY, INSTANCE_POLICY +from sqlbuild.compiler.attachments.main._render_native_attached_audit import ( + render_native_attached_audit, +) +from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit from sqlbuild.compiler.auditing.constants import ( MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY, MEASUREMENT_THRESHOLDS_HEADER_KEY, @@ -17,7 +22,7 @@ parse_minimum_samples, ) from sqlbuild.compiler.auditing.models import MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditRunScope, AuditSeverity from sqlbuild.compiler.compile._helpers.attachment.references import ( build_known_ref_names, build_known_seed_names, @@ -73,6 +78,8 @@ DiscoveredAuditFile, DiscoveredProjectInputs, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, @@ -668,26 +675,27 @@ def build_attached_audit_input( f"{owner_file} audit '{audit_instance.definition_name}': thresholds and " "minimum_samples are only valid for measurement audits" ) - merged_arguments: dict[str, object] = merge_audit_arguments( - owner_file=owner_file, - definition_name=audit_instance.definition_name, - implicit_arguments=implicit_arguments, - explicit_arguments=audit_instance.arguments, - ) - rendered_sql_body: str = render_generic_audit_sql( - sql=definition[1].sql_body, - arguments=merged_arguments, - owner_file=owner_file, - definition_name=audit_instance.definition_name, + native_rendering: NativeRenderedAudit | None = ( + render_native_attached_audit( + sql_body=definition[1].sql_body, + evidence_sql=definition[1].evidence_sql, + implicit_arguments=implicit_arguments, + explicit_arguments=audit_instance.arguments, + policies=_native_audit_policies(audit_instance=audit_instance, context=context), + ) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else None ) - rendered_evidence_sql: str | None = None - if definition[1].evidence_sql is not None: - rendered_evidence_sql = render_generic_audit_sql( - sql=definition[1].evidence_sql, - arguments=merged_arguments, + rendered_sql_body, rendered_evidence_sql = ( + (native_rendering.sql_body, native_rendering.evidence_sql) + if native_rendering is not None + else _rendered_audit_sql( + audit_instance=audit_instance, + definition=definition[1], owner_file=owner_file, - definition_name=audit_instance.definition_name, + implicit_arguments=implicit_arguments, ) + ) scoped_declarations: DeclarationExpansionContext = _scoped_audit_declarations( context=context, file_path=definition[0].file_path, @@ -746,18 +754,16 @@ def build_attached_audit_input( known_source_names=context.known_source_names, ) audit_label: str = f"{owner_file} audit '{audit_instance.definition_name}'" - resolved_severity: AuditSeverity = ( - measurement_policy_severity(audit_instance.thresholds) - if audit_instance.thresholds is not None - else resolve_audit_severity( - instance_severity=audit_instance.severity, - default_severity=context.default_audit_severity, - audit_label=audit_label, + resolved_severity: AuditSeverity + resolved_run_scope: str + resolved_severity, resolved_run_scope = ( + _native_audit_policy_values( + rendering=native_rendering, audit_instance=audit_instance, context=context + ) + if native_rendering is not None + else _audit_policy_values( + audit_instance=audit_instance, context=context, audit_label=audit_label ) - ) - resolved_run_scope: str = resolve_audit_run_scope( - instance_run_scope=audit_instance.run_scope, - default_run_scope=context.default_audit_run_scope, ) validate_model_attached_audit_references( references=references, @@ -795,6 +801,93 @@ def build_attached_audit_input( ) +def _rendered_audit_sql( + *, + audit_instance: SchemaAuditInstance, + definition: DiscoveredAuditBlock, + owner_file: Path, + implicit_arguments: dict[str, object], +) -> tuple[str, str | None]: + merged_arguments: dict[str, object] = merge_audit_arguments( + owner_file=owner_file, + definition_name=audit_instance.definition_name, + implicit_arguments=implicit_arguments, + explicit_arguments=audit_instance.arguments, + ) + rendered_sql_body: str = render_generic_audit_sql( + sql=definition.sql_body, + arguments=merged_arguments, + owner_file=owner_file, + definition_name=audit_instance.definition_name, + ) + rendered_evidence_sql: str | None = ( + None + if definition.evidence_sql is None + else render_generic_audit_sql( + sql=definition.evidence_sql, + arguments=merged_arguments, + owner_file=owner_file, + definition_name=audit_instance.definition_name, + ) + ) + return rendered_sql_body, rendered_evidence_sql + + +def _native_audit_policies( + *, audit_instance: SchemaAuditInstance, context: _AuditAttachmentContext +) -> NativeAuditPolicies: + thresholds: MeasurementThresholds | None = audit_instance.thresholds + return NativeAuditPolicies( + measurement=context.generic_audit_definitions[audit_instance.definition_name][ + 1 + ].evaluation_mode + == AuditEvaluationMode.MEASUREMENT, + has_thresholds=thresholds is not None, + has_minimum_samples=audit_instance.minimum_samples is not None, + threshold_error=thresholds is not None and thresholds.error is not None, + instance_severity=audit_instance.severity, + default_severity=context.default_audit_severity, + instance_run_scope=audit_instance.run_scope, + default_run_scope=context.default_audit_run_scope, + ) + + +def _audit_policy_values( + *, audit_instance: SchemaAuditInstance, context: _AuditAttachmentContext, audit_label: str +) -> tuple[AuditSeverity, str]: + severity: AuditSeverity = ( + measurement_policy_severity(audit_instance.thresholds) + if audit_instance.thresholds is not None + else resolve_audit_severity( + instance_severity=audit_instance.severity, + default_severity=context.default_audit_severity, + audit_label=audit_label, + ) + ) + return severity, resolve_audit_run_scope( + instance_run_scope=audit_instance.run_scope, + default_run_scope=context.default_audit_run_scope, + ) + + +def _native_audit_policy_values( + *, + rendering: NativeRenderedAudit, + audit_instance: SchemaAuditInstance, + context: _AuditAttachmentContext, +) -> tuple[AuditSeverity, str]: + run_scope: str | None = ( + audit_instance.run_scope + if rendering.run_scope_source == INSTANCE_POLICY + else context.default_audit_run_scope + if rendering.run_scope_source == DEFAULT_POLICY + else None + ) + return AuditSeverity(rendering.severity), ( + AuditRunScope.DELTA_AND_FINAL if run_scope is None else run_scope + ) + + def index_generic_audit_definitions( audit_files: tuple[DiscoveredAuditFile, ...], ) -> dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]]: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c63e030c05..ada62426b1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild.compiler.attachments.main._pair_native_seed_files import pair_native_seed_files from sqlbuild.compiler.compile._helpers.analysis.validation import validate_sql_syntax from sqlbuild.compiler.compile._helpers.attachment.model_config import ( _model_sql_validation_gate, @@ -1164,6 +1165,10 @@ def build_seed_inputs(discovered_inputs: DiscoveredProjectInputs) -> tuple[Compi for seed_entry in schema_file.seed_entries: seed_declarations.append((seed_entry, schema_file)) + if native_stage_enabled(NativeStage.ATTACHMENTS): + return _native_seed_inputs( + seed_declarations=seed_declarations, seed_files=discovered_inputs.seed_files + ) seed_files_by_name: dict[str, DiscoveredSeedFile] = { seed_file.file_path.stem: seed_file for seed_file in discovered_inputs.seed_files } @@ -1190,6 +1195,33 @@ def build_seed_inputs(discovered_inputs: DiscoveredProjectInputs) -> tuple[Compi return tuple(seed_inputs) +def _native_seed_inputs( + *, + seed_declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]], + seed_files: tuple[DiscoveredSeedFile, ...], +) -> tuple[CompileSeedInput, ...]: + pairs: list[int] | None + missing: int | None + pairs, missing = pair_native_seed_files( + declaration_names=[seed_entry.name for seed_entry, _ in seed_declarations], + file_stems=[seed_file.file_path.stem for seed_file in seed_files], + ) + if pairs is None: + seed_entry, seed_schema_file = seed_declarations[cast(int, missing)] + raise CompileInputError( + f"Seed declaration '{seed_entry.name}' in {seed_schema_file.relative_path} " + "has no matching CSV file under seeds/" + ) + return tuple( + CompileSeedInput( + seed_file=seed_files[file_index], + schema_entry=seed_entry, + schema_file=seed_schema_file, + ) + for (seed_entry, seed_schema_file), file_index in zip(seed_declarations, pairs, strict=True) + ) + + def build_effective_connection( *, project_config: ProjectConfig, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 08c14d5418..331e1e78bf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -25,12 +25,10 @@ merge_call_site_references, ) from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references +from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_template_data, -) from sqlbuild.compiler.compile.constants import ( PRESERVE_TARGET_VALUE, TABLE_FUNCTION_RETURN_KEYS, @@ -56,6 +54,8 @@ DiscoveredPythonFunctionFile, DiscoveredSqlFunctionFile, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot @@ -644,7 +644,7 @@ def _expand_function_header_value( *, raw_value: str, effective_vars: dict[str, object], context_label: str ) -> str: return str( - expand_template_data( + expand_config_templates( value=raw_value, variables=effective_vars, context_values={}, @@ -652,6 +652,7 @@ def _expand_function_header_value( allow_context=False, preserve_context_tokens=True, preserve_unknown_context=False, + native=native_stage_enabled(NativeStage.ATTACHMENTS), ) ) @@ -742,7 +743,7 @@ def _expand_function_environment_value( if raw_value == PRESERVE_TARGET_VALUE: return None return str( - expand_template_data( + expand_config_templates( value=raw_value, variables=effective_vars, context_values={}, @@ -750,5 +751,6 @@ def _expand_function_environment_value( allow_context=False, preserve_context_tokens=True, preserve_unknown_context=False, + native=native_stage_enabled(NativeStage.ATTACHMENTS), ) ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py index 417d45b0d9..eee929f819 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py @@ -9,12 +9,10 @@ from sqlbuild.compiler.compile._helpers.analysis.validation import ( validate_source_expression_syntax, ) +from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_template_data, -) from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, CompileSourceInput, @@ -28,6 +26,8 @@ DiscoveredProjectInputs, DiscoveredSourceFile, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.models import ResourceIdentity, UsageRecord, VisibilityRecord from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind from sqlbuild.spec.contracts.models import ( @@ -320,7 +320,7 @@ def _expand_source_template_value( def _expand_source_template_object( *, value: object, effective_vars: dict[str, object], context_label: str ) -> object: - return expand_template_data( + return expand_config_templates( value=value, variables=effective_vars, context_values={}, @@ -328,4 +328,5 @@ def _expand_source_template_object( allow_context=False, preserve_context_tokens=False, preserve_unknown_context=False, + native=native_stage_enabled(NativeStage.ATTACHMENTS), ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index 90a6663ab3..d5b6ff54c9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -279,8 +279,8 @@ def expand_config_templates( ) if outcome == INVALID_OUTCOME: raise NativeStageMismatchError( - f"Native model config rejected {context_label} templates that the Python model " - "config expands; run with SQLBUILD_COMPILER_ENGINE=python" + f"Native template expansion rejected {context_label} templates that Python " + "expands; run with SQLBUILD_COMPILER_ENGINE=python" ) return expanded diff --git a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py index f98fff0085..ca1a597625 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py @@ -5,11 +5,14 @@ import re from collections.abc import Callable +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import ( SQL_OPEN_PAREN_TOKEN, SQL_QUOTE_TOKENS, ) from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, @@ -59,6 +62,10 @@ def get_validated_model_cursor_intrinsics( def reject_cursor_intrinsics(*, sql: str, context: str) -> None: """Reject cursor intrinsics in SQL that does not own an execution interval.""" + if native_stage_enabled(NativeStage.ATTACHMENTS) and _native.sql_free_of_cursor_intrinsics( + sql, [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL] + ): + return _assert_no_reserved_cursor_markers(sql=sql, context=context) _, found = _transform_cursor_intrinsics( sql=sql, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index d3b115feab..203a118006 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_declaration_references_result, expand_declaration_references_with_spans, + expand_scanned_declaration_references, ) from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros_result, @@ -20,6 +21,7 @@ from sqlbuild.compiler.compile.constants import ( COMPILE_INPUT_READS, SQL_CONTEXT_NAME_EXTRA_TOKENS, + SQL_DOLLAR_QUOTE_TOKEN, SQL_IDENTIFIER_EXTRA_TOKEN, SQL_INTERPOLATION_TOKEN, SQL_QUOTE_TOKENS, @@ -36,6 +38,11 @@ MacroExpansionResult, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( + scan_native_declaration_references, +) from sqlbuild.compiler.sql_analysis.main._is_identifier_character import ( is_identifier_character as _is_identifier_continue, ) @@ -71,21 +78,47 @@ def expand_authored_sql_result( # noqa: PLR0913 ) -> AuthoredSqlExpansionResult: """Apply all expansion passes and retain facts emitted by those passes.""" - interpolated_sql: str = substitute_sql_vars( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, + native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) + prepared_sql: str | None = ( + prepare_static_project_vars_batch(sqls=(sql,), effective_vars=effective_vars)[0] + if native and SQL_DOLLAR_QUOTE_TOKEN not in sql + else None + ) + interpolated_sql: str = ( + prepared_sql + if prepared_sql is not None + else substitute_sql_vars( + sql=sql, + file_path=file_path, + effective_vars=effective_vars, + context_values=context_values, + ) ) declaration_context: DeclarationResolutionContext = ( declarations or DeclarationResolutionContext() ) - declaration_result: DeclarationExpansionResult = expand_declaration_references_result( - sql=interpolated_sql, - file_path=file_path, - declarations=declaration_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, + scanned_result: DeclarationExpansionResult | None = ( + expand_scanned_declaration_references( + sql=interpolated_sql, + references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], + file_path=file_path, + declarations=declaration_context, + value_renderer=value_renderer, + collection_rendering=collection_rendering, + ) + if native + else None + ) + declaration_result: DeclarationExpansionResult = ( + scanned_result + if scanned_result is not None + else expand_declaration_references_result( + sql=interpolated_sql, + file_path=file_path, + declarations=declaration_context, + value_renderer=value_renderer, + collection_rendering=collection_rendering, + ) ) macro_result: MacroExpansionResult = expand_sql_macros_result( sql=declaration_result.sql, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index ff40e10148..546639e827 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -89,6 +89,7 @@ SQL_STATEMENT_TERMINATOR_TOKEN: str = ";" SQL_SINGLE_QUOTE_TOKEN: str = "'" SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) +SQL_DOLLAR_QUOTE_TOKEN: str = "$" SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) SQL_IDENTIFIER_EXTRA_TOKEN: str = "_" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index b10c7aab0c..e9670e8424 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -24,6 +24,7 @@ NativeStage.MODEL_LOOP: NativeStageTier.PREVIEW, NativeStage.MACRO_CALLS: NativeStageTier.PREVIEW, NativeStage.MACRO_CALL_STORE: NativeStageTier.PREVIEW, + NativeStage.ATTACHMENTS: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index b1dd2339c0..70b4d146c3 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -22,6 +22,7 @@ class NativeStage(StrEnum): MODEL_LOOP = "model_loop" MACRO_CALLS = "macro_calls" MACRO_CALL_STORE = "macro_call_store" + ATTACHMENTS = "attachments" class NativeStageTier(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/attachments/__init__.py b/tests/integration/src/sqlbuild/compiler/attachments/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py new file mode 100644 index 0000000000..f9f49c6ebe --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class AuthoredSqlParityTestCase: + """Seeded authored SQL expanded by the Python engine and by the preview engine.""" + + description: str + seed: int + count: int + expected_minimum_expanded: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class AttachedAuditParityTestCase: + """Seeded attached audits rendered natively and by Python's attachment helpers.""" + + description: str + seed: int + count: int + expected_minimum_native: int + expected_minimum_deferred: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class AttachmentProjectTestCase: + """One project variant compiled to compile inputs by each engine.""" + + description: str + expected_preview_entries: frozenset[str] + + +@dataclass(frozen=True) +class CursorIntrinsicParityTestCase: + """Seeded SQL checked for cursor intrinsics natively and by Python.""" + + description: str + seed: int + count: int + expected_minimum_free: int + expected_minimum_python_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py new file mode 100644 index 0000000000..cde1aa80cd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -0,0 +1,477 @@ +"""Generated authored SQL and attached audits for the native attachment parity tests.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import cast + +import pytest + +import sqlbuild._native as _native +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.attachments.main._render_native_attached_audit import ( + render_native_attached_audit, +) +from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit +from sqlbuild.compiler.auditing.types import AuditRunScope, AuditSeverity +from sqlbuild.compiler.compile._helpers.attachment.audits import ( + merge_audit_arguments, + render_generic_audit_sql, + resolve_audit_run_scope, + resolve_audit_severity, +) +from sqlbuild.compiler.compile._helpers.render import macros +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics +from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + AuthoredSqlExpansionResult, + CompileAdapterContext, + CompileProjectInputs, + MacroContext, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage +from sqlbuild.compiler.planner.constants import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, +) +from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( + resolve_effective_collection_rendering, +) +from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.model_loop.helpers import ( + DECLARATIONS, + generated_reference_sql, +) + +EFFECTIVE_VARS: dict[str, object] = { + "region": "north", + "limit_rows": 10, + "ratio": 0.5, + "enabled": True, + "unset": None, + "regions": ["north", "south"], +} +_VARIABLE_TOKENS: tuple[str, ...] = ( + "@@region", + "@@limit_rows ", + "'@@region'", + "@@ratio", + "@@enabled", + "@@unset", + "@@@window_start", + "@@ENV:SQB_ORDERS_REGION", + "@@ENV:SQB_MISSING_REGION", + "@@regions", + "@@missing", + "@@CTX:run.target", + "-- @@missing\n", +) +_MACRO_CONTEXT: MacroContext = MacroContext( + adapter_name="duckdb", sql_analysis_enabled=True, target_name="dev" +) + + +def generated_authored_sql(*, rng: random.Random) -> str: + """Return SQL mixing project variables with enum and constant references.""" + + parts: list[str] = [ + generated_reference_sql(rng=rng), + *rng.choices(_VARIABLE_TOKENS, k=rng.randint(0, 3)), + ] + rng.shuffle(parts) + return " ".join(parts) + + +def authored_outcome( + *, sql: str, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> AuthoredSqlExpansionResult | str: + """Expand one string under `engine`, returning the result or Python's error text.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + monkeypatch.setenv("SQB_ORDERS_REGION", "south") + try: + return expand_authored_sql_result( + sql=sql, + file_path=Path("audits/orders.sql"), + effective_vars=EFFECTIVE_VARS, + loaded_macros={}, + macro_context=_MACRO_CONTEXT, + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + declarations=DECLARATIONS, + ) + except CompileInputError as error: + return str(error) + + +_SQL_PIECES: tuple[str, ...] = ( + "SELECT * FROM t WHERE ", + "@column", + "@'column'", + "@values", + "@'values'", + "@limit_rows", + "@@column", + "@@@window", + "@tidy(@column)", + "@column (x)", + " AND ", + "'@column'", + "@column\u00e9", + "@column\u00a0(", + "\n", +) +_VALUES: tuple[object, ...] = ( + "status", + "it's", + 1, + 1.0, + -0.0, + True, + None, + ["placed", 2, None], + ("placed",), + {"nested": 1}, + float("nan"), + AuditSeverity.WARN, + 10**20, +) +_SEVERITIES: tuple[str | None, ...] = (None, None, None, "warn", "error", AuditSeverity.WARN) +_RUN_SCOPES: tuple[str | None, ...] = (None, None, None, "final", "delta_and_final") +_RARE: tuple[tuple[str, object], ...] = ( + ("sql", "@missing"), + ("severity", "fatal"), + ("severity", ""), + ("run_scope", "delta"), + ("column", "amount"), + ("column", 1), +) + + +@dataclass(frozen=True) +class GeneratedAudit: + """One attachment: SQL, arguments and authored policies.""" + + sql_body: str + evidence_sql: str + implicit_arguments: dict[str, object] + explicit_arguments: dict[str, object] + instance_severity: str | None + default_severity: str | None + instance_run_scope: str | None + default_run_scope: str | None + + +@dataclass(frozen=True) +class AuditParity: + """Python's rendering (or error text) and the native rendering (or None).""" + + audit: GeneratedAudit + python: tuple[object, ...] | str + native: NativeRenderedAudit | None + + +def generated_audit(*, rng: random.Random) -> GeneratedAudit: + """Return one attachment mixing parameter shapes, argument values and policies.""" + + rare: dict[str, object] = dict(rng.choices(_RARE, k=int(rng.random() < 0.3))) + explicit: dict[str, object] = { + "values": rng.choice(_VALUES), + "limit_rows": rng.choice(_VALUES), + } + explicit.update(filter(_is_column_override, rare.items())) + return GeneratedAudit( + sql_body="".join(rng.choices(_SQL_PIECES, k=rng.randint(1, 8))) + str(rare.get("sql", "")), + evidence_sql="SELECT @column FROM t", + implicit_arguments={"column": rng.choice(("status", "amount"))}, + explicit_arguments=explicit, + instance_severity=cast(str | None, rare.get("severity", rng.choice(_SEVERITIES))), + default_severity=rng.choice(_SEVERITIES), + instance_run_scope=cast(str | None, rare.get("run_scope", rng.choice(_RUN_SCOPES))), + default_run_scope=rng.choice(_RUN_SCOPES), + ) + + +def _is_column_override(item: tuple[str, object]) -> bool: + return item[0] == "column" + + +def audit_parity(audit: GeneratedAudit) -> AuditParity: + """Render one attachment with Python's helpers and natively.""" + + return AuditParity( + audit=audit, + python=_python_rendering(audit), + native=render_native_attached_audit( + sql_body=audit.sql_body, + evidence_sql=audit.evidence_sql, + implicit_arguments=audit.implicit_arguments, + explicit_arguments=audit.explicit_arguments, + policies=NativeAuditPolicies( + measurement=False, + has_thresholds=False, + has_minimum_samples=False, + threshold_error=False, + instance_severity=audit.instance_severity, + default_severity=audit.default_severity, + instance_run_scope=audit.instance_run_scope, + default_run_scope=audit.default_run_scope, + ), + ), + ) + + +def is_native(parity: AuditParity) -> bool: + """Whether the native rendering answered instead of deferring to Python.""" + + return parity.native is not None + + +def native_outcome(parity: AuditParity) -> tuple[object, ...]: + """The native rendering spelled like Python's, with the run scope value it selects.""" + + rendered: NativeRenderedAudit = cast(NativeRenderedAudit, parity.native) + run_scope: str | None = { + "instance": parity.audit.instance_run_scope, + "default": parity.audit.default_run_scope, + }.get(rendered.run_scope_source, AuditRunScope.DELTA_AND_FINAL) + return (rendered.sql_body, rendered.evidence_sql, AuditSeverity(rendered.severity), run_scope) + + +def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: + owner: Path = Path("models/orders.sql") + try: + merged: dict[str, object] = merge_audit_arguments( + owner_file=owner, + definition_name="floor", + implicit_arguments=audit.implicit_arguments, + explicit_arguments=audit.explicit_arguments, + ) + rendered: tuple[str, ...] = tuple( + render_generic_audit_sql( + sql=sql, arguments=merged, owner_file=owner, definition_name="floor" + ) + for sql in (audit.sql_body, audit.evidence_sql) + ) + severity: AuditSeverity = resolve_audit_severity( + instance_severity=audit.instance_severity, + default_severity=audit.default_severity, + audit_label="floor", + ) + run_scope: str = resolve_audit_run_scope( + instance_run_scope=audit.instance_run_scope, + default_run_scope=audit.default_run_scope, + ) + except CompileInputError as error: + return str(error) + return (*rendered, severity, run_scope) + + +NATIVE_ATTACHMENT_ENTRIES: tuple[str, ...] = ( + "pair_seed_files", + "render_attached_generic_audit", + "expand_config_templates", + "substitute_static_project_vars", + "scan_sql_declaration_references", +) +_MISSING_ENV: str = "SQB_ATTACHMENTS_UNSET" +ATTACHMENT_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nregion = "north"\n\n' + '[connections.local]\ndatabase = "orders.duckdb"\n\n' + '[targets.dev]\nconnection = "local"\nschema = "dev"\n' + ), + "macros/labels.py": ( + "def tidy_label(expression: str) -> str:\n" + ' """Trim and lower-case a label."""\n' + ' return f"LOWER(TRIM({expression}))"\n' + ), + "seeds/channel_codes.csv": "id,label\n1,web\n2,store\n", + "seeds/channel_codes.yml": ( + "seeds:\n - name: channel_codes\n description: Channel codes.\n columns:\n" + " - name: id\n type: INTEGER\n audits:\n - not_null\n" + " - name: label\n type: VARCHAR\n" + ), + "seeds/product_codes.csv": "id\n1\n", + "seeds/product_codes.yml": ( + "seeds:\n - name: product_codes\n columns:\n - name: id\n type: INTEGER\n" + ), + "sources/events.yml": ( + "sources:\n - name: order_events\n" + f" description: \"Order feed ${{coalesce(ENV:{_MISSING_ENV}, 'events')}}\"\n" + " expression: \"(SELECT 1 AS id, @tidy_label('@@region') AS region)\"\n" + " columns:\n - name: id\n type: INTEGER\n audits:\n" + " - accepted_values:\n values: [1, 2]\n" + " - name: region\n type: VARCHAR\n" + ), + "functions/sql/order_label.sql": ( + 'FUNCTION (\n description "Label an order status.",\n' + f" schema \"${{coalesce(ENV:{_MISSING_ENV}, 'udfs')}}\",\n" + " arguments (raw_status STRING),\n returns STRING,\n);\n\n" + 'UPPER(@tidy_label("raw_status"))\n' + ), + "audits/generic/amount_floor.sql": ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum ' + "AND @tidy_label(\"label\") <> @'label'\n" + ), + "models/orders.sql": ( + 'MODEL (\n materialized table,\n description "Orders.",\n' + ' audits [amount_floor (column amount, minimum -5, label "it\'s", severity warn)],\n' + ");\n\nSELECT c.id, 1.5 AS amount, c.label\n" + 'FROM __seed("channel_codes") c\nJOIN __source("order_events") e ON e.id = c.id\n' + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\nWITH\n__seed__channel_codes AS (\n" + " SELECT 1 AS id, @tidy_label(\"' Web '\") AS label\n),\n" + "__source__order_events AS (\n SELECT 1 AS id, 'north' AS region\n),\n" + "__expected__orders AS (\n SELECT 1 AS id, 1.5 AS amount, 'web' AS label\n)\n" + "SELECT 1\n" + ), + "tests/scenarios/orders_scenario.sql": ( + 'SCENARIO (\n description "Orders join their channel"\n);\n\n' + "WITH\n__seed__channel_codes AS (\n SELECT 1 AS id, @tidy_label(\"'Web'\") AS label\n),\n" + "__source__order_events AS (\n SELECT 1 AS id, 'north' AS region\n),\n" + "__expected__orders AS (\n SELECT 1 AS id, 1.5 AS amount, 'web' AS label\n)\n" + "SELECT 1\n" + ), +} + + +def attachment_engine_outcome( + *, + project_dir: Path, + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[frozenset[str], str]: + """Build compile inputs under `engine`; return the native entries called and the outcome.""" + + for relative_path, contents in ATTACHMENT_PROJECT.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + called: set[str] = set() + for name in NATIVE_ATTACHMENT_ENTRIES: + monkeypatch.setattr(_native, name, _recorded(called=called, name=name)) + bridged: list[Path] = [] + monkeypatch.setattr(macros, "_bridged_sql_macros", _recorded_bridge(bridged=bridged)) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + inputs: CompileProjectInputs = compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial(_compile_inputs, project_dir=project_dir), + ) + called.update(f"bridged:{str(path).replace(str(project_dir), '')}" for path in bridged) + return frozenset(called), repr( + ( + inputs.seed_inputs, + inputs.source_inputs, + inputs.sql_function_inputs, + inputs.audit_inputs, + inputs.test_inputs, + inputs.scenario_inputs, + ) + ).replace(str(project_dir), "") + + +def _compile_inputs(*, project_dir: Path) -> CompileProjectInputs: + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + adapter: DuckDbAdapter = DuckDbAdapter() + return build_compile_inputs( + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="20261008T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + + +def _recorded_bridge(*, bridged: list[Path]) -> Callable[..., object]: + entry: Callable[..., object] = cast(Callable[..., object], macros._bridged_sql_macros) + + def recorded(*, consumer_path: Path, **arguments: object) -> object: + bridged.append(consumer_path) + return entry(consumer_path=consumer_path, **arguments) + + return recorded + + +def _recorded(*, called: set[str], name: str) -> Callable[..., object]: + entry: Callable[..., object] = getattr(_native, name) + + def recorded(*args: object) -> object: + called.add(name) + return entry(*args) + + return recorded + + +_INTRINSIC_PIECES: tuple[str, ...] = ( + "SELECT ", + "__cursor_start", + "__cursor_end", + "()", + "( )", + "(1)", + "x", + "_", + "\u00e9", + "'", + '"', + "`", + "``", + "''", + "$$", + "$tag$", + "$1", + "--", + "\n", + "/*", + "*/", + " ", + MICROBATCH_START_SENTINEL, +) + + +def generated_intrinsic_sql(*, rng: random.Random) -> str: + """Return SQL mixing intrinsic names with quotes, comments and identifier neighbours.""" + + return "".join(rng.choices(_INTRINSIC_PIECES, k=rng.randint(1, 10))) + + +def python_intrinsic_outcome(sql: str) -> str | None: + """Python's rejection message, or None where Python accepts the SQL.""" + + try: + reject_cursor_intrinsics(sql=sql, context="Source expression 'orders'") + except CompileInputError as error: + return str(error) + return None + + +def native_intrinsic_free(sql: str) -> bool: + """Whether the native check accepts the SQL without Python.""" + + return _native.sql_free_of_cursor_intrinsics( + sql, [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL] + ) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py new file mode 100644 index 0000000000..e05a158982 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py @@ -0,0 +1,78 @@ +"""Every attachment kind compiles like Python under preview, with macros through the bridge.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + AttachmentProjectTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + attachment_engine_outcome, +) + +_BRIDGED_CONSUMERS: frozenset[str] = frozenset( + { + "bridged:/audits/generic/accepted_values.sql", + "bridged:/audits/generic/not_null.sql", + "bridged:/audits/generic/amount_floor.sql", + "bridged:/functions/sql/order_label.sql", + "bridged:/models/orders.sql", + "bridged:/sources/events.yml", + "bridged:/tests/scenarios/orders_scenario.sql", + "bridged:/tests/unit/test_orders.sql", + } +) +_PREVIEW_ONLY_ENTRIES: frozenset[str] = ( + frozenset({"pair_seed_files", "render_attached_generic_audit", "expand_config_templates"}) + | _BRIDGED_CONSUMERS +) +_ATTACHMENT_ENTRIES: frozenset[str] = frozenset( + { + "pair_seed_files", + "render_attached_generic_audit", + "expand_config_templates", + "substitute_static_project_vars", + "scan_sql_declaration_references", + } +) + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentProjectTestCase( + description="macros in every attachment kind, seeds, templates and generic audits", + expected_preview_entries=_ATTACHMENT_ENTRIES | _BRIDGED_CONSUMERS, + ), + ], + ids=lambda case: case.description, +) +def test_given_attachment_project_when_building_inputs_then_preview_matches_python( + test_case: AttachmentProjectTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + outcomes: dict[CompilerEngine, tuple[frozenset[str], str]] = { + engine: attachment_engine_outcome( + project_dir=tmp_path / engine.value, + engine=engine, + monkeypatch=monkeypatch, + ) + for engine in CompilerEngine + } + python_called, python_outcome = outcomes[CompilerEngine.PYTHON] + native_called, native_outcome = outcomes[CompilerEngine.NATIVE] + preview_called, preview_outcome = outcomes[CompilerEngine.NATIVE_PREVIEW] + + assert ( + preview_outcome.replace(CompilerEngine.NATIVE_PREVIEW.value, "python"), + native_outcome.replace(CompilerEngine.NATIVE.value, "python"), + preview_called >= test_case.expected_preview_entries, + (python_called | native_called) & _PREVIEW_ONLY_ENTRIES, + ) == (python_outcome, python_outcome, True, frozenset()), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py new file mode 100644 index 0000000000..c2bd59ba97 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py @@ -0,0 +1,60 @@ +"""Natively rendered attached audits match Python's argument merge, SQL and policies.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + AttachedAuditParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + AuditParity, + audit_parity, + generated_audit, + is_native, + native_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + AttachedAuditParityTestCase( + description="seeded parameters, argument values, overrides and policies", + seed=20261008, + count=5000, + expected_minimum_native=800, + expected_minimum_deferred=2000, + expected_minimum_python_errors=1000, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_attachments_when_rendering_natively_then_python_rendering_matches( + test_case: AttachedAuditParityTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + + parities: list[AuditParity] = [ + audit_parity(generated_audit(rng=rng)) for _ in range(test_case.count) + ] + + answered: list[AuditParity] = list(filter(is_native, parities)) + assert ( + mismatches( + inputs=[parity.audit for parity in answered], + expected=[parity.python for parity in answered], + actual=list(map(native_outcome, answered)), + ), + len(answered) >= test_case.expected_minimum_native, + len(parities) - len(answered) >= test_case.expected_minimum_deferred, + sum(isinstance(parity.python, str) for parity in parities) + >= test_case.expected_minimum_python_errors, + ) == ([], True, True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py new file mode 100644 index 0000000000..1d6ed4a94b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py @@ -0,0 +1,58 @@ +"""Authored SQL outside models expands identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.compile.models import AuthoredSqlExpansionResult +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + AuthoredSqlParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + authored_outcome, + generated_authored_sql, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + AuthoredSqlParityTestCase( + description="seeded variables, enum and constant references, quotes and comments", + seed=20261008, + count=2000, + expected_minimum_expanded=600, + expected_minimum_python_errors=300, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_authored_sql_when_expanding_with_preview_then_python_output_matches( + test_case: AuthoredSqlParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_authored_sql(rng=rng) for _ in range(test_case.count)] + + python: list[AuthoredSqlExpansionResult | str] = [ + authored_outcome(sql=sql, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + for sql in sqls + ] + preview: list[AuthoredSqlExpansionResult | str] = [ + authored_outcome(sql=sql, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch) + for sql in sqls + ] + + assert ( + mismatches(inputs=[*sqls], expected=[*python], actual=[*preview]), + sum(isinstance(item, AuthoredSqlExpansionResult) for item in python) + >= test_case.expected_minimum_expanded, + sum(isinstance(item, str) for item in python) >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_cursor_intrinsics.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_cursor_intrinsics.py new file mode 100644 index 0000000000..5276eb1a88 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_cursor_intrinsics.py @@ -0,0 +1,54 @@ +"""The native cursor intrinsic check accepts only SQL that Python accepts.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + CursorIntrinsicParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + generated_intrinsic_sql, + native_intrinsic_free, + python_intrinsic_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + CursorIntrinsicParityTestCase( + description="seeded intrinsic names, quotes, comments and identifier neighbours", + seed=20261008, + count=20000, + expected_minimum_free=10000, + expected_minimum_python_errors=8000, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_sql_when_checking_natively_then_only_python_accepted_sql_is_free( + test_case: CursorIntrinsicParityTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_intrinsic_sql(rng=rng) for _ in range(test_case.count)] + + free: list[str] = list(filter(native_intrinsic_free, sqls)) + python: list[str | None] = list(map(python_intrinsic_outcome, sqls)) + + assert ( + mismatches( + inputs=[*free], + expected=[None] * len(free), + actual=[*map(python_intrinsic_outcome, free)], + ), + len(free) >= test_case.expected_minimum_free, + len(python) - python.count(None) >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 509a1ebdba..65aa6aa58f 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -177,7 +177,7 @@ def test_given_generated_templates_when_expanding_then_native_matches_python_or_ engine="native-preview", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 6, + "expand_config_templates": 7, "config_contains_template": 2, "config_contains_macro_call": 2, }, @@ -212,7 +212,7 @@ def test_given_engine_tier_when_building_model_inputs_then_native_config_runs_on NativeRejectionTestCase( description="templates", native_entry="expand_config_templates", - expected_message="templates that the Python model config expands", + expected_message="templates that Python expands", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 02134d10f3..5a9b4953ad 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -139,6 +139,24 @@ stage=NativeStage.MACRO_CALL_STORE, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_attachments", + engine=CompilerEngine.PYTHON, + stage=NativeStage.ATTACHMENTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_attachments", + engine=CompilerEngine.NATIVE, + stage=NativeStage.ATTACHMENTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_attachments", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.ATTACHMENTS, + expected_enabled=True, + ), ], ids=lambda case: case.description, )