From 2d4cc630b493101d693f8d84036bbfedb3094dbc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:47:15 +0100 Subject: [PATCH 1/2] feat(native): assemble project resource facts natively behind the preview engine Native project assembly derives model, function and audit dependencies, managed source and seed destinations (with variable, environment and context templates and their recorded reads), and model and hook syntax validation. Anything native cannot reproduce exactly hands the whole resource assembly back to Python with a project_assembly deferral record. --- Cargo.lock | 1 + crates/sqlbuild-analysis/Cargo.toml | 1 + .../src/assembly/project/_helpers/deps.rs | 51 ++++ .../src/assembly/project/_helpers/mod.rs | 4 + .../src/assembly/project/_helpers/syntax.rs | 86 ++++++ .../src/assembly/project/_helpers/targets.rs | 176 ++++++++++++ .../assembly/project/_helpers/templates.rs | 125 +++++++++ .../src/assembly/project/constants.rs | 36 +++ .../main/assemble_project_resources.rs | 73 +++++ .../src/assembly/project/main/mod.rs | 1 + .../src/assembly/project/mod.rs | 9 + .../src/assembly/project/models.rs | 123 ++++++++ .../src/assembly/project/tests.rs | 6 + .../src/assembly/project/tests/helpers.rs | 166 +++++++++++ .../project/tests/test_project_assembly.rs | 264 ++++++++++++++++++ .../src/assembly/project/tests/test_types.rs | 73 +++++ .../src/assembly/project/types.rs | 7 + .../bindings/_helpers/project_assembly/mod.rs | 1 + .../_helpers/project_assembly/registration.rs | 6 +- .../_helpers/project_assembly/resources.rs | 207 ++++++++++++++ src/sqlbuild/_native.pyi | 26 ++ .../_helpers/analysis/syntax_checks.py | 65 +++++ .../compile/_helpers/analysis/validation.py | 14 + .../compile/_helpers/assembly/project.py | 129 +++++---- .../_helpers/native_stages/assembly.py | 34 ++- .../project_assembly/_helpers/__init__.py | 1 + .../project_assembly/_helpers/deferrals.py | 24 ++ .../project_assembly/_helpers/request.py | 116 ++++++++ .../project_assembly/_helpers/resources.py | 81 ++++++ .../compiler/project_assembly/constants.py | 12 + .../main/_assemble_native_project.py | 23 -- .../_assemble_native_project_resources.py | 30 ++ .../compiler/project_assembly/models.py | 19 ++ .../compiler/project_assembly/types.py | 35 +++ .../cli/commands/main/compile/helpers.py | 2 +- .../compile/test_native_analysis_seams.py | 4 +- .../compiler/project_assembly/__init__.py | 0 .../compiler/project_assembly/_test_types.py | 37 +++ .../compiler/project_assembly/helpers.py | 228 +++++++++++++++ .../test_native_project_assembly.py | 221 +++++++++++++++ 40 files changed, 2425 insertions(+), 92 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/constants.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/models.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs create mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py create mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/__init__.py create mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py create mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/request.py create mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/resources.py create mode 100644 src/sqlbuild/compiler/project_assembly/constants.py delete mode 100644 src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py create mode 100644 src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py create mode 100644 src/sqlbuild/compiler/project_assembly/models.py create mode 100644 src/sqlbuild/compiler/project_assembly/types.py create mode 100644 tests/integration/src/sqlbuild/compiler/project_assembly/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py diff --git a/Cargo.lock b/Cargo.lock index 98a2b0cc60..e18ef87ff8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -703,6 +703,7 @@ dependencies = [ "serde_json", "sha2", "sqlbuild-core", + "sqlbuild-model-config", "sqlbuild-scopes", "sqlbuild-sqltext", ] diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 12bdc8ea41..4899c13372 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs new file mode 100644 index 0000000000..da8005a6de --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs @@ -0,0 +1,51 @@ +//! Python's dependency keys for compiled resources (`compile/_helpers/deps/dependencies.py`). + +use std::collections::HashSet; + +use crate::assembly::project::constants::{ + ATTACHED_AUDIT_KINDS, AUDIT_TARGET_DEFERRAL, DBT_REF_KIND, KIND_RESOURCES, MODEL_RESOURCE, +}; +use crate::assembly::project::models::{AuditFacts, Reference}; +use crate::assembly::project::types::Fact; +use crate::assembly::project::types::ObjectKey; + +/// `model_build_deps` and `function_build_deps`: each reference's key, first occurrence kept. +pub(crate) fn reference_deps(references: &[Reference]) -> Vec { + deduped(references.iter().map(reference_dep).collect()) +} + +/// `audit_scope_deps`: the references, then the attached resource, first occurrence kept. +pub(crate) fn audit_deps(audit: &AuditFacts) -> Fact> { + let mut keys: Vec = reference_deps(&audit.references); + if let Some((kind, name)) = &audit.attached { + if !ATTACHED_AUDIT_KINDS.contains(&kind.as_str()) { + return Err(AUDIT_TARGET_DEFERRAL.to_owned()); + } + keys.push((kind.clone(), name.clone())); + } + Ok(deduped(keys)) +} + +/// `_reference_dep`. +fn reference_dep(reference: &Reference) -> ObjectKey { + let kind: &str = reference.kind.as_str(); + if KIND_RESOURCES.contains(&kind) { + return (kind.to_owned(), reference.name.clone()); + } + if kind == DBT_REF_KIND { + let name: String = match &reference.package { + Some(package) => format!("{package}.{}", reference.name), + None => reference.name.clone(), + }; + return (DBT_REF_KIND.to_owned(), name); + } + (MODEL_RESOURCE.to_owned(), reference.name.clone()) +} + +/// `_dedupe_object_keys`. +fn deduped(keys: Vec) -> Vec { + let mut seen: HashSet = HashSet::new(); + keys.into_iter() + .filter(|key| seen.insert(key.clone())) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs new file mode 100644 index 0000000000..000b88358d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod deps; +pub(crate) mod syntax; +pub(crate) mod targets; +pub(crate) mod templates; diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs new file mode 100644 index 0000000000..179b6f4b4e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs @@ -0,0 +1,86 @@ +//! Python's `validate_sql_syntax` and `validate_hook_sql_syntax` outcome, without the message. + +use std::collections::HashMap; + +use polyglot_sql::{ComplexityGuardOptions, Dialect, ValidationOptions, validate_with_dialect}; +use serde_json::json; + +use crate::assembly::project::constants::{ + DIALECT_DEFERRAL, MAX_FUNCTION_CALL_DEPTH, NORMALIZATION_DEFERRAL, PLACEHOLDER_DEFERRAL, + PLACEHOLDER_PREFIX, +}; +use crate::assembly::project::models::SyntaxCheck; +use crate::assembly::project::types::Fact; +use crate::semantic_validation::main::normalize::normalize_analysis_sql; +use crate::semantic_validation::models::NormalizationInput; + +/// Whether Polyglot's `validate` accepts the cleaned SQL; Python raises where it does not. +pub(crate) fn syntax_valid(check: &SyntaxCheck, dialect: &str) -> Fact { + let cleaned: String = normalize_analysis_sql(NormalizationInput { + sql: check.sql.clone(), + dialect: dialect.to_owned(), + stubs: HashMap::new(), + placeholders: HashMap::new(), + }) + .map_err(|_| NORMALIZATION_DEFERRAL.to_owned())?; + let cleaned: String = if check.placeholders.is_empty() { + cleaned + } else { + placeholder_defaults(&cleaned, &check.placeholders)? + }; + let dialect: Dialect = + Dialect::get_by_name(dialect).ok_or_else(|| DIALECT_DEFERRAL.to_owned())?; + let complexity_guard: ComplexityGuardOptions = + serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) + .map_err(|_| DIALECT_DEFERRAL.to_owned())?; + let options = ValidationOptions { + complexity_guard: Some(complexity_guard), + strict_syntax: false, + semantic: false, + }; + Ok(validate_with_dialect(&cleaned, &dialect, &options).valid) +} + +/// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. +fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> Fact { + let mut result: String = String::with_capacity(sql.len()); + let mut rest: &str = sql; + while let Some(start) = rest.find(PLACEHOLDER_PREFIX) { + result.push_str(&rest[..start]); + let after: &str = &rest[start + PLACEHOLDER_PREFIX.len()..]; + let length: usize = after + .find(|character: char| !is_word(character)) + .unwrap_or(after.len()); + if after[length..] + .chars() + .next() + .is_some_and(|next| !next.is_ascii()) + { + return Err(PLACEHOLDER_DEFERRAL.to_owned()); + } + if length == 0 { + result.push_str(&rest[start..=start]); + rest = &rest[start + 1..]; + continue; + } + let name: &str = &after[..length]; + match placeholders + .iter() + .find(|(placeholder, _)| placeholder == name) + { + Some((_, default)) => result.push_str(default), + None => { + result.push_str(PLACEHOLDER_PREFIX); + result.push_str(name); + } + } + rest = &after[length..]; + } + result.push_str(rest); + Ok(result) +} + +/// An ASCII character Python's `\w` matches. +fn is_word(character: char) -> bool { + character.is_ascii_alphanumeric() || character == '_' +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs new file mode 100644 index 0000000000..5a8cdeeaca --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs @@ -0,0 +1,176 @@ +//! Python's managed source and seed targets (`assembly/project.py`, `assembly/targets.py`). + +use crate::assembly::project::_helpers::templates::{Context, TemplateInputs, expanded_text}; +use crate::assembly::project::constants::{ + DESTINATION_DATABASE_CONTEXT, DESTINATION_QUALIFIED_CONTEXT, DESTINATION_SCHEMA_CONTEXT, + DESTINATION_TABLE_CONTEXT, MODEL_ALIAS_CONTEXT, MODEL_DATABASE_CONTEXT, MODEL_NAME_CONTEXT, + MODEL_SCHEMA_CONTEXT, PRESERVE_TARGET_VALUE, PRESERVED_NAMESPACE_DEFERRAL, +}; +use crate::assembly::project::models::{ + Namespace, SeedDefaults, SeedFacts, SourceFacts, TargetNamespace, +}; +use crate::assembly::project::types::Fact; + +/// A managed source's `(database, schema)`. +pub(crate) type SourceNamespace = (Option, Option); + +/// `_build_source_relation_entry`: None where the entry stays as authored. +pub(crate) fn managed_source( + source: &SourceFacts, + target: Option<&TargetNamespace>, + inputs: &TemplateInputs<'_>, +) -> Fact> { + let Some(target) = target.filter(|_| source.managed) else { + return Ok(None); + }; + let loader_schema: Option<&String> = + python_or(target.loader_schema.as_ref(), target.schema.as_ref()); + let loader_target = TargetNamespace { + database: target.database.clone(), + schema: loader_schema.cloned(), + loader_schema: target.loader_schema.clone(), + }; + preserved_namespace( + Some(&loader_target), + source.database.as_ref(), + source.schema.as_ref(), + )?; + let database: Option = match &source.database { + Some(database) => Some(database.clone()), + None => optional_text(target.database.as_deref(), inputs)?, + }; + let schema: Option = match &source.schema { + Some(schema) => Some(schema.clone()), + None => optional_text(loader_schema.map(String::as_str), inputs)?, + }; + Ok(Some((database, schema))) +} + +/// `build_seed_relation_target`. +pub(crate) fn seed_target( + seed: &SeedFacts, + defaults: &SeedDefaults, + target: Option<&TargetNamespace>, + inputs: &TemplateInputs<'_>, +) -> Fact { + let mut logical_database: Option = environment_text( + defaults + .seed_database + .as_ref() + .or(defaults.database.as_ref()), + inputs, + )?; + let mut logical_schema: Option = environment_text( + defaults.seed_schema.as_ref().or(defaults.schema.as_ref()), + inputs, + )?; + if let Some(raw) = &seed.database { + let context = seed_context(&seed.name, logical_database.clone(), logical_schema.clone()); + logical_database = seed_text(raw, inputs, &context)?; + } + if let Some(raw) = &seed.schema { + let context = seed_context(&seed.name, logical_database.clone(), logical_schema.clone()); + logical_schema = seed_text(raw, inputs, &context)?; + } + preserved_namespace(target, logical_database.as_ref(), logical_schema.as_ref())?; + let (database, schema) = match target { + None => (logical_database.clone(), logical_schema.clone()), + Some(target) => ( + target_override(target.database.as_ref(), logical_database.clone(), inputs)?, + target_override(target.schema.as_ref(), logical_schema.clone(), inputs)?, + ), + }; + Ok(Namespace { + database, + schema, + logical_database, + logical_schema, + }) +} + +/// `validate_preserved_logical_namespace`; Python raises where Err. +fn preserved_namespace( + target: Option<&TargetNamespace>, + logical_database: Option<&String>, + logical_schema: Option<&String>, +) -> Fact<()> { + let Some(target) = target else { + return Ok(()); + }; + let preserved = + |value: Option<&String>| value.is_some_and(|value| value == PRESERVE_TARGET_VALUE); + if (preserved(target.database.as_ref()) && logical_database.is_none()) + || (preserved(target.schema.as_ref()) && logical_schema.is_none()) + { + return Err(PRESERVED_NAMESPACE_DEFERRAL.to_owned()); + } + Ok(()) +} + +/// `_expand_target_value`. +fn optional_text(value: Option<&str>, inputs: &TemplateInputs<'_>) -> Fact> { + value + .map(|value| expanded_text(value, inputs, None)) + .transpose() +} + +/// `_expand_seed_default_value` and `_expand_seed_environment_value`. +fn environment_text(raw: Option<&String>, inputs: &TemplateInputs<'_>) -> Fact> { + match raw { + None => Ok(None), + Some(raw) if raw == PRESERVE_TARGET_VALUE => Ok(None), + Some(raw) => expanded_text(raw, inputs, None).map(Some), + } +} + +/// `_expand_seed_target_value`. +fn seed_text( + raw: &str, + inputs: &TemplateInputs<'_>, + context: &Context<'_>, +) -> Fact> { + if raw == PRESERVE_TARGET_VALUE { + return Ok(None); + } + expanded_text(raw, inputs, Some(context)).map(Some) +} + +/// `_apply_seed_target_overrides` for one dimension. +fn target_override( + raw: Option<&String>, + logical: Option, + inputs: &TemplateInputs<'_>, +) -> Fact> { + match raw { + Some(raw) if raw != PRESERVE_TARGET_VALUE => environment_text(Some(raw), inputs), + _ => Ok(logical), + } +} + +/// The context values a seed's own database and schema templates read. +fn seed_context( + name: &str, + database: Option, + schema: Option, +) -> Vec<(&'static str, Option)> { + let qualified: Option = match (&database, &schema) { + (Some(database), Some(schema)) => Some(format!("{database}.{schema}.{name}")), + (None, Some(schema)) => Some(format!("{schema}.{name}")), + _ => None, + }; + vec![ + (MODEL_NAME_CONTEXT, Some(name.to_owned())), + (MODEL_DATABASE_CONTEXT, database.clone()), + (MODEL_SCHEMA_CONTEXT, schema.clone()), + (MODEL_ALIAS_CONTEXT, Some(name.to_owned())), + (DESTINATION_DATABASE_CONTEXT, database), + (DESTINATION_SCHEMA_CONTEXT, schema), + (DESTINATION_TABLE_CONTEXT, Some(name.to_owned())), + (DESTINATION_QUALIFIED_CONTEXT, qualified), + ] +} + +/// Python's `left or right` over optional text: an empty string falls through. +fn python_or<'a>(left: Option<&'a String>, right: Option<&'a String>) -> Option<&'a String> { + left.filter(|value| !value.is_empty()).or(right) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs new file mode 100644 index 0000000000..7ea87fca3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs @@ -0,0 +1,125 @@ +//! Python's `str(expand_template_data(...))` over scalar variables, the environment and context. + +use std::cell::RefCell; + +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::models::{ + ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, +}; +use sqlbuild_model_config::templates::types::TemplateHost; + +use crate::assembly::project::constants::{ + PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE, TEMPLATE_DEFERRAL, TEMPLATE_OPEN_TOKEN, +}; +use crate::assembly::project::models::{InputRead, Variable}; +use crate::assembly::project::types::Fact; + +/// Context values by name; None is a known key whose value Python holds as None. +pub(crate) type Context<'a> = [(&'static str, Option)]; + +/// The variables and environment target templates read, and every lookup they make in order. +pub(crate) struct TemplateInputs<'a> { + pub(crate) variables: &'a [(String, Variable)], + pub(crate) environment: &'a [(String, String)], + pub(crate) reads: RefCell>, +} + +/// One expansion's inputs and context values. +struct ScalarHost<'a> { + inputs: &'a TemplateInputs<'a>, + context: &'a Context<'a>, +} + +impl TemplateHost for ScalarHost<'_> { + type Value = Scalar; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + match self + .inputs + .variables + .iter() + .find(|(variable, _)| variable == name) + { + None => Ok(None), + Some((_, Variable::Scalar(value))) => Ok(Some(value.clone())), + Some((_, Variable::Unsupported)) => Err(TemplateFailure::Unsupported), + } + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.inputs + .reads + .borrow_mut() + .push(InputRead::Environment(name.to_owned())); + Ok(self + .inputs + .environment + .iter() + .find(|(variable, _)| variable == name) + .map(|(_, value)| Scalar::Text(value.clone()))) + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.inputs + .reads + .borrow_mut() + .push(InputRead::Context(name.to_owned())); + Ok(match self.context.iter().find(|(key, _)| *key == name) { + None => ContextValue::Unknown, + Some((_, None)) => ContextValue::Unavailable, + Some((_, Some(value))) => ContextValue::Value(Scalar::Text(value.clone())), + }) + } + + fn text(&self, text: &str) -> Result { + Ok(Scalar::Text(text.to_owned())) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(Scalar::Bool(flag)) + } + + fn null(&self) -> Self::Value { + Scalar::Null + } + + fn scalar(&self, value: &Self::Value) -> Option { + Some(value.clone()) + } +} + +/// `str(expand_template_data(value=text, ...))`; context is allowed only where Some. +pub(crate) fn expanded_text( + text: &str, + inputs: &TemplateInputs<'_>, + context: Option<&Context<'_>>, +) -> Fact { + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(text.to_owned()); + } + let options = TemplateOptions { + allow_context: context.is_some(), + preserve_context_tokens: false, + preserve_unknown_context: false, + }; + let host = ScalarHost { + inputs, + context: context.unwrap_or_default(), + }; + match expand_template_string(&host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(text.to_owned()), + Ok(StringExpansion::Value(value)) => Ok(python_str(value)), + Ok(StringExpansion::Text(expanded)) => Ok(expanded), + Err(_) => Err(TEMPLATE_DEFERRAL.to_owned()), + } +} + +/// Python's `str()` of a scalar. +fn python_str(value: Scalar) -> String { + match value { + Scalar::Null => PYTHON_NONE.to_owned(), + Scalar::Bool(true) => PYTHON_TRUE.to_owned(), + Scalar::Bool(false) => PYTHON_FALSE.to_owned(), + Scalar::Text(text) => text, + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs new file mode 100644 index 0000000000..d550f366b9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -0,0 +1,36 @@ +//! Python's resource, reference and target constants the project assembly reproduces. + +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +/// The reference kinds whose dependency keeps the reference's kind as its resource type. +pub(crate) const KIND_RESOURCES: [&str; 4] = + [SOURCE_KIND, SEED_KIND, UDF_KIND, TABLE_FUNCTION_KIND]; +/// `AttachedAuditTargetKind` values, each also its resource type. +pub(crate) const ATTACHED_AUDIT_KINDS: [&str; 3] = [MODEL_RESOURCE, SOURCE_KIND, SEED_KIND]; +pub(crate) const PRESERVE_TARGET_VALUE: &str = "preserve"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const PYTHON_NONE: &str = "None"; +pub(crate) const PYTHON_TRUE: &str = "True"; +pub(crate) const PYTHON_FALSE: &str = "False"; +pub(crate) const PLACEHOLDER_PREFIX: &str = "@@@"; +pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; +pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; +pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; +pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; +pub(crate) const MODEL_ALIAS_CONTEXT: &str = "model.alias"; +pub(crate) const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; +pub(crate) const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; +pub(crate) const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; +pub(crate) const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; +pub(crate) const AUDIT_TARGET_DEFERRAL: &str = "audit_target_kind"; +pub(crate) const TEMPLATE_DEFERRAL: &str = "target_template"; +pub(crate) const PRESERVED_NAMESPACE_DEFERRAL: &str = "preserved_namespace"; +pub(crate) const SYNTAX_DEFERRAL: &str = "syntax_error"; +pub(crate) const PLACEHOLDER_DEFERRAL: &str = "placeholder_word"; +pub(crate) const NORMALIZATION_DEFERRAL: &str = "normalization"; +pub(crate) const DIALECT_DEFERRAL: &str = "dialect"; +pub(crate) const PANIC_DEFERRAL: &str = "panic"; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs b/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs new file mode 100644 index 0000000000..add612c439 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs @@ -0,0 +1,73 @@ +//! Python's resource assembly facts for one project, or why Python must assemble it. + +use std::cell::RefCell; + +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::assembly::project::_helpers::deps::{audit_deps, reference_deps}; +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; +use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::constants::{PANIC_DEFERRAL, SYNTAX_DEFERRAL}; +use crate::assembly::project::models::{ModelFacts, ProjectRequest, ProjectResources}; +use crate::assembly::project::types::Fact; + +/// Each resource's dependencies and namespace, or the kind of work Python must do instead. +pub fn assemble_project_resources(request: &ProjectRequest) -> Result { + catch_compiler_panic(|| resources(request)).map_err(|kind| { + if kind == PANIC_MESSAGE { + PANIC_DEFERRAL.to_owned() + } else { + kind + } + }) +} + +fn resources(request: &ProjectRequest) -> Fact { + for model in &request.models { + valid_syntax(model, &request.dialect)?; + } + let target = request.target.as_ref(); + let inputs = TemplateInputs { + variables: &request.variables, + environment: &request.environment, + reads: RefCell::new(Vec::new()), + }; + let sources = request + .sources + .iter() + .map(|source| managed_source(source, target, &inputs)) + .collect::>()?; + let seeds = request + .seeds + .iter() + .map(|seed| seed_target(seed, &request.defaults, target, &inputs)) + .collect::>()?; + Ok(ProjectResources { + model_deps: request + .models + .iter() + .map(|model| reference_deps(&model.references)) + .collect(), + sources, + seeds, + function_deps: request + .functions + .iter() + .map(|references| reference_deps(references)) + .collect(), + audit_deps: request.audits.iter().map(audit_deps).collect::>()?, + reads: inputs.reads.into_inner(), + }) +} + +/// Python's model and hook syntax validation, which raises for the first SQL it rejects. +fn valid_syntax(model: &ModelFacts, dialect: &str) -> Fact<()> { + for check in &model.syntax_checks { + if !syntax_valid(check, dialect)? { + return Err(SYNTAX_DEFERRAL.to_owned()); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs new file mode 100644 index 0000000000..d53093ec5a --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -0,0 +1 @@ +pub mod assemble_project_resources; diff --git a/crates/sqlbuild-analysis/src/assembly/project/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/mod.rs index 2c6b3134d3..4439eb2731 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/mod.rs @@ -1 +1,10 @@ //! Compiled project assembly: resources, dependencies, destinations and their diagnostics. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/assembly/project/models.rs b/crates/sqlbuild-analysis/src/assembly/project/models.rs new file mode 100644 index 0000000000..be6dd41e7b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/models.rs @@ -0,0 +1,123 @@ +//! The project assembly request Python builds from its compile inputs, and the facts it returns. + +use sqlbuild_model_config::templates::models::Scalar; + +use crate::assembly::analysis_session::types::Pairs; +use crate::assembly::project::types::ObjectKey; + +/// One SQL reference a resource makes: Python's `CompileSqlReference`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reference { + /// The `SqlReferenceKind` value. + pub kind: String, + pub name: String, + pub package: Option, +} + +/// One project variable, as a scalar Python renders with `str()`, or a value only Python renders. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Variable { + Scalar(Scalar), + Unsupported, +} + +/// One environment or context lookup a template made, which Python records for compile reuse. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InputRead { + Environment(String), + Context(String), +} + +/// The selected target's namespace settings. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TargetNamespace { + pub database: Option, + pub schema: Option, + pub loader_schema: Option, +} + +/// The project defaults a seed's logical namespace reads. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SeedDefaults { + pub database: Option, + pub schema: Option, + pub seed_database: Option, + pub seed_schema: Option, +} + +/// One model or hook SQL string Python's syntax validation checks, with its placeholders. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyntaxCheck { + pub sql: String, + pub placeholders: Pairs, +} + +/// One model's references and the SQL strings its assembly validates. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModelFacts { + pub references: Vec, + pub syntax_checks: Vec, +} + +/// One source's name, whether a loader manages it, and its authored namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceFacts { + pub name: String, + pub managed: bool, + pub database: Option, + pub schema: Option, +} + +/// One seed's name and authored namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SeedFacts { + pub name: String, + pub database: Option, + pub schema: Option, +} + +/// One audit's references and the resource it is attached to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditFacts { + pub references: Vec, + /// The attached resource's `(resource type, name)`. + pub attached: Option<(String, String)>, +} + +/// Everything one project's resource assembly reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectRequest { + /// The analysis dialect syntax validation parses with, `generic` when the adapter names none. + pub dialect: String, + pub target: Option, + pub defaults: SeedDefaults, + pub variables: Vec<(String, Variable)>, + /// The process environment `os.environ` holds. + pub environment: Vec<(String, String)>, + pub models: Vec, + pub sources: Vec, + pub seeds: Vec, + pub functions: Vec>, + pub audits: Vec, +} + +/// A relation's physical and logical namespace. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Namespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +/// The facts Python's resource assembly derives, in input order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProjectResources { + pub model_deps: Vec>, + /// A managed source's `(database, schema)`; None where the entry stays as authored. + pub sources: Vec, Option)>>, + pub seeds: Vec, + pub function_deps: Vec>, + pub audit_deps: Vec>, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs new file mode 100644 index 0000000000..b8a5f96c5e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_project_assembly.rs"] +mod project_assembly; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs new file mode 100644 index 0000000000..77f00584a1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs @@ -0,0 +1,166 @@ +use std::cell::RefCell; + +use sqlbuild_model_config::templates::models::Scalar; + +use crate::assembly::project::_helpers::deps::audit_deps; +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; +use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::models::{ + AuditFacts, InputRead, Namespace, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, + TargetNamespace, Variable, +}; +use crate::assembly::project::tests::test_types::{ExpectedNamespace, SeedSpec}; +use crate::assembly::project::types::ObjectKey; + +/// `(database, schema, loader schema)` of a selected target. +type TargetSpec = ( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, +); + +pub(crate) fn deps( + references: &[(&str, &str, Option<&str>)], + attached: Option<(&str, &str)>, +) -> Option> { + audit_deps(&AuditFacts { + references: references.iter().map(reference).collect(), + attached: attached.map(|(kind, name)| (kind.to_owned(), name.to_owned())), + }) + .ok() +} + +pub(crate) fn seed( + seed: SeedSpec, + defaults: ExpectedNamespace, + target: Option, +) -> Option { + let (name, database, schema) = seed; + let (default_database, default_schema, seed_database, seed_schema) = defaults; + seed_target( + &SeedFacts { + name: name.to_owned(), + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + &SeedDefaults { + database: default_database.map(str::to_owned), + schema: default_schema.map(str::to_owned), + seed_database: seed_database.map(str::to_owned), + seed_schema: seed_schema.map(str::to_owned), + }, + target.map(namespace_target).as_ref(), + &template_inputs(&variables()), + ) + .ok() +} + +/// The seed's namespace with `environment` set, and the reads its templates made. +pub(crate) fn seed_reads( + seed: SeedSpec, + environment: &[(String, String)], +) -> (Option, Vec) { + let (name, database, schema) = seed; + let variables: Vec<(String, Variable)> = variables(); + let inputs = TemplateInputs { + variables: &variables, + environment, + reads: RefCell::new(Vec::new()), + }; + let resolved = seed_target( + &SeedFacts { + name: name.to_owned(), + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + &SeedDefaults::default(), + None, + &inputs, + ) + .ok(); + (resolved, inputs.reads.into_inner()) +} + +pub(crate) fn source( + source: (bool, Option<&str>, Option<&str>), + target: Option, +) -> Option, Option)>> { + let (managed, database, schema) = source; + managed_source( + &SourceFacts { + name: "orders".to_owned(), + managed, + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + target.map(namespace_target).as_ref(), + &template_inputs(&variables()), + ) + .ok() +} + +pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)]) -> Option { + syntax_valid( + &SyntaxCheck { + sql: sql.to_owned(), + placeholders: placeholders + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(), + }, + "duckdb", + ) + .ok() +} + +pub(crate) fn namespace(expected: ExpectedNamespace) -> Namespace { + let (database, schema, logical_database, logical_schema) = expected; + Namespace { + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + logical_database: logical_database.map(str::to_owned), + logical_schema: logical_schema.map(str::to_owned), + } +} + +fn reference(spec: &(&str, &str, Option<&str>)) -> Reference { + let (kind, name, package) = spec; + Reference { + kind: (*kind).to_owned(), + name: (*name).to_owned(), + package: package.map(str::to_owned), + } +} + +fn namespace_target(spec: TargetSpec) -> TargetNamespace { + let (database, schema, loader_schema) = spec; + TargetNamespace { + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + loader_schema: loader_schema.map(str::to_owned), + } +} + +fn template_inputs(variables: &[(String, Variable)]) -> TemplateInputs<'_> { + TemplateInputs { + variables, + environment: &[], + reads: RefCell::new(Vec::new()), + } +} + +fn variables() -> Vec<(String, Variable)> { + vec![ + ( + "env_name".to_owned(), + Variable::Scalar(Scalar::Text("prod".to_owned())), + ), + ( + "n".to_owned(), + Variable::Scalar(Scalar::Text("3".to_owned())), + ), + ("flag".to_owned(), Variable::Scalar(Scalar::Bool(true))), + ("tags".to_owned(), Variable::Unsupported), + ] +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs new file mode 100644 index 0000000000..c6b82aeb41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs @@ -0,0 +1,264 @@ +use crate::assembly::project::models::InputRead; +use crate::assembly::project::tests::helpers::{deps, namespace, seed, seed_reads, source, valid}; +use crate::assembly::project::tests::test_types::{ + DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxTestCase, keys, +}; + +#[test] +fn given_references_when_deriving_deps_then_matches_python_keys() { + let test_cases = [ + DepsTestCase { + description: "each kind maps to its resource and repeats keep the first key", + references: &[ + ("ref", "orders", None), + ("source", "raw_orders", None), + ("ref", "orders", None), + ("dbt_ref", "customers", Some("upstream")), + ("dbt_ref", "products", None), + ("udf", "tax", None), + ("table_fn", "series", None), + ("seed", "countries", None), + ], + attached: Some(("model", "orders")), + expected_deps: Some(&[ + ("model", "orders"), + ("source", "raw_orders"), + ("dbt_ref", "upstream.customers"), + ("dbt_ref", "products"), + ("udf", "tax"), + ("table_fn", "series"), + ("seed", "countries"), + ]), + }, + DepsTestCase { + description: "an attached target joins the references", + references: &[("ref", "orders", None)], + attached: Some(("seed", "countries")), + expected_deps: Some(&[("model", "orders"), ("seed", "countries")]), + }, + DepsTestCase { + description: "an attached kind Python rejects", + references: &[], + attached: Some(("function", "tax")), + expected_deps: None, + }, + ]; + for test_case in test_cases { + let derived = deps(test_case.references, test_case.attached); + + assert_eq!( + derived, + test_case.expected_deps.map(keys), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_seeds_when_resolving_targets_then_matches_python_or_defers() { + let test_cases = [ + SeedTestCase { + description: "defaults and the seed's templates read variables and updated context", + seed: ( + "countries", + Some("${n}"), + Some("seeds_${env_name}_${CTX:model.database}"), + ), + defaults: (Some("db_${env_name}"), None, None, Some("raw")), + target: None, + expected_namespace: Some(( + Some("3"), + Some("seeds_prod_3"), + Some("3"), + Some("seeds_prod_3"), + )), + }, + SeedTestCase { + description: "a target overrides the schema and preserves the database", + seed: ("countries", None, None), + defaults: (Some("analytics"), Some("main"), None, None), + target: Some((Some("preserve"), Some("dev_${env_name}"), None)), + expected_namespace: Some(( + Some("analytics"), + Some("dev_prod"), + Some("analytics"), + Some("main"), + )), + }, + SeedTestCase { + description: "the qualified destination context", + seed: ("countries", None, Some("${CTX:destination.qualified}_x")), + defaults: (Some("analytics"), Some("main"), None, None), + target: None, + expected_namespace: Some(( + Some("analytics"), + Some("analytics.main.countries_x"), + Some("analytics"), + Some("analytics.main.countries_x"), + )), + }, + SeedTestCase { + description: "a whole-template boolean renders as Python's str", + seed: ("countries", None, Some("${flag}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: Some((None, Some("True"), None, Some("True"))), + }, + SeedTestCase { + description: "a preserved schema the seed does not own raises in Python", + seed: ("countries", None, None), + defaults: (None, None, None, None), + target: Some((None, Some("preserve"), None)), + expected_namespace: None, + }, + SeedTestCase { + description: "environment reads stay with Python", + seed: ("countries", None, Some("${ENV:SEED_SCHEMA}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: None, + }, + SeedTestCase { + description: "a variable only Python renders", + seed: ("countries", None, Some("${tags}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: None, + }, + ]; + for test_case in test_cases { + let resolved = seed(test_case.seed, test_case.defaults, test_case.target); + + assert_eq!( + resolved, + test_case.expected_namespace.map(namespace), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sources_when_resolving_managed_targets_then_matches_python() { + let test_cases = [ + SourceTestCase { + description: "an empty loader schema falls back to the target schema", + source: (true, None, None), + target: Some((Some("wh_${env_name}"), Some("main"), Some(""))), + expected_namespace: Some(Some((Some("wh_prod"), Some("main")))), + }, + SourceTestCase { + description: "an authored schema wins over the loader schema", + source: (true, None, Some("landing")), + target: Some((Some("wh"), None, Some("loaders"))), + expected_namespace: Some(Some((Some("wh"), Some("landing")))), + }, + SourceTestCase { + description: "an unmanaged source stays as authored", + source: (false, None, None), + target: Some((Some("wh"), Some("main"), None)), + expected_namespace: Some(None), + }, + ]; + for test_case in test_cases { + let resolved = source(test_case.source, test_case.target); + + assert_eq!( + resolved, + test_case.expected_namespace.map(|namespace| { + namespace.map(|(database, schema)| { + (database.map(str::to_owned), schema.map(str::to_owned)) + }) + }), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_sql_when_validating_syntax_then_matches_python_or_defers() { + let test_cases = [ + SyntaxTestCase { + description: "markers and placeholder defaults parse", + sql: "SELECT @@@x AS a FROM __ref(\"orders\")", + placeholders: &[("x", "1")], + expected_valid: Some(true), + }, + SyntaxTestCase { + description: "a syntax error", + sql: "SELECT FROM WHERE (", + placeholders: &[], + expected_valid: Some(false), + }, + SyntaxTestCase { + description: "a placeholder run retried one character later, as re.sub does", + sql: "SELECT @@@@@abc AS a", + placeholders: &[("abc", "version")], + expected_valid: Some(true), + }, + SyntaxTestCase { + description: "a placeholder followed by text Python's \\w may extend", + sql: "SELECT @@@x\u{e9} AS a", + placeholders: &[("x", "1")], + expected_valid: None, + }, + ]; + for test_case in test_cases { + let checked = valid(test_case.sql, test_case.placeholders); + + assert_eq!( + checked, test_case.expected_valid, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_environment_templates_when_resolving_seeds_then_matches_python_values_and_reads() { + let test_cases = [ + EnvironmentTestCase { + description: "a coalesced unset variable, a set variable and a context key", + seed: ( + "countries", + None, + Some( + "${coalesce(ENV:SQB_UNSET_SCHEMA, 'main')}_${ENV:SQB_SET_SCHEMA}_${CTX:model.name}", + ), + ), + environment: &[("SQB_SET_SCHEMA", "landing")], + expected_schema: Some("main_landing_countries"), + expected_reads: vec![ + InputRead::Environment("SQB_UNSET_SCHEMA".to_owned()), + InputRead::Environment("SQB_SET_SCHEMA".to_owned()), + InputRead::Context("model.name".to_owned()), + ], + }, + EnvironmentTestCase { + description: "an unset variable Python raises for", + seed: ("countries", None, Some("${ENV:SQB_UNSET_SCHEMA}")), + environment: &[], + expected_schema: None, + expected_reads: vec![InputRead::Environment("SQB_UNSET_SCHEMA".to_owned())], + }, + ]; + for test_case in test_cases { + let environment: Vec<(String, String)> = test_case + .environment + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + + let (resolved, reads) = seed_reads(test_case.seed, &environment); + + assert_eq!( + resolved.and_then(|namespace| namespace.schema), + test_case.expected_schema.map(str::to_owned), + "{}", + test_case.description + ); + assert_eq!(reads, test_case.expected_reads, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs new file mode 100644 index 0000000000..9e513a6cc9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -0,0 +1,73 @@ +use crate::assembly::project::models::InputRead; +use crate::assembly::project::types::ObjectKey; + +/// `(name, database, schema)` of one authored seed. +pub(crate) type SeedSpec = (&'static str, Option<&'static str>, Option<&'static str>); +/// `(database, schema, logical database, logical schema)`. +pub(crate) type ExpectedNamespace = ( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, +); + +pub(crate) struct DepsTestCase { + pub(crate) description: &'static str, + /// `(kind, name, package)` references. + pub(crate) references: &'static [(&'static str, &'static str, Option<&'static str>)], + pub(crate) attached: Option<(&'static str, &'static str)>, + /// None where Python rejects the audit. + pub(crate) expected_deps: Option<&'static [(&'static str, &'static str)]>, +} + +pub(crate) struct SeedTestCase { + pub(crate) description: &'static str, + pub(crate) seed: SeedSpec, + /// `(database, schema, seed database, seed schema)` defaults. + pub(crate) defaults: ExpectedNamespace, + /// `(database, schema, loader schema)` of the selected target. + pub(crate) target: Option<( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + )>, + /// None where Python raises or expands the target itself. + pub(crate) expected_namespace: Option, +} + +pub(crate) struct SourceTestCase { + pub(crate) description: &'static str, + /// `(managed, database, schema)` of the authored source. + pub(crate) source: (bool, Option<&'static str>, Option<&'static str>), + pub(crate) target: Option<( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + )>, + /// Outer None where Python raises, inner None where the entry stays as authored. + pub(crate) expected_namespace: Option, Option<&'static str>)>>, +} + +pub(crate) struct SyntaxTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) placeholders: &'static [(&'static str, &'static str)], + /// None where native defers to Python. + pub(crate) expected_valid: Option, +} + +pub(crate) fn keys(values: &[(&str, &str)]) -> Vec { + values + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect() +} + +pub(crate) struct EnvironmentTestCase { + pub(crate) description: &'static str, + pub(crate) seed: SeedSpec, + pub(crate) environment: &'static [(&'static str, &'static str)], + /// None where Python raises. + pub(crate) expected_schema: Option<&'static str>, + pub(crate) expected_reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/types.rs b/crates/sqlbuild-analysis/src/assembly/project/types.rs new file mode 100644 index 0000000000..8d39093007 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/types.rs @@ -0,0 +1,7 @@ +//! Project assembly key and result aliases. + +/// One step's result, or the deferral kind Python must assemble the project for. +pub(crate) type Fact = Result; + +/// A `CompiledObjectKey`: `(resource type, name)`. +pub type ObjectKey = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 7444bd0374..79eb3404a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1 +1,2 @@ pub(crate) mod registration; +pub(crate) mod resources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 63be94a863..67e7f084fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -2,6 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { - Ok(()) +use crate::bindings::_helpers::project_assembly::resources; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + resources::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs new file mode 100644 index 0000000000..2774b9486b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -0,0 +1,207 @@ +//! Python's resource assembly facts for one project, computed natively for the preview engine. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::models::{ + AuditFacts, InputRead, ModelFacts, Namespace, ProjectRequest, ProjectResources, Reference, + SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, +}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_model_config::templates::models::Scalar; + +use crate::bindings::types::CompilerDetach; + +const TEXT_VARIABLE: &str = "text"; +const BOOLEAN_VARIABLE: &str = "bool"; +const NONE_VARIABLE: &str = "none"; +const TRUE_TEXT: &str = "1"; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; + +/// `(kind, name, package)`. +type ReferenceRow = (String, String, Option); +/// `(database, schema, loader schema)`. +type TargetRow = (Option, Option, Option); +/// `(database, schema, seed database, seed schema)`. +type DefaultsRow = ( + Option, + Option, + Option, + Option, +); +/// `(name, kind, text)`; kind `text` (also exact ints), `bool`, `none` or anything else. +type VariableRow = (String, String, String); +/// `(references, [(sql, placeholders)])`. +type ModelRow = (Vec, Vec<(String, Pairs)>); +/// `(name, managed, database, schema)`. +type SourceRow = (String, bool, Option, Option); +/// `(name, database, schema)`. +type SeedRow = (String, Option, Option); +/// `(references, attached (kind, name))`. +type AuditRow = (Vec, Option<(String, String)>); +/// The dialect, target, defaults, variables, environment and resources one project assembles. +type RequestRow = ( + String, + Option, + DefaultsRow, + Vec, + Vec<(String, String)>, + Vec, + Vec, + Vec, + Vec>, + Vec, +); +/// `(database, schema, logical database, logical schema)`. +type NamespaceRow = ( + Option, + Option, + Option, + Option, +); +/// Deps, managed source and seed namespaces, and the `(env or ctx, name)` template reads. +type ResourcesRow = ( + Vec>, + Vec, Option)>>, + Vec, + Vec>, + Vec>, + Vec<(&'static str, String)>, +); + +/// The project's resource facts, or None with the reason Python must assemble it. +#[pyfunction] +fn assemble_project_resource_facts( + py: Python<'_>, + request: RequestRow, +) -> (Option, Option) { + let request: ProjectRequest = project_request(request); + match py.compiler_detach(|| assemble_project_resources(&request)) { + Ok(resources) => (Some(resources_row(resources)), None), + Err(reason) => (None, Some(reason)), + } +} + +fn project_request(request: RequestRow) -> ProjectRequest { + let ( + dialect, + target, + defaults, + variables, + environment, + models, + sources, + seeds, + functions, + audits, + ) = request; + let (database, schema, seed_database, seed_schema) = defaults; + ProjectRequest { + dialect, + target: target.map(|(database, schema, loader_schema)| TargetNamespace { + database, + schema, + loader_schema, + }), + defaults: SeedDefaults { + database, + schema, + seed_database, + seed_schema, + }, + variables: variables.into_iter().map(variable).collect(), + environment, + models: models.into_iter().map(model).collect(), + sources: sources + .into_iter() + .map(|(name, managed, database, schema)| SourceFacts { + name, + managed, + database, + schema, + }) + .collect(), + seeds: seeds + .into_iter() + .map(|(name, database, schema)| SeedFacts { + name, + database, + schema, + }) + .collect(), + functions: functions.into_iter().map(references).collect(), + audits: audits + .into_iter() + .map(|(rows, attached)| AuditFacts { + references: references(rows), + attached, + }) + .collect(), + } +} + +fn variable(row: VariableRow) -> (String, Variable) { + let (name, kind, text) = row; + let value: Variable = match kind.as_str() { + TEXT_VARIABLE => Variable::Scalar(Scalar::Text(text)), + BOOLEAN_VARIABLE => Variable::Scalar(Scalar::Bool(text == TRUE_TEXT)), + NONE_VARIABLE => Variable::Scalar(Scalar::Null), + _ => Variable::Unsupported, + }; + (name, value) +} + +fn model(row: ModelRow) -> ModelFacts { + let (rows, checks) = row; + ModelFacts { + references: references(rows), + syntax_checks: checks + .into_iter() + .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) + .collect(), + } +} + +fn references(rows: Vec) -> Vec { + rows.into_iter() + .map(|(kind, name, package)| Reference { + kind, + name, + package, + }) + .collect() +} + +fn resources_row(resources: ProjectResources) -> ResourcesRow { + ( + resources.model_deps, + resources.sources, + resources.seeds.into_iter().map(namespace_row).collect(), + resources.function_deps, + resources.audit_deps, + resources.reads.into_iter().map(read_row).collect(), + ) +} + +fn read_row(read: InputRead) -> (&'static str, String) { + match read { + InputRead::Environment(name) => (ENVIRONMENT_READ, name), + InputRead::Context(name) => (CONTEXT_READ, name), + } +} + +fn namespace_row(namespace: Namespace) -> NamespaceRow { + ( + namespace.database, + namespace.schema, + namespace.logical_database, + namespace.logical_schema, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b98b504587..f5fce9e74c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -622,6 +622,32 @@ def plan_compiled_sql_tests( def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. +def assemble_project_resource_facts( + request: tuple[ + str, + tuple[str | None, str | None, str | None] | None, + tuple[str | None, str | None, str | None, str | None], + list[tuple[str, str, str]], + list[tuple[str, str]], + list[tuple[list[tuple[str, str, str | None]], list[tuple[str, list[tuple[str, str]]]]]], + list[tuple[str, bool, str | None, str | None]], + list[tuple[str, str | None, str | None]], + list[list[tuple[str, str, str | None]]], + list[tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]], + ], + /, +) -> tuple[ + tuple[ + list[list[tuple[str, str]]], + list[tuple[str | None, str | None] | None], + list[tuple[str | None, str | None, str | None, str | None]], + list[list[tuple[str, str]]], + list[list[tuple[str, str]]], + list[tuple[str, str]], + ] + | None, + str | None, +]: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py new file mode 100644 index 0000000000..260ee5f815 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py @@ -0,0 +1,65 @@ +"""Model placeholders and the model and hook SQL Python's assembly validates.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile._helpers.analysis.validation import hook_sql_statements +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import CompileModelInput + +_HOOK_NAMES: tuple[str, ...] = ("pre_hooks", "post_hooks") + + +def model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: + """Return the model's `@@@name` placeholder defaults as strings, or None.""" + + raw_placeholders: object | None = model_input.config.values.get("placeholders") + return ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + + +def model_syntax_checks( + *, + model_inputs: tuple[CompileModelInput, ...], + analysis_model_names: frozenset[str] | None, + analysis_succeeded: frozenset[str], +) -> tuple[tuple[tuple[str, dict[str, str] | None], ...], ...]: + """Return each model's `(sql, placeholders)` that assembly would pass to syntax validation.""" + + return tuple( + _syntax_checks( + model_input=model_input, + analysis_model_names=analysis_model_names, + analysis_succeeded=analysis_succeeded, + ) + for model_input in model_inputs + ) + + +def _syntax_checks( + *, + model_input: CompileModelInput, + analysis_model_names: frozenset[str] | None, + analysis_succeeded: frozenset[str], +) -> tuple[tuple[str, dict[str, str] | None], ...]: + model_name: str = model_input.model_file.file_path.stem + if not model_input.sql_validation_enabled or ( + analysis_model_names is not None and model_name not in analysis_model_names + ): + return () + placeholders: dict[str, str] | None = model_placeholders(model_input) + statements: list[str] = [] + for hook_name in _HOOK_NAMES: + statements.extend(hook_sql_statements(model_input.config.values.get(hook_name))) + if model_name not in analysis_succeeded: + statements.append( + cursor_intrinsics_analysis_sql( + sql=model_input.query_sql, + cursor_type=model_input.config.values.get("cursor_type"), + ) + ) + return tuple((statement, placeholders) for statement in statements) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py index dbb1e9d2b0..8ef57f338b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py @@ -144,6 +144,20 @@ def validate_hook_sql_syntax( ) +def hook_sql_statements(value: object) -> tuple[str, ...]: + """Return the hook SQL strings `validate_hook_sql_syntax` validates, in its order.""" + + if isinstance(value, str): + return (value,) + if isinstance(value, SqlHookEntry): + return hook_sql_statements(value.statement) + statements: list[str] = [] + if isinstance(value, list | tuple): + for item in value: + statements.extend(hook_sql_statements(item)) + return tuple(statements) + + def validate_source_expression_syntax( *, expression: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index b4360ccdd3..87bf84e5ab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -34,6 +34,9 @@ from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( analyze_dynamic_column_contract, ) +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( + model_placeholders as _model_placeholders, +) from sqlbuild.compiler.compile._helpers.analysis.validation import ( validate_hook_sql_syntax, validate_sql_syntax, @@ -81,9 +84,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( analyze_model_sql_by_engine, - assemble_project_by_engine, dynamic_column_contract_by_engine, expression_source_shapes_by_engine, + project_resources_by_engine, ) from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( assemble_sql_tests_by_engine, @@ -163,6 +166,7 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main.record import record_compile_timing +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.models import ScopeIndex @@ -215,16 +219,6 @@ def assemble_compiled_project( ) -> CompiledProject: """Convert attached compile inputs into the planner-ready project view.""" - native_project: CompiledProject | None = assemble_project_by_engine( - inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, - ) - if native_project is not None: - return native_project sql_analysis_enabled: bool = ( inputs.effective_settings.sql_analysis and not skip_column_inference ) @@ -360,6 +354,16 @@ def assemble_compiled_project( python_analysis=python_analysis, dynamic_families_by_table=dynamic_families_by_table, ) + native: NativeProjectResources | None = project_resources_by_engine( + inputs=inputs, + dialect=profile.sql_analysis_dialect, + analysis_model_names=analysis_model_names, + analysis_succeeded=frozenset( + name + for name, analysis in model_sql_analysis_by_name.items() + if analysis.polyglot_analysis.analysis_succeeded + ), + ) scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -412,7 +416,7 @@ def assemble_compiled_project( sql_validation_enabled=( analysis_model_names is None or _model_name(model_input) in analysis_model_names ), - seed_names=seed_names, + native_deps=native.model_deps[index] if native else None, column_nullability_by_table=column_nullability_by_table, column_types_by_table=column_types_by_table, dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, @@ -420,15 +424,16 @@ def assemble_compiled_project( sql_analysis=model_sql_analysis_by_name.get(model_input.model_file.file_path.stem), allow_compact_analysis=allow_compact_analysis, ) - for model_input in inputs.model_inputs + for index, model_input in enumerate(inputs.model_inputs) ), sources=tuple( _assemble_compiled_source( source_input=source_input, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, + native_entry=native.source_entries[index] if native else None, ) - for source_input in inputs.source_inputs + for index, source_input in enumerate(inputs.source_inputs) ), seeds=tuple( _assemble_compiled_seed( @@ -436,14 +441,25 @@ def assemble_compiled_project( defaults=inputs.project_config.defaults, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, + native_destination=native.seed_destinations[index] if native else None, ) - for seed_input in inputs.seed_inputs + for index, seed_input in enumerate(inputs.seed_inputs) ), functions=tuple( - _assemble_compiled_function(function_input=function_input, seed_names=seed_names) - for function_input in inputs.sql_function_inputs + _assemble_compiled_function( + function_input=function_input, + seed_names=seed_names, + native_deps=native.function_deps[index] if native else None, + ) + for index, function_input in enumerate(inputs.sql_function_inputs) + ), + audits=tuple( + _assemble_compiled_audit( + audit_input=audit_input, + native_scope_deps=native.audit_scope_deps[index] if native else None, + ) + for index, audit_input in enumerate(inputs.audit_inputs) ), - audits=tuple(_assemble_compiled_audit(audit_input) for audit_input in inputs.audit_inputs), sql_tests=assemble_sql_tests_by_engine( inputs=inputs, assemble_python_test=lambda test_input: _assemble_compiled_sql_test( @@ -503,7 +519,7 @@ def _assemble_compiled_model( model_input: CompileModelInput, sql_analysis_enabled: bool, sql_validation_enabled: bool = True, - seed_names: frozenset[str] = frozenset(), + native_deps: tuple[CompiledObjectKey, ...] | None = None, column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, column_types_by_table: dict[str, dict[str, str]] | None = None, dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, @@ -512,6 +528,7 @@ def _assemble_compiled_model( allow_compact_analysis: bool = False, ) -> CompiledModel: model_name: str = model_input.model_file.file_path.stem + syntax_validated: bool = native_deps is not None profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() analysis_query_sql: str = cursor_intrinsics_analysis_sql( sql=model_input.query_sql, @@ -524,7 +541,7 @@ def _assemble_compiled_model( placeholders: dict[str, str] | None = ( sql_analysis.placeholders if sql_analysis is not None else _model_placeholders(model_input) ) - if sql_validation_enabled and model_input.sql_validation_enabled: + if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: for hook_name in ("pre_hooks", "post_hooks"): validate_hook_sql_syntax( value=model_input.config.values.get(hook_name), @@ -554,7 +571,7 @@ def _assemble_compiled_model( fast_lineage_has_star = polyglot_analysis.has_star fast_lineage_star_resolved = polyglot_analysis.star_resolved else: - if model_input.sql_validation_enabled: + if model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, model_name=model_name, @@ -568,7 +585,7 @@ def _assemble_compiled_model( column_nullability_by_table=column_nullability_by_table, inference_profile=profile, ) - elif sql_validation_enabled and model_input.sql_validation_enabled: + elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, model_name=model_name, @@ -613,7 +630,11 @@ def _assemble_compiled_model( fast_lineage_has_star = True return CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=model_name), - deps=model_build_deps(references=model_input.references, seed_names=seed_names), + deps=( + native_deps + if native_deps is not None + else model_build_deps(references=model_input.references) + ), name=model_name, relative_path=model_input.model_file.relative_path, query_sql=model_input.query_sql, @@ -1367,15 +1388,6 @@ def _model_sql_analysis_request( ) -def _model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: - raw_placeholders: object | None = model_input.config.values.get("placeholders") - return ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - - def _should_recover_cte_facts(model_input: CompileModelInput) -> bool: return model_input.config.values.get("contract") == ContractPolicy.ENFORCED or ( model_input.schema_entry is not None and bool(model_input.schema_entry.type_enforcement) @@ -1496,11 +1508,16 @@ def _assemble_compiled_source( source_input: CompileSourceInput, target_config: TargetConfig | None, effective_vars: dict[str, object], + native_entry: SourceEntry | None = None, ) -> CompiledSource: - source_entry: SourceEntry = _build_source_relation_entry( - source_entry=source_input.source_entry, - target_config=target_config, - effective_vars=effective_vars, + source_entry: SourceEntry = ( + native_entry + if native_entry is not None + else _build_source_relation_entry( + source_entry=source_input.source_entry, + target_config=target_config, + effective_vars=effective_vars, + ) ) return CompiledSource( key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=source_entry.name), @@ -1572,12 +1589,17 @@ def _assemble_compiled_seed( defaults: DefaultsConfig, target_config: TargetConfig | None, effective_vars: dict[str, object], + native_destination: CompiledRelationLocation | None = None, ) -> CompiledSeed: - target: CompiledRelationLocation = build_seed_relation_target( - seed_entry=seed_input.schema_entry, - defaults=defaults, - target_config=target_config, - effective_vars=effective_vars, + target: CompiledRelationLocation = ( + native_destination + if native_destination is not None + else build_seed_relation_target( + seed_entry=seed_input.schema_entry, + defaults=defaults, + target_config=target_config, + effective_vars=effective_vars, + ) ) return CompiledSeed( key=CompiledObjectKey( @@ -1596,6 +1618,7 @@ def _assemble_compiled_function( *, function_input: CompileSqlFunctionInput, seed_names: frozenset[str] = frozenset(), + native_deps: tuple[CompiledObjectKey, ...] | None = None, ) -> CompiledFunction: return CompiledFunction( key=CompiledObjectKey( @@ -1604,7 +1627,11 @@ def _assemble_compiled_function( ), name=function_input.name, ), - deps=function_build_deps(references=function_input.references, seed_names=seed_names), + deps=( + native_deps + if native_deps is not None + else function_build_deps(references=function_input.references, seed_names=seed_names) + ), name=function_input.name, relative_path=function_input.function_file.relative_path, arguments=function_input.arguments, @@ -1638,17 +1665,25 @@ def _assemble_compiled_function( ) -def _assemble_compiled_audit(audit_input: CompileAuditInput) -> CompiledAudit: +def _assemble_compiled_audit( + *, + audit_input: CompileAuditInput, + native_scope_deps: tuple[CompiledObjectKey, ...] | None = None, +) -> CompiledAudit: audit_name: str = _resolve_audit_name(audit_input) normalized_target_kind: AttachedAuditTargetKind | None = None if audit_input.attached_target_kind is not None: normalized_target_kind = AttachedAuditTargetKind(audit_input.attached_target_kind) return CompiledAudit( key=CompiledObjectKey(resource_type=CompiledResourceType.AUDIT, name=audit_name), - scope_deps=audit_scope_deps( - references=audit_input.references, - attached_target_kind=audit_input.attached_target_kind, - attached_target_name=audit_input.attached_target_name, + scope_deps=( + native_scope_deps + if native_scope_deps is not None + else audit_scope_deps( + references=audit_input.references, + attached_target_kind=audit_input.attached_target_kind, + attached_target_name=audit_input.attached_target_name, + ) ), name=audit_name, definition_name=(audit_input.audit_block.name or audit_input.audit_file.file_path.stem), diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 145745939d..2c1f1bc21d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -3,7 +3,6 @@ from __future__ import annotations from functools import partial -from pathlib import Path from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( @@ -16,44 +15,43 @@ prove_native_dynamic_contract, ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_syntax_checks from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, ) from sqlbuild.compiler.compile.models import ( - CompiledProject, CompileProjectInputs, DynamicColumnContractProof, ModelSqlAnalysis, ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.project_assembly.main._assemble_native_project import ( - assemble_native_project, +from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( + assemble_native_project_resources, ) +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily -def assemble_project_by_engine( +def project_resources_by_engine( *, inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile | None, - skip_column_inference: bool, - column_lineage_mode: ColumnLineageMode, - analysis_cache_dir: Path | None, + dialect: str | None, analysis_model_names: frozenset[str] | None, -) -> CompiledProject | None: - """Return the natively assembled project, or None where Python must assemble it.""" + analysis_succeeded: frozenset[str], +) -> NativeProjectResources | None: + """Return the natively assembled resource facts, or None where Python must assemble them.""" if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): return None - return assemble_native_project( + return assemble_native_project_resources( inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, + dialect=dialect, + syntax_checks=model_syntax_checks( + model_inputs=inputs.model_inputs, + analysis_model_names=analysis_model_names, + analysis_succeeded=analysis_succeeded, + ), ) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py new file mode 100644 index 0000000000..2a1bd2a676 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native project assembly helpers.""" diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py b/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py new file mode 100644 index 0000000000..296c742539 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py @@ -0,0 +1,24 @@ +"""Debug-only record of the project assembly work native hands back to Python.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from sqlbuild.compiler.lineage.constants import ANALYSIS_DEFERRAL_RECORD_PREFIX +from sqlbuild.compiler.project_assembly.constants import PROJECT_ASSEMBLY_DEFERRAL_SITE +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR + + +def record_project_assembly_deferral(*, kind: str) -> None: + """Append one `{"kind", "site"}` line when the analysis record directory is set.""" + + directory: str | None = os.environ.get(ANALYSIS_RECORD_DIR_ENV_VAR) + if not directory: + return + target: Path = Path(directory) + target.mkdir(parents=True, exist_ok=True) + record: str = json.dumps({"kind": kind, "site": PROJECT_ASSEMBLY_DEFERRAL_SITE}) + with open(target / f"{ANALYSIS_DEFERRAL_RECORD_PREFIX}{os.getpid()}.jsonl", "a") as handle: + _ = handle.write(record + "\n") diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py new file mode 100644 index 0000000000..00ea349bdc --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/request.py @@ -0,0 +1,116 @@ +"""Build the native project assembly request from Python's compile inputs.""" + +from __future__ import annotations + +import os + +from sqlbuild.compiler.compile.models import CompileProjectInputs, CompileSqlReference +from sqlbuild.compiler.project_assembly.constants import ( + BOOLEAN_VARIABLE, + FALSE_TEXT, + NATIVE_INT_MAX, + NATIVE_INT_MIN, + NONE_VARIABLE, + TEXT_VARIABLE, + TRUE_TEXT, + UNSUPPORTED_VARIABLE, +) +from sqlbuild.compiler.project_assembly.types import ( + AuditRow, + ProjectRequestRow, + ReferenceRow, + SyntaxCheckRow, + VariableRow, +) +from sqlbuild.spec.contracts.models import DefaultsConfig, TargetConfig + + +def project_request( + *, + inputs: CompileProjectInputs, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> ProjectRequestRow: + """Return the request row; `syntax_checks` holds each model's SQL Python would validate.""" + + target: TargetConfig | None = inputs.effective_target + defaults: DefaultsConfig = inputs.project_config.defaults + return ( + dialect or "generic", + (target.database, target.schema, target.loader_schema) if target is not None else None, + (defaults.database, defaults.schema, defaults.seed_database, defaults.seed_schema), + [_variable_row(name=name, value=value) for name, value in inputs.effective_vars.items()], + list(os.environ.items()), + [ + (_reference_rows(model_input.references), _syntax_check_rows(checks)) + for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) + ], + [ + ( + source_input.source_entry.name, + source_input.source_entry.loader is not None, + source_input.source_entry.database, + source_input.source_entry.schema, + ) + for source_input in inputs.source_inputs + ], + [ + ( + seed_input.schema_entry.name, + seed_input.schema_entry.database, + seed_input.schema_entry.schema, + ) + for seed_input in inputs.seed_inputs + ], + [ + _reference_rows(function_input.references) + for function_input in inputs.sql_function_inputs + ], + [ + _audit_row( + references=audit_input.references, + attached_kind=audit_input.attached_target_kind, + attached_name=audit_input.attached_target_name, + ) + for audit_input in inputs.audit_inputs + ], + ) + + +def _variable_row(*, name: str, value: object) -> VariableRow: + if value is None: + return (name, NONE_VARIABLE, "") + if type(value) is bool: + return (name, BOOLEAN_VARIABLE, TRUE_TEXT if value else FALSE_TEXT) + if type(value) is str: + return (name, TEXT_VARIABLE, value) + if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: + return (name, TEXT_VARIABLE, str(value)) + return (name, UNSUPPORTED_VARIABLE, "") + + +def _reference_rows(references: tuple[CompileSqlReference, ...]) -> list[ReferenceRow]: + return [ + (str(reference.ref_kind), reference.ref_name, reference.ref_package) + for reference in references + ] + + +def _syntax_check_rows( + checks: tuple[tuple[str, dict[str, str] | None], ...], +) -> list[SyntaxCheckRow]: + return [(sql, list((placeholders or {}).items())) for sql, placeholders in checks] + + +def _audit_row( + *, + references: tuple[CompileSqlReference, ...], + attached_kind: object, + attached_name: str | None, +) -> AuditRow: + attached: tuple[str, str] | None = ( + (str(attached_kind), attached_name) + if attached_kind is not None and attached_name is not None + else None + ) + return (_reference_rows(references), attached) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py new file mode 100644 index 0000000000..4475507e1e --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py @@ -0,0 +1,81 @@ +"""Materialise the native project assembly result as the objects Python's assembly builds.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, + CompileProjectInputs, + CompileSeedInput, + CompileSourceInput, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.project_assembly.constants import ENVIRONMENT_READ +from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.types import ( + NamespaceRow, + ObjectKeyRow, + ProjectResourcesRow, +) +from sqlbuild.spec.contracts.models import SourceEntry + + +def project_resources( + *, inputs: CompileProjectInputs, row: ProjectResourcesRow +) -> NativeProjectResources: + """Return the resource facts the native row describes, in input order.""" + + model_deps, source_namespaces, seed_namespaces, function_deps, audit_deps, reads = row + _ = [_replay_read(kind=kind, name=name) for kind, name in reads] + return NativeProjectResources( + model_deps=tuple(_keys(keys) for keys in model_deps), + source_entries=tuple( + _source_entry(source_input=source_input, namespace=namespace) + for source_input, namespace in zip(inputs.source_inputs, source_namespaces, strict=True) + ), + seed_destinations=tuple( + _seed_destination(seed_input=seed_input, namespace=namespace) + for seed_input, namespace in zip(inputs.seed_inputs, seed_namespaces, strict=True) + ), + function_deps=tuple(_keys(keys) for keys in function_deps), + audit_scope_deps=tuple(_keys(keys) for keys in audit_deps), + ) + + +def _replay_read(*, kind: str, name: str) -> None: + if kind == ENVIRONMENT_READ: + COMPILE_INPUT_READS.environment_read(name) + else: + COMPILE_INPUT_READS.context_read(name) + + +def _keys(rows: list[ObjectKeyRow]) -> tuple[CompiledObjectKey, ...]: + return tuple( + CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) + for kind, name in rows + ) + + +def _source_entry( + *, source_input: CompileSourceInput, namespace: tuple[str | None, str | None] | None +) -> SourceEntry: + if namespace is None: + return source_input.source_entry + return replace(source_input.source_entry, database=namespace[0], schema=namespace[1]) + + +def _seed_destination( + *, seed_input: CompileSeedInput, namespace: NamespaceRow +) -> CompiledRelationLocation: + database, schema, logical_database, logical_schema = namespace + return CompiledRelationLocation( + database=database, + schema=schema, + name=seed_input.schema_entry.name, + qualified_name=None, + logical_schema=logical_schema, + logical_database=logical_database, + ) diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py new file mode 100644 index 0000000000..e3f7a86326 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/constants.py @@ -0,0 +1,12 @@ +"""Native project assembly request kinds and deferral record site.""" + +PROJECT_ASSEMBLY_DEFERRAL_SITE: str = "project_assembly" +TEXT_VARIABLE: str = "text" +BOOLEAN_VARIABLE: str = "bool" +NONE_VARIABLE: str = "none" +UNSUPPORTED_VARIABLE: str = "unsupported" +TRUE_TEXT: str = "1" +FALSE_TEXT: str = "0" +NATIVE_INT_MIN: int = -(2**63) +NATIVE_INT_MAX: int = 2**63 - 1 +ENVIRONMENT_READ: str = "env" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py deleted file mode 100644 index 74ba3e90fb..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Assemble the compiled project natively for the preview compiler engine.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs -from sqlbuild.compiler.lineage.types import ColumnLineageMode - - -def assemble_native_project( - *, - inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile | None, - skip_column_inference: bool, - column_lineage_mode: ColumnLineageMode, - analysis_cache_dir: Path | None, - analysis_model_names: frozenset[str] | None, -) -> CompiledProject | None: - """Return the compiled project, or None where Python must assemble it.""" - - return None diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py new file mode 100644 index 0000000000..fd3b5d63d0 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py @@ -0,0 +1,30 @@ +"""Assemble the compiled project's resource facts natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.project_assembly._helpers.deferrals import record_project_assembly_deferral +from sqlbuild.compiler.project_assembly._helpers.request import project_request +from sqlbuild.compiler.project_assembly._helpers.resources import project_resources +from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.types import ProjectResourcesRow + + +def assemble_native_project_resources( + *, + inputs: CompileProjectInputs, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> NativeProjectResources | None: + """Return the resource facts, or None with a deferral record where Python must assemble.""" + + row: ProjectResourcesRow | None + deferral: str | None + row, deferral = _native.assemble_project_resource_facts( + project_request(inputs=inputs, dialect=dialect, syntax_checks=syntax_checks) + ) + if row is None: + _ = record_project_assembly_deferral(kind=deferral or "unknown") + return None + return project_resources(inputs=inputs, row=row) diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py new file mode 100644 index 0000000000..8c6631c2cf --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -0,0 +1,19 @@ +"""Resource facts the native project assembly returns, as Python's assembly builds them.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.spec.contracts.models import SourceEntry + + +@dataclass(frozen=True, slots=True) +class NativeProjectResources: + """Per-resource facts in input order; every requested SQL string passed syntax validation.""" + + model_deps: tuple[tuple[CompiledObjectKey, ...], ...] + source_entries: tuple[SourceEntry, ...] + seed_destinations: tuple[CompiledRelationLocation, ...] + function_deps: tuple[tuple[CompiledObjectKey, ...], ...] + audit_scope_deps: tuple[tuple[CompiledObjectKey, ...], ...] diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py new file mode 100644 index 0000000000..a54e5884c6 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/types.py @@ -0,0 +1,35 @@ +"""Plain-data rows exchanged with the native project assembly.""" + +from __future__ import annotations + +type ReferenceRow = tuple[str, str, str | None] +type SyntaxCheckRow = tuple[str, list[tuple[str, str]]] +type VariableRow = tuple[str, str, str] +type ModelRow = tuple[list[ReferenceRow], list[SyntaxCheckRow]] +type SourceRow = tuple[str, bool, str | None, str | None] +type SeedRow = tuple[str, str | None, str | None] +type AuditRow = tuple[list[ReferenceRow], tuple[str, str] | None] +type TargetRow = tuple[str | None, str | None, str | None] +type DefaultsRow = tuple[str | None, str | None, str | None, str | None] +type ProjectRequestRow = tuple[ + str, + TargetRow | None, + DefaultsRow, + list[VariableRow], + list[tuple[str, str]], + list[ModelRow], + list[SourceRow], + list[SeedRow], + list[list[ReferenceRow]], + list[AuditRow], +] +type ObjectKeyRow = tuple[str, str] +type NamespaceRow = tuple[str | None, str | None, str | None, str | None] +type ProjectResourcesRow = tuple[ + list[list[ObjectKeyRow]], + list[tuple[str | None, str | None] | None], + list[NamespaceRow], + list[list[ObjectKeyRow]], + list[list[ObjectKeyRow]], + list[tuple[str, str]], +] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 0d3a7c4dc8..c66fd2bc8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4882,7 +4882,7 @@ def engine_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_stages, "assemble_native_project"), + (native_stages, "assemble_native_project_resources"), (native_stages, "infer_native_expression_source_shapes"), (native_stages, "analyze_native_model_sql"), (native_sql_test_stage, "assemble_native_sql_tests"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index cf88cb5277..bd27e5c2d5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, ) @@ -32,7 +33,7 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_preview_returns={ - "assemble_native_project": [None], + "assemble_native_project_resources": [ANY], "infer_native_expression_source_shapes": [ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [None], @@ -79,6 +80,7 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ test_case.expected_preview_returns, ) assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) + assert isinstance(preview_seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/__init__.py b/tests/integration/src/sqlbuild/compiler/project_assembly/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py new file mode 100644 index 0000000000..913c32b186 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedAssemblyParityTestCase: + """Seeded generated projects whose resource assembly both engines must agree on.""" + + description: str + seed: int + count: int + model_count: int + environment: dict[str, str] + expected_minimum_python_calls: int + expected_minimum_environment_reads: int + + +@dataclass(frozen=True) +class AssemblyDeferralTestCase: + """A project native hands back to Python, recording why, with Python's exact outcome.""" + + description: str + files: dict[str, str] + expected_kind: str + expected_error: str + + +@dataclass(frozen=True) +class DeferredAssemblyTestCase: + """A project native hands back to Python, which assembles it without error.""" + + description: str + effective_vars: dict[str, object] + seed_schema: str + expected_schema: str + expected_kind: str diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py new file mode 100644 index 0000000000..d37d832b5e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -0,0 +1,228 @@ +"""Generated projects and engine views for native project assembly parity.""" + +from __future__ import annotations + +import random +from collections import Counter +from collections.abc import Callable +from operator import methodcaller +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.assembly import project as project_module +from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, + CompiledProject, + CompileProjectInputs, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import deferral_kinds + +_PYTHON_RESOURCE_FUNCTIONS: tuple[str, ...] = ( + "_build_source_relation_entry", + "build_seed_relation_target", + "validate_sql_syntax", + "validate_hook_sql_syntax", + "model_build_deps", + "function_build_deps", + "audit_scope_deps", +) +_ASSEMBLY_RECORD_PREFIX: str = "project_assembly:" +_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, +) +_DEFAULTS: tuple[str, ...] = ( + "", + 'seed_schema = "raw"\n', + 'database = "db_${env_name}"\nschema = "main"\n', + 'seed_database = "${n}"\nseed_schema = "seeds_${env_name}"\n', +) +_TARGETS: tuple[str, ...] = ( + 'schema = "dev_${env_name}"\nloader_schema = "loaders"\n', + 'database = "wh_${n}"\nschema = "main"\n', + 'schema = "dev"\n', + 'database = "wh"\nschema = "analytics"\nloader_schema = "raw_${flag}"\n', + "schema = \"${coalesce(ENV:SQB_ASSEMBLY_UNSET, 'main')}\"\n" + 'loader_schema = "${ENV:SQB_ASSEMBLY_LOADERS}"\n', +) +_SEED_NAMESPACES: tuple[str, ...] = ( + "", + " schema: seeds_${env_name}\n", + ' schema: "${CTX:model.name}_s"\n', + ' database: "${n}"\n schema: "${CTX:model.database}_x"\n', + ' schema: "${flag}"\n', + ' schema: "${ENV:SQB_ASSEMBLY_LOADERS}_${CTX:model.alias}"\n', +) +_SOURCE_NAMESPACES: tuple[str, ...] = ("", " schema: landing\n", " database: lake\n") +_HOOKS: tuple[str, ...] = ( + "", + ' pre_hooks [inline_sql("SELECT 0 AS n")],\n', + " post_hooks [inline_sql('SELECT 1'), inline_sql('SELECT 2')],\n", + " pre_hooks [inline_sql('SELECT 3')],\n post_hooks [inline_sql('SELECT 4')],\n", +) +_AUDITS: tuple[str, ...] = ("", " columns (n (audits [not_null])),\n") + + +def generated_assembly_files(*, rng: random.Random, model_count: int) -> dict[str, str]: + """Return a project with managed sources, templated seeds, hooks, audits and a function.""" + + references: list[str] = [ + '__source("raw_orders")', + '__source("landing_orders")', + '__seed("countries")', + '__seed("regions")', + *[f'__ref("orders_{index}")' for index in range(model_count - 1)], + ] + models: dict[str, str] = { + f"models/orders_{index}.sql": ( + f'MODEL (\n description "Generated orders model {index}",\n' + f"{rng.choice(_HOOKS)}{rng.choice(_AUDITS)});\n\n" + f"SELECT 1 AS n FROM {rng.choice(references[: 4 + index])}\n" + ) + for index in range(model_count) + } + return { + "sqlbuild_project.toml": ( + 'name = "assembly"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n' + '[vars]\nenv_name = "prod"\nn = "3"\nflag = "true"\n\n' + f"[defaults]\n{rng.choice(_DEFAULTS)}\n[targets.dev]\n{rng.choice(_TARGETS)}" + ), + "seeds/lookups.yml": ( + "seeds:\n" + f" - name: countries\n description: Countries.\n{rng.choice(_SEED_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + f" - name: regions\n description: Regions.\n{rng.choice(_SEED_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + ), + "seeds/countries.csv": "n\n1\n", + "seeds/regions.csv": "n\n2\n", + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Raw orders.\n managed: true\n" + f" write_strategy: table\n{rng.choice(_SOURCE_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + " - name: landing_orders\n description: Landing orders.\n" + f"{rng.choice(_SOURCE_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + ), + "python/loaders/raw.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\ndef raw_orders(ctx):\n '''Load raw orders.'''\n return [{'n': 1}]\n" + ), + "functions/sql/order_numbers.sql": ( + 'FUNCTION (\n description "Order numbers.",\n arguments (p_n INTEGER),\n' + " returns table (n INTEGER)\n);\n\n" + 'SELECT n FROM __ref("orders_0") WHERE n = p_n\n' + ), + **models, + } + + +def seed_yml(schema: str) -> str: + """Return a one-seed declaration whose schema is `schema`.""" + + return ( + "seeds:\n - name: countries\n description: Countries.\n" + f' schema: "{schema}"\n columns:\n - name: n\n type: INTEGER\n' + ) + + +def project_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileProjectInputs: + """Write `files` and attach their compile inputs as the compile pipeline does.""" + + _ = [ + _write(path=project_dir / relative_path, contents=contents) + for relative_path, contents in files.items() + ] + return build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=_ADAPTER_CONTEXT, + run_id="integration_run", + defer_model_sql_validation=True, + ) + + +def assemble_with( + *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> CompiledProject: + """Assemble `inputs` with `engine`.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return assemble_compiled_project(inputs=inputs) + + +def recorded_assembly( + *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> tuple[object, tuple[tuple[str, ...], bool]]: + """Assemble `inputs` with `engine`; return its assembly view and the input reads recorded.""" + + with COMPILE_INPUT_READS.recording() as reads: + project: CompiledProject = assemble_with( + inputs=inputs, engine=engine, monkeypatch=monkeypatch + ) + return assembly_view(project), (reads.environment_names, reads.read_run_id) + + +def assembly_view(project: CompiledProject) -> object: + """Return every resource fact native project assembly derives, with the diagnostics.""" + + return ( + tuple((model.name, model.deps, model.destination) for model in project.models), + tuple(source.source_entry for source in project.sources), + tuple(seed.destination for seed in project.seeds), + tuple((function.name, function.deps) for function in project.functions), + tuple((audit.name, audit.scope_deps) for audit in project.audits), + project.diagnostics, + ) + + +def assembly_deferrals(record_dir: Path) -> Counter[str]: + """Count the project assembly `site:kind` deferral records below `record_dir`.""" + + return Counter( + filter( + methodcaller("startswith", _ASSEMBLY_RECORD_PREFIX), + deferral_kinds(record_dir).elements(), + ) + ) + + +def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count calls to Python's resource-fact functions from project assembly.""" + + calls: Counter[str] = Counter() + _ = [ + monkeypatch.setattr( + project_module, + name, + _counted(name=name, function=getattr(project_module, name), calls=calls), + ) + for name in _PYTHON_RESOURCE_FUNCTIONS + ] + return calls + + +def _write(*, path: Path, contents: str) -> int: + path.parent.mkdir(parents=True, exist_ok=True) + return path.write_text(contents, encoding="utf-8") + + +def _counted(*, name: str, function: Callable[..., Any], calls: Counter[str]) -> Callable[..., Any]: + def counted(*args: Any, **kwargs: Any) -> Any: + calls[name] += 1 + return function(*args, **kwargs) + + return counted diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py new file mode 100644 index 0000000000..a37f1943ed --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -0,0 +1,221 @@ +"""Native project assembly equals Python's resource assembly, or hands the project back.""" + +from __future__ import annotations + +import random +from collections import Counter +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.project_assembly._test_types import ( + AssemblyDeferralTestCase, + DeferredAssemblyTestCase, + GeneratedAssemblyParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( + assemble_with, + assembly_deferrals, + assembly_view, + generated_assembly_files, + project_inputs, + python_resource_calls, + recorded_assembly, + seed_yml, +) + +_PROJECT_TOML: str = ( + 'name = "assembly"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n\n' +) +_ORDERS_MODEL: str = 'MODEL (description "Orders");\n\nSELECT 1 AS n\n' +_SEED_FILES: dict[str, str] = { + "seeds/countries.csv": "n\n1\n", + "models/orders.sql": _ORDERS_MODEL, +} + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedAssemblyParityTestCase( + description="managed sources, templated seeds and targets, hooks, audits, a function", + seed=20261009, + count=8, + model_count=12, + environment={"SQB_ASSEMBLY_LOADERS": "loaders"}, + expected_minimum_python_calls=200, + expected_minimum_environment_reads=4, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_projects_when_assembling_natively_then_matches_python( + test_case: GeneratedAssemblyParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + _ = [monkeypatch.setenv(name, value) for name, value in test_case.environment.items()] + calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) + rng: random.Random = random.Random(test_case.seed) + inputs: list[CompileProjectInputs] = [ + project_inputs( + project_dir=tmp_path / f"project_{index}", + files=generated_assembly_files(rng=rng, model_count=test_case.model_count), + ) + for index in range(test_case.count) + ] + python: list[tuple[object, tuple[tuple[str, ...], bool]]] = [ + recorded_assembly(inputs=item, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + for item in inputs + ] + python_calls: int = calls.total() + native: list[tuple[object, tuple[tuple[str, ...], bool]]] = [ + recorded_assembly( + inputs=item, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + for item in inputs + ] + + assert ( + mismatches( + inputs=list(range(test_case.count)), + expected=list[object](python), + actual=list[object](native), + ) + == [] + ) + assert sum(len(reads[0]) for _, reads in native) >= test_case.expected_minimum_environment_reads + assert python_calls >= test_case.expected_minimum_python_calls + assert calls.total() == python_calls + assert assembly_deferrals(record_dir) == Counter() + + +@pytest.mark.parametrize( + "test_case", + [ + AssemblyDeferralTestCase( + description="model SQL the parser rejects", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": 'MODEL (description "Orders");\n\nSELECT FROM WHERE (\n', + }, + expected_kind="syntax_error", + expected_error="SQL syntax error in model 'orders'", + ), + AssemblyDeferralTestCase( + description="hook SQL the parser rejects", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + 'MODEL (\n description "Orders",\n pre_hooks [inline_sql("SELECT FROM (")],\n' + ");\n\nSELECT 1 AS n\n" + ), + }, + expected_kind="syntax_error", + expected_error="Polyglot could not parse model 'orders' pre_hooks[0]", + ), + AssemblyDeferralTestCase( + description="a seed schema reading an unset environment variable", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml("${ENV:SQB_UNSET_SEED_SCHEMA}"), + **_SEED_FILES, + }, + expected_kind="target_template", + expected_error="references missing ENV variable 'SQB_UNSET_SEED_SCHEMA'", + ), + AssemblyDeferralTestCase( + description="a preserved target schema the seed does not own", + files={ + "sqlbuild_project.toml": ( + 'name = "assembly"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "preserve"\n' + ), + "seeds/lookups.yml": ( + "seeds:\n - name: countries\n description: Countries.\n" + " columns:\n - name: n\n type: INTEGER\n" + ), + "seeds/countries.csv": "n\n1\n", + "models/orders.sql": ( + 'MODEL (description "Orders", schema "orders");\n\nSELECT 1 AS n\n' + ), + }, + expected_kind="preserved_namespace", + expected_error="Seed 'countries' has no logical schema", + ), + ], + ids=lambda case: case.description, +) +def test_given_projects_python_rejects_when_assembling_natively_then_defers_with_python_error( + test_case: AssemblyDeferralTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = project_inputs(project_dir=tmp_path, files=test_case.files) + + with pytest.raises(CompileInputError) as python_error: + _ = assemble_with(inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + with pytest.raises(CompileInputError) as native_error: + _ = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert test_case.expected_error in str(python_error.value) + assert str(native_error.value) == str(python_error.value) + assert assembly_deferrals(record_dir) == Counter( + {f"project_assembly:{test_case.expected_kind}": 1} + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeferredAssemblyTestCase( + description="a seed schema reading a list variable", + effective_vars={"tags": ["daily"]}, + seed_schema="${tags}", + expected_schema="['daily']", + expected_kind="target_template", + ) + ], + ids=lambda case: case.description, +) +def test_given_a_variable_only_python_renders_when_assembling_natively_then_python_assembles( + test_case: DeferredAssemblyTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = replace( + project_inputs( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml(test_case.seed_schema), + **_SEED_FILES, + }, + ), + effective_vars=test_case.effective_vars, + ) + + python: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + native: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert assembly_view(native) == assembly_view(python) + assert native.seeds[0].destination.schema == test_case.expected_schema + assert assembly_deferrals(record_dir) == Counter( + {f"project_assembly:{test_case.expected_kind}": 1} + ) From 80ba17b4db1e9335f11717b7d5e4640e6590d2b5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:42:33 +0100 Subject: [PATCH 2/2] fix(native): defer project assembly for unbuilt dialects and resolve ENV names through Python The native syntax check now uses the shared parser-dialect guard, so an adapter dialect this parser build lacks defers with unsupported_dialect instead of accepting SQL Python rejects. ENV template names are resolved by Python's own os.environ lookup and sent with the request, so Windows' case-insensitive keys behave as in Python; a name the request does not carry defers. --- .../src/assembly/project/_helpers/syntax.rs | 13 +++--- .../assembly/project/_helpers/templates.rs | 10 +++-- .../src/assembly/project/constants.rs | 1 + .../src/assembly/project/models.rs | 4 +- .../src/assembly/project/tests/helpers.rs | 6 +-- .../project/tests/test_project_assembly.rs | 35 +++++++++++++--- .../src/assembly/project/tests/test_types.rs | 4 +- .../src/lineage/_helpers/parsed_lineage.rs | 19 --------- .../src/lineage/main/build_fast_lineage.rs | 5 ++- .../sqlbuild-analysis/src/lineage/main/mod.rs | 1 + .../src/lineage/main/parser_dialect.rs | 22 ++++++++++ .../_helpers/project_assembly/resources.rs | 2 +- .../_helpers/analysis_corpus/assembly.py | 38 ++++++++++++++++- src/sqlbuild/_native.pyi | 2 +- .../project_assembly/_helpers/request.py | 27 +++++++++++- .../compiler/project_assembly/constants.py | 1 + .../compiler/project_assembly/types.py | 2 +- .../compiler/project_assembly/_test_types.py | 10 +++++ .../compiler/project_assembly/helpers.py | 10 +++++ .../test_native_project_assembly.py | 42 +++++++++++++++++++ 20 files changed, 209 insertions(+), 45 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs index 179b6f4b4e..59d20a7422 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs @@ -2,15 +2,18 @@ use std::collections::HashMap; -use polyglot_sql::{ComplexityGuardOptions, Dialect, ValidationOptions, validate_with_dialect}; +use polyglot_sql::{ + ComplexityGuardOptions, Dialect, DialectType, ValidationOptions, validate_with_dialect, +}; use serde_json::json; use crate::assembly::project::constants::{ DIALECT_DEFERRAL, MAX_FUNCTION_CALL_DEPTH, NORMALIZATION_DEFERRAL, PLACEHOLDER_DEFERRAL, - PLACEHOLDER_PREFIX, + PLACEHOLDER_PREFIX, UNSUPPORTED_DIALECT_DEFERRAL, }; use crate::assembly::project::models::SyntaxCheck; use crate::assembly::project::types::Fact; +use crate::lineage::main::parser_dialect::parser_dialect; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::NormalizationInput; @@ -28,8 +31,8 @@ pub(crate) fn syntax_valid(check: &SyntaxCheck, dialect: &str) -> Fact { } else { placeholder_defaults(&cleaned, &check.placeholders)? }; - let dialect: Dialect = - Dialect::get_by_name(dialect).ok_or_else(|| DIALECT_DEFERRAL.to_owned())?; + let dialect: DialectType = + parser_dialect(Some(dialect)).ok_or_else(|| UNSUPPORTED_DIALECT_DEFERRAL.to_owned())?; let complexity_guard: ComplexityGuardOptions = serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) .map_err(|_| DIALECT_DEFERRAL.to_owned())?; @@ -38,7 +41,7 @@ pub(crate) fn syntax_valid(check: &SyntaxCheck, dialect: &str) -> Fact { strict_syntax: false, semantic: false, }; - Ok(validate_with_dialect(&cleaned, &dialect, &options).valid) + Ok(validate_with_dialect(&cleaned, &Dialect::get(dialect), &options).valid) } /// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs index 7ea87fca3a..4a5b17d1ab 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs @@ -20,7 +20,8 @@ pub(crate) type Context<'a> = [(&'static str, Option)]; /// The variables and environment target templates read, and every lookup they make in order. pub(crate) struct TemplateInputs<'a> { pub(crate) variables: &'a [(String, Variable)], - pub(crate) environment: &'a [(String, String)], + /// Python's `os.environ.get` of each name the templates may read. + pub(crate) environment: &'a [(String, Option)], pub(crate) reads: RefCell>, } @@ -51,12 +52,15 @@ impl TemplateHost for ScalarHost<'_> { .reads .borrow_mut() .push(InputRead::Environment(name.to_owned())); - Ok(self + match self .inputs .environment .iter() .find(|(variable, _)| variable == name) - .map(|(_, value)| Scalar::Text(value.clone()))) + { + None => Err(TemplateFailure::Unsupported), + Some((_, value)) => Ok(value.clone().map(Scalar::Text)), + } } fn context(&self, name: &str) -> Result, TemplateFailure> { diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index d550f366b9..2fb32b6c70 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -33,4 +33,5 @@ pub(crate) const SYNTAX_DEFERRAL: &str = "syntax_error"; pub(crate) const PLACEHOLDER_DEFERRAL: &str = "placeholder_word"; pub(crate) const NORMALIZATION_DEFERRAL: &str = "normalization"; pub(crate) const DIALECT_DEFERRAL: &str = "dialect"; +pub(crate) const UNSUPPORTED_DIALECT_DEFERRAL: &str = "unsupported_dialect"; pub(crate) const PANIC_DEFERRAL: &str = "panic"; diff --git a/crates/sqlbuild-analysis/src/assembly/project/models.rs b/crates/sqlbuild-analysis/src/assembly/project/models.rs index be6dd41e7b..92d0f34a3c 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/models.rs @@ -92,8 +92,8 @@ pub struct ProjectRequest { pub target: Option, pub defaults: SeedDefaults, pub variables: Vec<(String, Variable)>, - /// The process environment `os.environ` holds. - pub environment: Vec<(String, String)>, + /// Python's `os.environ.get` of every name an `ENV:` template in the request names. + pub environment: Vec<(String, Option)>, pub models: Vec, pub sources: Vec, pub seeds: Vec, diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs index 77f00584a1..5f245315d3 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs @@ -59,7 +59,7 @@ pub(crate) fn seed( /// The seed's namespace with `environment` set, and the reads its templates made. pub(crate) fn seed_reads( seed: SeedSpec, - environment: &[(String, String)], + environment: &[(String, Option)], ) -> (Option, Vec) { let (name, database, schema) = seed; let variables: Vec<(String, Variable)> = variables(); @@ -100,7 +100,7 @@ pub(crate) fn source( .ok() } -pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)]) -> Option { +pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)], dialect: &str) -> Option { syntax_valid( &SyntaxCheck { sql: sql.to_owned(), @@ -109,7 +109,7 @@ pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)]) -> Option { .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) .collect(), }, - "duckdb", + dialect, ) .ok() } diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs index c6b82aeb41..e7a7b02ccc 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs @@ -181,32 +181,43 @@ fn given_sources_when_resolving_managed_targets_then_matches_python() { fn given_model_sql_when_validating_syntax_then_matches_python_or_defers() { let test_cases = [ SyntaxTestCase { + dialect: "duckdb", description: "markers and placeholder defaults parse", sql: "SELECT @@@x AS a FROM __ref(\"orders\")", placeholders: &[("x", "1")], expected_valid: Some(true), }, SyntaxTestCase { + dialect: "duckdb", description: "a syntax error", sql: "SELECT FROM WHERE (", placeholders: &[], expected_valid: Some(false), }, SyntaxTestCase { + dialect: "duckdb", description: "a placeholder run retried one character later, as re.sub does", sql: "SELECT @@@@@abc AS a", placeholders: &[("abc", "version")], expected_valid: Some(true), }, SyntaxTestCase { + dialect: "duckdb", description: "a placeholder followed by text Python's \\w may extend", sql: "SELECT @@@x\u{e9} AS a", placeholders: &[("x", "1")], expected_valid: None, }, + SyntaxTestCase { + dialect: "trino", + description: "a dialect this parser build does not carry", + sql: "SELECT 1 AS a FROM t QUALIFY a = 1", + placeholders: &[], + expected_valid: None, + }, ]; for test_case in test_cases { - let checked = valid(test_case.sql, test_case.placeholders); + let checked = valid(test_case.sql, test_case.placeholders, test_case.dialect); assert_eq!( checked, test_case.expected_valid, @@ -228,7 +239,10 @@ fn given_environment_templates_when_resolving_seeds_then_matches_python_values_a "${coalesce(ENV:SQB_UNSET_SCHEMA, 'main')}_${ENV:SQB_SET_SCHEMA}_${CTX:model.name}", ), ), - environment: &[("SQB_SET_SCHEMA", "landing")], + environment: &[ + ("SQB_UNSET_SCHEMA", None), + ("SQB_SET_SCHEMA", Some("landing")), + ], expected_schema: Some("main_landing_countries"), expected_reads: vec![ InputRead::Environment("SQB_UNSET_SCHEMA".to_owned()), @@ -239,16 +253,27 @@ fn given_environment_templates_when_resolving_seeds_then_matches_python_values_a EnvironmentTestCase { description: "an unset variable Python raises for", seed: ("countries", None, Some("${ENV:SQB_UNSET_SCHEMA}")), - environment: &[], + environment: &[("SQB_UNSET_SCHEMA", None)], expected_schema: None, expected_reads: vec![InputRead::Environment("SQB_UNSET_SCHEMA".to_owned())], }, + EnvironmentTestCase { + description: "a name Python did not resolve stays with Python", + seed: ( + "countries", + None, + Some("${coalesce(ENV:app_schema, 'main')}"), + ), + environment: &[("APP_SCHEMA", Some("landing"))], + expected_schema: None, + expected_reads: vec![InputRead::Environment("app_schema".to_owned())], + }, ]; for test_case in test_cases { - let environment: Vec<(String, String)> = test_case + let environment: Vec<(String, Option)> = test_case .environment .iter() - .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .map(|(name, value)| ((*name).to_owned(), value.map(str::to_owned))) .collect(); let (resolved, reads) = seed_reads(test_case.seed, &environment); diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index 9e513a6cc9..f39fb71bfd 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -50,6 +50,7 @@ pub(crate) struct SourceTestCase { pub(crate) struct SyntaxTestCase { pub(crate) description: &'static str, + pub(crate) dialect: &'static str, pub(crate) sql: &'static str, pub(crate) placeholders: &'static [(&'static str, &'static str)], /// None where native defers to Python. @@ -66,7 +67,8 @@ pub(crate) fn keys(values: &[(&str, &str)]) -> Vec { pub(crate) struct EnvironmentTestCase { pub(crate) description: &'static str, pub(crate) seed: SeedSpec, - pub(crate) environment: &'static [(&'static str, &'static str)], + /// Python's `os.environ.get` of each name. + pub(crate) environment: &'static [(&'static str, Option<&'static str>)], /// None where Python raises. pub(crate) expected_schema: Option<&'static str>, pub(crate) expected_reads: Vec, diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs index ad66de574c..55e9557c5f 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs @@ -26,25 +26,6 @@ pub(crate) fn proxy_parse_options() -> Result { }) } -/// Python's `parse_one(dialect=dialect or "generic")` dialect, if this parser build carries it. -pub(crate) fn lineage_dialect(name: Option<&str>) -> Option { - let name = name.filter(|name| !name.is_empty()).unwrap_or("generic"); - let Ok(dialect) = name.parse::() else { - return None; - }; - matches!( - dialect, - DialectType::Generic - | DialectType::PostgreSQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::DuckDB - | DialectType::TSQL - | DialectType::Databricks - ) - .then_some(dialect) -} - pub(crate) struct ParsedModel<'a> { pub(crate) query_sql: &'a str, pub(crate) inferred_names: &'a [String], diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index f1266f2eff..4eeb8c3c90 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -8,10 +8,11 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use crate::lineage::_helpers::parsed_lineage::{ - ParsedModel, lineage_dialect, parsed_model_lineage, proxy_parse_options, + ParsedModel, parsed_model_lineage, proxy_parse_options, }; use crate::lineage::_helpers::references::physical_resources; use crate::lineage::_helpers::stars::{schema_mapping, star_lineage}; +use crate::lineage::main::parser_dialect::parser_dialect; use crate::lineage::models::{ FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageDeferral, }; @@ -29,7 +30,7 @@ pub fn build_fast_lineage( /// A parser panic defers only its own model. fn model_outcomes(request: &FastLineageRequest) -> Vec { let schema = schema_mapping(&request.schema); - let dialect = lineage_dialect(request.dialect.as_deref()); + let dialect = parser_dialect(request.dialect.as_deref()); let options: Result = proxy_parse_options(); request .models diff --git a/crates/sqlbuild-analysis/src/lineage/main/mod.rs b/crates/sqlbuild-analysis/src/lineage/main/mod.rs index f1d5cd565b..6b75bd22ea 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/mod.rs @@ -1 +1,2 @@ pub mod build_fast_lineage; +pub mod parser_dialect; diff --git a/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs b/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs new file mode 100644 index 0000000000..61a7005ee7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs @@ -0,0 +1,22 @@ +//! The polyglot dialect Python's `dialect or "generic"` names, if this parser build carries it. + +use polyglot_sql::DialectType; + +/// Python's `parse_one(dialect=dialect or "generic")` dialect, if this parser build carries it. +pub fn parser_dialect(name: Option<&str>) -> Option { + let name = name.filter(|name| !name.is_empty()).unwrap_or("generic"); + let Ok(dialect) = name.parse::() else { + return None; + }; + matches!( + dialect, + DialectType::Generic + | DialectType::PostgreSQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::DuckDB + | DialectType::TSQL + | DialectType::Databricks + ) + .then_some(dialect) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index 2774b9486b..d2fbcc1350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -47,7 +47,7 @@ type RequestRow = ( Option, DefaultsRow, Vec, - Vec<(String, String)>, + Vec<(String, Option)>, Vec, Vec, Vec, diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py index 7cb668e3bb..e4a1f2475d 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py @@ -1,9 +1,45 @@ """Failure cases owned by the native project assembly and target validation lane.""" +from scripts.compiler_differential._helpers.corpus.case_builder import failure_case +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, +) from scripts.compiler_differential.models import FailureCase +_TRINO_ADAPTER_PATH: str = "adapters/warehouse/duckdb_trino.py" +_TRINO_ADAPTER: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class DuckDbTrinoAdapter(DuckDbAdapter):\n" + " adapter_name = 'duckdb_trino'\n" + " sql_analysis_dialect_name = 'trino'\n" +) +_QUALIFY_HOOK_MART: str = ( + "MODEL (\n" + ' description "Customer totals",\n' + ' post_hooks [inline_sql("CREATE TABLE audit_copy AS SELECT 1 AS a, 2 AS b ' + 'FROM (SELECT 1) AS t QUALIFY row_number() OVER (PARTITION BY a ORDER BY b) = 1")]\n' + ");\n\n" + 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" +) + def assembly_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" - return () + return ( + failure_case( + name="assembly-hook-syntax-unbuilt-dialect", + expected_code="P001", + expected_message="post_hooks", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace( + 'adapter = "duckdb"', 'adapter = "duckdb_trino"' + ), + _TRINO_ADAPTER_PATH: _TRINO_ADAPTER, + FAILURE_MART_PATH: _QUALIFY_HOOK_MART, + }, + ), + ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f5fce9e74c..5d194a0bf1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -628,7 +628,7 @@ def assemble_project_resource_facts( tuple[str | None, str | None, str | None] | None, tuple[str | None, str | None, str | None, str | None], list[tuple[str, str, str]], - list[tuple[str, str]], + list[tuple[str, str | None]], list[tuple[list[tuple[str, str, str | None]], list[tuple[str, list[tuple[str, str]]]]]], list[tuple[str, bool, str | None, str | None]], list[tuple[str, str | None, str | None]], diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py index 00ea349bdc..145c5a8964 100644 --- a/src/sqlbuild/compiler/project_assembly/_helpers/request.py +++ b/src/sqlbuild/compiler/project_assembly/_helpers/request.py @@ -3,10 +3,13 @@ from __future__ import annotations import os +import re +from collections.abc import Iterable from sqlbuild.compiler.compile.models import CompileProjectInputs, CompileSqlReference from sqlbuild.compiler.project_assembly.constants import ( BOOLEAN_VARIABLE, + ENVIRONMENT_NAME_PATTERN, FALSE_TEXT, NATIVE_INT_MAX, NATIVE_INT_MIN, @@ -35,12 +38,24 @@ def project_request( target: TargetConfig | None = inputs.effective_target defaults: DefaultsConfig = inputs.project_config.defaults + target_values: tuple[str | None, ...] = ( + (target.database, target.schema, target.loader_schema) if target is not None else () + ) + default_values: tuple[str | None, ...] = ( + defaults.database, + defaults.schema, + defaults.seed_database, + defaults.seed_schema, + ) + seed_values: list[str | None] = [] + for seed_input in inputs.seed_inputs: + seed_values.extend((seed_input.schema_entry.database, seed_input.schema_entry.schema)) return ( dialect or "generic", (target.database, target.schema, target.loader_schema) if target is not None else None, (defaults.database, defaults.schema, defaults.seed_database, defaults.seed_schema), [_variable_row(name=name, value=value) for name, value in inputs.effective_vars.items()], - list(os.environ.items()), + _environment_rows((*target_values, *default_values, *seed_values)), [ (_reference_rows(model_input.references), _syntax_check_rows(checks)) for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) @@ -77,6 +92,16 @@ def project_request( ) +def _environment_rows(values: Iterable[str | None]) -> list[tuple[str, str | None]]: + """Python's own `os.environ` lookup of every name an `ENV:` template may read.""" + + names: dict[str, None] = {} + for value in values: + if value is not None: + names.update(dict.fromkeys(re.findall(ENVIRONMENT_NAME_PATTERN, value))) + return [(name, os.environ.get(name)) for name in names] + + def _variable_row(*, name: str, value: object) -> VariableRow: if value is None: return (name, NONE_VARIABLE, "") diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py index e3f7a86326..9b809a1ecd 100644 --- a/src/sqlbuild/compiler/project_assembly/constants.py +++ b/src/sqlbuild/compiler/project_assembly/constants.py @@ -10,3 +10,4 @@ NATIVE_INT_MIN: int = -(2**63) NATIVE_INT_MAX: int = 2**63 - 1 ENVIRONMENT_READ: str = "env" +ENVIRONMENT_NAME_PATTERN: str = r"ENV:([^\s(),'\"}]+)" diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py index a54e5884c6..a3e8ccf62b 100644 --- a/src/sqlbuild/compiler/project_assembly/types.py +++ b/src/sqlbuild/compiler/project_assembly/types.py @@ -16,7 +16,7 @@ TargetRow | None, DefaultsRow, list[VariableRow], - list[tuple[str, str]], + list[tuple[str, str | None]], list[ModelRow], list[SourceRow], list[SeedRow], diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index 913c32b186..680d93254c 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -35,3 +35,13 @@ class DeferredAssemblyTestCase: seed_schema: str expected_schema: str expected_kind: str + + +@dataclass(frozen=True) +class WindowsEnvironmentTestCase: + """A seed schema reading ENV through a mapping that upper-cases keys, as Windows does.""" + + description: str + environment: dict[str, str] + seed_schema: str + expected_schema: str diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index d37d832b5e..05d2a1188c 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import random from collections import Counter from collections.abc import Callable @@ -131,6 +132,15 @@ def generated_assembly_files(*, rng: random.Random, model_count: int) -> dict[st } +def windows_environ(environment: dict[str, str]) -> os._Environ[str]: + """Return this process's environment plus `environment` with Windows' upper-cased keys.""" + + data: dict[str, str] = { + name.upper(): value for name, value in {**os.environ, **environment}.items() + } + return os._Environ(data, str.upper, str, str, str) + + def seed_yml(schema: str) -> str: """Return a one-seed declaration whose schema is `schema`.""" diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index a37f1943ed..5b9ef5783e 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import random from collections import Counter from dataclasses import replace @@ -18,6 +19,7 @@ AssemblyDeferralTestCase, DeferredAssemblyTestCase, GeneratedAssemblyParityTestCase, + WindowsEnvironmentTestCase, ) from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( assemble_with, @@ -28,6 +30,7 @@ python_resource_calls, recorded_assembly, seed_yml, + windows_environ, ) _PROJECT_TOML: str = ( @@ -219,3 +222,42 @@ def test_given_a_variable_only_python_renders_when_assembling_natively_then_pyth assert assembly_deferrals(record_dir) == Counter( {f"project_assembly:{test_case.expected_kind}": 1} ) + + +@pytest.mark.parametrize( + "test_case", + [ + WindowsEnvironmentTestCase( + description="a lower-case ENV name Windows resolves to an upper-case variable", + environment={"APP_SCHEMA": "landing"}, + seed_schema="${coalesce(ENV:app_schema, 'main')}", + expected_schema="landing", + ) + ], + ids=lambda case: case.description, +) +def test_given_windows_environment_when_assembling_natively_then_env_lookups_match_python( + test_case: WindowsEnvironmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setattr(os, "environ", windows_environ(test_case.environment)) + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = project_inputs( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml(test_case.seed_schema), + **_SEED_FILES, + }, + ) + + python: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + native: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert python.seeds[0].destination.schema == test_case.expected_schema + assert assembly_view(native) == assembly_view(python) + assert assembly_deferrals(record_dir) == Counter()