diff --git a/Cargo.lock b/Cargo.lock index 98a2b0cc60..e18ef87ff8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -703,6 +703,7 @@ dependencies = [ "serde_json", "sha2", "sqlbuild-core", + "sqlbuild-model-config", "sqlbuild-scopes", "sqlbuild-sqltext", ] diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 12bdc8ea41..4899c13372 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs new file mode 100644 index 0000000000..da8005a6de --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs @@ -0,0 +1,51 @@ +//! Python's dependency keys for compiled resources (`compile/_helpers/deps/dependencies.py`). + +use std::collections::HashSet; + +use crate::assembly::project::constants::{ + ATTACHED_AUDIT_KINDS, AUDIT_TARGET_DEFERRAL, DBT_REF_KIND, KIND_RESOURCES, MODEL_RESOURCE, +}; +use crate::assembly::project::models::{AuditFacts, Reference}; +use crate::assembly::project::types::Fact; +use crate::assembly::project::types::ObjectKey; + +/// `model_build_deps` and `function_build_deps`: each reference's key, first occurrence kept. +pub(crate) fn reference_deps(references: &[Reference]) -> Vec { + deduped(references.iter().map(reference_dep).collect()) +} + +/// `audit_scope_deps`: the references, then the attached resource, first occurrence kept. +pub(crate) fn audit_deps(audit: &AuditFacts) -> Fact> { + let mut keys: Vec = reference_deps(&audit.references); + if let Some((kind, name)) = &audit.attached { + if !ATTACHED_AUDIT_KINDS.contains(&kind.as_str()) { + return Err(AUDIT_TARGET_DEFERRAL.to_owned()); + } + keys.push((kind.clone(), name.clone())); + } + Ok(deduped(keys)) +} + +/// `_reference_dep`. +fn reference_dep(reference: &Reference) -> ObjectKey { + let kind: &str = reference.kind.as_str(); + if KIND_RESOURCES.contains(&kind) { + return (kind.to_owned(), reference.name.clone()); + } + if kind == DBT_REF_KIND { + let name: String = match &reference.package { + Some(package) => format!("{package}.{}", reference.name), + None => reference.name.clone(), + }; + return (DBT_REF_KIND.to_owned(), name); + } + (MODEL_RESOURCE.to_owned(), reference.name.clone()) +} + +/// `_dedupe_object_keys`. +fn deduped(keys: Vec) -> Vec { + let mut seen: HashSet = HashSet::new(); + keys.into_iter() + .filter(|key| seen.insert(key.clone())) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs new file mode 100644 index 0000000000..000b88358d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod deps; +pub(crate) mod syntax; +pub(crate) mod targets; +pub(crate) mod templates; diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs new file mode 100644 index 0000000000..59d20a7422 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs @@ -0,0 +1,89 @@ +//! Python's `validate_sql_syntax` and `validate_hook_sql_syntax` outcome, without the message. + +use std::collections::HashMap; + +use polyglot_sql::{ + ComplexityGuardOptions, Dialect, DialectType, ValidationOptions, validate_with_dialect, +}; +use serde_json::json; + +use crate::assembly::project::constants::{ + DIALECT_DEFERRAL, MAX_FUNCTION_CALL_DEPTH, NORMALIZATION_DEFERRAL, PLACEHOLDER_DEFERRAL, + PLACEHOLDER_PREFIX, UNSUPPORTED_DIALECT_DEFERRAL, +}; +use crate::assembly::project::models::SyntaxCheck; +use crate::assembly::project::types::Fact; +use crate::lineage::main::parser_dialect::parser_dialect; +use crate::semantic_validation::main::normalize::normalize_analysis_sql; +use crate::semantic_validation::models::NormalizationInput; + +/// Whether Polyglot's `validate` accepts the cleaned SQL; Python raises where it does not. +pub(crate) fn syntax_valid(check: &SyntaxCheck, dialect: &str) -> Fact { + let cleaned: String = normalize_analysis_sql(NormalizationInput { + sql: check.sql.clone(), + dialect: dialect.to_owned(), + stubs: HashMap::new(), + placeholders: HashMap::new(), + }) + .map_err(|_| NORMALIZATION_DEFERRAL.to_owned())?; + let cleaned: String = if check.placeholders.is_empty() { + cleaned + } else { + placeholder_defaults(&cleaned, &check.placeholders)? + }; + let dialect: DialectType = + parser_dialect(Some(dialect)).ok_or_else(|| UNSUPPORTED_DIALECT_DEFERRAL.to_owned())?; + let complexity_guard: ComplexityGuardOptions = + serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) + .map_err(|_| DIALECT_DEFERRAL.to_owned())?; + let options = ValidationOptions { + complexity_guard: Some(complexity_guard), + strict_syntax: false, + semantic: false, + }; + Ok(validate_with_dialect(&cleaned, &Dialect::get(dialect), &options).valid) +} + +/// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. +fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> Fact { + let mut result: String = String::with_capacity(sql.len()); + let mut rest: &str = sql; + while let Some(start) = rest.find(PLACEHOLDER_PREFIX) { + result.push_str(&rest[..start]); + let after: &str = &rest[start + PLACEHOLDER_PREFIX.len()..]; + let length: usize = after + .find(|character: char| !is_word(character)) + .unwrap_or(after.len()); + if after[length..] + .chars() + .next() + .is_some_and(|next| !next.is_ascii()) + { + return Err(PLACEHOLDER_DEFERRAL.to_owned()); + } + if length == 0 { + result.push_str(&rest[start..=start]); + rest = &rest[start + 1..]; + continue; + } + let name: &str = &after[..length]; + match placeholders + .iter() + .find(|(placeholder, _)| placeholder == name) + { + Some((_, default)) => result.push_str(default), + None => { + result.push_str(PLACEHOLDER_PREFIX); + result.push_str(name); + } + } + rest = &after[length..]; + } + result.push_str(rest); + Ok(result) +} + +/// An ASCII character Python's `\w` matches. +fn is_word(character: char) -> bool { + character.is_ascii_alphanumeric() || character == '_' +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs new file mode 100644 index 0000000000..5a8cdeeaca --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs @@ -0,0 +1,176 @@ +//! Python's managed source and seed targets (`assembly/project.py`, `assembly/targets.py`). + +use crate::assembly::project::_helpers::templates::{Context, TemplateInputs, expanded_text}; +use crate::assembly::project::constants::{ + DESTINATION_DATABASE_CONTEXT, DESTINATION_QUALIFIED_CONTEXT, DESTINATION_SCHEMA_CONTEXT, + DESTINATION_TABLE_CONTEXT, MODEL_ALIAS_CONTEXT, MODEL_DATABASE_CONTEXT, MODEL_NAME_CONTEXT, + MODEL_SCHEMA_CONTEXT, PRESERVE_TARGET_VALUE, PRESERVED_NAMESPACE_DEFERRAL, +}; +use crate::assembly::project::models::{ + Namespace, SeedDefaults, SeedFacts, SourceFacts, TargetNamespace, +}; +use crate::assembly::project::types::Fact; + +/// A managed source's `(database, schema)`. +pub(crate) type SourceNamespace = (Option, Option); + +/// `_build_source_relation_entry`: None where the entry stays as authored. +pub(crate) fn managed_source( + source: &SourceFacts, + target: Option<&TargetNamespace>, + inputs: &TemplateInputs<'_>, +) -> Fact> { + let Some(target) = target.filter(|_| source.managed) else { + return Ok(None); + }; + let loader_schema: Option<&String> = + python_or(target.loader_schema.as_ref(), target.schema.as_ref()); + let loader_target = TargetNamespace { + database: target.database.clone(), + schema: loader_schema.cloned(), + loader_schema: target.loader_schema.clone(), + }; + preserved_namespace( + Some(&loader_target), + source.database.as_ref(), + source.schema.as_ref(), + )?; + let database: Option = match &source.database { + Some(database) => Some(database.clone()), + None => optional_text(target.database.as_deref(), inputs)?, + }; + let schema: Option = match &source.schema { + Some(schema) => Some(schema.clone()), + None => optional_text(loader_schema.map(String::as_str), inputs)?, + }; + Ok(Some((database, schema))) +} + +/// `build_seed_relation_target`. +pub(crate) fn seed_target( + seed: &SeedFacts, + defaults: &SeedDefaults, + target: Option<&TargetNamespace>, + inputs: &TemplateInputs<'_>, +) -> Fact { + let mut logical_database: Option = environment_text( + defaults + .seed_database + .as_ref() + .or(defaults.database.as_ref()), + inputs, + )?; + let mut logical_schema: Option = environment_text( + defaults.seed_schema.as_ref().or(defaults.schema.as_ref()), + inputs, + )?; + if let Some(raw) = &seed.database { + let context = seed_context(&seed.name, logical_database.clone(), logical_schema.clone()); + logical_database = seed_text(raw, inputs, &context)?; + } + if let Some(raw) = &seed.schema { + let context = seed_context(&seed.name, logical_database.clone(), logical_schema.clone()); + logical_schema = seed_text(raw, inputs, &context)?; + } + preserved_namespace(target, logical_database.as_ref(), logical_schema.as_ref())?; + let (database, schema) = match target { + None => (logical_database.clone(), logical_schema.clone()), + Some(target) => ( + target_override(target.database.as_ref(), logical_database.clone(), inputs)?, + target_override(target.schema.as_ref(), logical_schema.clone(), inputs)?, + ), + }; + Ok(Namespace { + database, + schema, + logical_database, + logical_schema, + }) +} + +/// `validate_preserved_logical_namespace`; Python raises where Err. +fn preserved_namespace( + target: Option<&TargetNamespace>, + logical_database: Option<&String>, + logical_schema: Option<&String>, +) -> Fact<()> { + let Some(target) = target else { + return Ok(()); + }; + let preserved = + |value: Option<&String>| value.is_some_and(|value| value == PRESERVE_TARGET_VALUE); + if (preserved(target.database.as_ref()) && logical_database.is_none()) + || (preserved(target.schema.as_ref()) && logical_schema.is_none()) + { + return Err(PRESERVED_NAMESPACE_DEFERRAL.to_owned()); + } + Ok(()) +} + +/// `_expand_target_value`. +fn optional_text(value: Option<&str>, inputs: &TemplateInputs<'_>) -> Fact> { + value + .map(|value| expanded_text(value, inputs, None)) + .transpose() +} + +/// `_expand_seed_default_value` and `_expand_seed_environment_value`. +fn environment_text(raw: Option<&String>, inputs: &TemplateInputs<'_>) -> Fact> { + match raw { + None => Ok(None), + Some(raw) if raw == PRESERVE_TARGET_VALUE => Ok(None), + Some(raw) => expanded_text(raw, inputs, None).map(Some), + } +} + +/// `_expand_seed_target_value`. +fn seed_text( + raw: &str, + inputs: &TemplateInputs<'_>, + context: &Context<'_>, +) -> Fact> { + if raw == PRESERVE_TARGET_VALUE { + return Ok(None); + } + expanded_text(raw, inputs, Some(context)).map(Some) +} + +/// `_apply_seed_target_overrides` for one dimension. +fn target_override( + raw: Option<&String>, + logical: Option, + inputs: &TemplateInputs<'_>, +) -> Fact> { + match raw { + Some(raw) if raw != PRESERVE_TARGET_VALUE => environment_text(Some(raw), inputs), + _ => Ok(logical), + } +} + +/// The context values a seed's own database and schema templates read. +fn seed_context( + name: &str, + database: Option, + schema: Option, +) -> Vec<(&'static str, Option)> { + let qualified: Option = match (&database, &schema) { + (Some(database), Some(schema)) => Some(format!("{database}.{schema}.{name}")), + (None, Some(schema)) => Some(format!("{schema}.{name}")), + _ => None, + }; + vec![ + (MODEL_NAME_CONTEXT, Some(name.to_owned())), + (MODEL_DATABASE_CONTEXT, database.clone()), + (MODEL_SCHEMA_CONTEXT, schema.clone()), + (MODEL_ALIAS_CONTEXT, Some(name.to_owned())), + (DESTINATION_DATABASE_CONTEXT, database), + (DESTINATION_SCHEMA_CONTEXT, schema), + (DESTINATION_TABLE_CONTEXT, Some(name.to_owned())), + (DESTINATION_QUALIFIED_CONTEXT, qualified), + ] +} + +/// Python's `left or right` over optional text: an empty string falls through. +fn python_or<'a>(left: Option<&'a String>, right: Option<&'a String>) -> Option<&'a String> { + left.filter(|value| !value.is_empty()).or(right) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs new file mode 100644 index 0000000000..4a5b17d1ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs @@ -0,0 +1,129 @@ +//! Python's `str(expand_template_data(...))` over scalar variables, the environment and context. + +use std::cell::RefCell; + +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::models::{ + ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, +}; +use sqlbuild_model_config::templates::types::TemplateHost; + +use crate::assembly::project::constants::{ + PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE, TEMPLATE_DEFERRAL, TEMPLATE_OPEN_TOKEN, +}; +use crate::assembly::project::models::{InputRead, Variable}; +use crate::assembly::project::types::Fact; + +/// Context values by name; None is a known key whose value Python holds as None. +pub(crate) type Context<'a> = [(&'static str, Option)]; + +/// The variables and environment target templates read, and every lookup they make in order. +pub(crate) struct TemplateInputs<'a> { + pub(crate) variables: &'a [(String, Variable)], + /// Python's `os.environ.get` of each name the templates may read. + pub(crate) environment: &'a [(String, Option)], + pub(crate) reads: RefCell>, +} + +/// One expansion's inputs and context values. +struct ScalarHost<'a> { + inputs: &'a TemplateInputs<'a>, + context: &'a Context<'a>, +} + +impl TemplateHost for ScalarHost<'_> { + type Value = Scalar; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + match self + .inputs + .variables + .iter() + .find(|(variable, _)| variable == name) + { + None => Ok(None), + Some((_, Variable::Scalar(value))) => Ok(Some(value.clone())), + Some((_, Variable::Unsupported)) => Err(TemplateFailure::Unsupported), + } + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.inputs + .reads + .borrow_mut() + .push(InputRead::Environment(name.to_owned())); + match self + .inputs + .environment + .iter() + .find(|(variable, _)| variable == name) + { + None => Err(TemplateFailure::Unsupported), + Some((_, value)) => Ok(value.clone().map(Scalar::Text)), + } + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.inputs + .reads + .borrow_mut() + .push(InputRead::Context(name.to_owned())); + Ok(match self.context.iter().find(|(key, _)| *key == name) { + None => ContextValue::Unknown, + Some((_, None)) => ContextValue::Unavailable, + Some((_, Some(value))) => ContextValue::Value(Scalar::Text(value.clone())), + }) + } + + fn text(&self, text: &str) -> Result { + Ok(Scalar::Text(text.to_owned())) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(Scalar::Bool(flag)) + } + + fn null(&self) -> Self::Value { + Scalar::Null + } + + fn scalar(&self, value: &Self::Value) -> Option { + Some(value.clone()) + } +} + +/// `str(expand_template_data(value=text, ...))`; context is allowed only where Some. +pub(crate) fn expanded_text( + text: &str, + inputs: &TemplateInputs<'_>, + context: Option<&Context<'_>>, +) -> Fact { + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(text.to_owned()); + } + let options = TemplateOptions { + allow_context: context.is_some(), + preserve_context_tokens: false, + preserve_unknown_context: false, + }; + let host = ScalarHost { + inputs, + context: context.unwrap_or_default(), + }; + match expand_template_string(&host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(text.to_owned()), + Ok(StringExpansion::Value(value)) => Ok(python_str(value)), + Ok(StringExpansion::Text(expanded)) => Ok(expanded), + Err(_) => Err(TEMPLATE_DEFERRAL.to_owned()), + } +} + +/// Python's `str()` of a scalar. +fn python_str(value: Scalar) -> String { + match value { + Scalar::Null => PYTHON_NONE.to_owned(), + Scalar::Bool(true) => PYTHON_TRUE.to_owned(), + Scalar::Bool(false) => PYTHON_FALSE.to_owned(), + Scalar::Text(text) => text, + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs new file mode 100644 index 0000000000..2fb32b6c70 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -0,0 +1,37 @@ +//! Python's resource, reference and target constants the project assembly reproduces. + +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +/// The reference kinds whose dependency keeps the reference's kind as its resource type. +pub(crate) const KIND_RESOURCES: [&str; 4] = + [SOURCE_KIND, SEED_KIND, UDF_KIND, TABLE_FUNCTION_KIND]; +/// `AttachedAuditTargetKind` values, each also its resource type. +pub(crate) const ATTACHED_AUDIT_KINDS: [&str; 3] = [MODEL_RESOURCE, SOURCE_KIND, SEED_KIND]; +pub(crate) const PRESERVE_TARGET_VALUE: &str = "preserve"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const PYTHON_NONE: &str = "None"; +pub(crate) const PYTHON_TRUE: &str = "True"; +pub(crate) const PYTHON_FALSE: &str = "False"; +pub(crate) const PLACEHOLDER_PREFIX: &str = "@@@"; +pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; +pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; +pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; +pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; +pub(crate) const MODEL_ALIAS_CONTEXT: &str = "model.alias"; +pub(crate) const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; +pub(crate) const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; +pub(crate) const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; +pub(crate) const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; +pub(crate) const AUDIT_TARGET_DEFERRAL: &str = "audit_target_kind"; +pub(crate) const TEMPLATE_DEFERRAL: &str = "target_template"; +pub(crate) const PRESERVED_NAMESPACE_DEFERRAL: &str = "preserved_namespace"; +pub(crate) const SYNTAX_DEFERRAL: &str = "syntax_error"; +pub(crate) const PLACEHOLDER_DEFERRAL: &str = "placeholder_word"; +pub(crate) const NORMALIZATION_DEFERRAL: &str = "normalization"; +pub(crate) const DIALECT_DEFERRAL: &str = "dialect"; +pub(crate) const UNSUPPORTED_DIALECT_DEFERRAL: &str = "unsupported_dialect"; +pub(crate) const PANIC_DEFERRAL: &str = "panic"; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs b/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs new file mode 100644 index 0000000000..add612c439 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/assemble_project_resources.rs @@ -0,0 +1,73 @@ +//! Python's resource assembly facts for one project, or why Python must assemble it. + +use std::cell::RefCell; + +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::assembly::project::_helpers::deps::{audit_deps, reference_deps}; +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; +use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::constants::{PANIC_DEFERRAL, SYNTAX_DEFERRAL}; +use crate::assembly::project::models::{ModelFacts, ProjectRequest, ProjectResources}; +use crate::assembly::project::types::Fact; + +/// Each resource's dependencies and namespace, or the kind of work Python must do instead. +pub fn assemble_project_resources(request: &ProjectRequest) -> Result { + catch_compiler_panic(|| resources(request)).map_err(|kind| { + if kind == PANIC_MESSAGE { + PANIC_DEFERRAL.to_owned() + } else { + kind + } + }) +} + +fn resources(request: &ProjectRequest) -> Fact { + for model in &request.models { + valid_syntax(model, &request.dialect)?; + } + let target = request.target.as_ref(); + let inputs = TemplateInputs { + variables: &request.variables, + environment: &request.environment, + reads: RefCell::new(Vec::new()), + }; + let sources = request + .sources + .iter() + .map(|source| managed_source(source, target, &inputs)) + .collect::>()?; + let seeds = request + .seeds + .iter() + .map(|seed| seed_target(seed, &request.defaults, target, &inputs)) + .collect::>()?; + Ok(ProjectResources { + model_deps: request + .models + .iter() + .map(|model| reference_deps(&model.references)) + .collect(), + sources, + seeds, + function_deps: request + .functions + .iter() + .map(|references| reference_deps(references)) + .collect(), + audit_deps: request.audits.iter().map(audit_deps).collect::>()?, + reads: inputs.reads.into_inner(), + }) +} + +/// Python's model and hook syntax validation, which raises for the first SQL it rejects. +fn valid_syntax(model: &ModelFacts, dialect: &str) -> Fact<()> { + for check in &model.syntax_checks { + if !syntax_valid(check, dialect)? { + return Err(SYNTAX_DEFERRAL.to_owned()); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs new file mode 100644 index 0000000000..d53093ec5a --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -0,0 +1 @@ +pub mod assemble_project_resources; diff --git a/crates/sqlbuild-analysis/src/assembly/project/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/mod.rs index 2c6b3134d3..4439eb2731 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/mod.rs @@ -1 +1,10 @@ //! Compiled project assembly: resources, dependencies, destinations and their diagnostics. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/assembly/project/models.rs b/crates/sqlbuild-analysis/src/assembly/project/models.rs new file mode 100644 index 0000000000..92d0f34a3c --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/models.rs @@ -0,0 +1,123 @@ +//! The project assembly request Python builds from its compile inputs, and the facts it returns. + +use sqlbuild_model_config::templates::models::Scalar; + +use crate::assembly::analysis_session::types::Pairs; +use crate::assembly::project::types::ObjectKey; + +/// One SQL reference a resource makes: Python's `CompileSqlReference`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reference { + /// The `SqlReferenceKind` value. + pub kind: String, + pub name: String, + pub package: Option, +} + +/// One project variable, as a scalar Python renders with `str()`, or a value only Python renders. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Variable { + Scalar(Scalar), + Unsupported, +} + +/// One environment or context lookup a template made, which Python records for compile reuse. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InputRead { + Environment(String), + Context(String), +} + +/// The selected target's namespace settings. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TargetNamespace { + pub database: Option, + pub schema: Option, + pub loader_schema: Option, +} + +/// The project defaults a seed's logical namespace reads. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SeedDefaults { + pub database: Option, + pub schema: Option, + pub seed_database: Option, + pub seed_schema: Option, +} + +/// One model or hook SQL string Python's syntax validation checks, with its placeholders. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyntaxCheck { + pub sql: String, + pub placeholders: Pairs, +} + +/// One model's references and the SQL strings its assembly validates. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModelFacts { + pub references: Vec, + pub syntax_checks: Vec, +} + +/// One source's name, whether a loader manages it, and its authored namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceFacts { + pub name: String, + pub managed: bool, + pub database: Option, + pub schema: Option, +} + +/// One seed's name and authored namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SeedFacts { + pub name: String, + pub database: Option, + pub schema: Option, +} + +/// One audit's references and the resource it is attached to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditFacts { + pub references: Vec, + /// The attached resource's `(resource type, name)`. + pub attached: Option<(String, String)>, +} + +/// Everything one project's resource assembly reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectRequest { + /// The analysis dialect syntax validation parses with, `generic` when the adapter names none. + pub dialect: String, + pub target: Option, + pub defaults: SeedDefaults, + pub variables: Vec<(String, Variable)>, + /// Python's `os.environ.get` of every name an `ENV:` template in the request names. + pub environment: Vec<(String, Option)>, + pub models: Vec, + pub sources: Vec, + pub seeds: Vec, + pub functions: Vec>, + pub audits: Vec, +} + +/// A relation's physical and logical namespace. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Namespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +/// The facts Python's resource assembly derives, in input order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProjectResources { + pub model_deps: Vec>, + /// A managed source's `(database, schema)`; None where the entry stays as authored. + pub sources: Vec, Option)>>, + pub seeds: Vec, + pub function_deps: Vec>, + pub audit_deps: Vec>, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs new file mode 100644 index 0000000000..b8a5f96c5e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_project_assembly.rs"] +mod project_assembly; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs new file mode 100644 index 0000000000..5f245315d3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs @@ -0,0 +1,166 @@ +use std::cell::RefCell; + +use sqlbuild_model_config::templates::models::Scalar; + +use crate::assembly::project::_helpers::deps::audit_deps; +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; +use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::models::{ + AuditFacts, InputRead, Namespace, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, + TargetNamespace, Variable, +}; +use crate::assembly::project::tests::test_types::{ExpectedNamespace, SeedSpec}; +use crate::assembly::project::types::ObjectKey; + +/// `(database, schema, loader schema)` of a selected target. +type TargetSpec = ( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, +); + +pub(crate) fn deps( + references: &[(&str, &str, Option<&str>)], + attached: Option<(&str, &str)>, +) -> Option> { + audit_deps(&AuditFacts { + references: references.iter().map(reference).collect(), + attached: attached.map(|(kind, name)| (kind.to_owned(), name.to_owned())), + }) + .ok() +} + +pub(crate) fn seed( + seed: SeedSpec, + defaults: ExpectedNamespace, + target: Option, +) -> Option { + let (name, database, schema) = seed; + let (default_database, default_schema, seed_database, seed_schema) = defaults; + seed_target( + &SeedFacts { + name: name.to_owned(), + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + &SeedDefaults { + database: default_database.map(str::to_owned), + schema: default_schema.map(str::to_owned), + seed_database: seed_database.map(str::to_owned), + seed_schema: seed_schema.map(str::to_owned), + }, + target.map(namespace_target).as_ref(), + &template_inputs(&variables()), + ) + .ok() +} + +/// The seed's namespace with `environment` set, and the reads its templates made. +pub(crate) fn seed_reads( + seed: SeedSpec, + environment: &[(String, Option)], +) -> (Option, Vec) { + let (name, database, schema) = seed; + let variables: Vec<(String, Variable)> = variables(); + let inputs = TemplateInputs { + variables: &variables, + environment, + reads: RefCell::new(Vec::new()), + }; + let resolved = seed_target( + &SeedFacts { + name: name.to_owned(), + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + &SeedDefaults::default(), + None, + &inputs, + ) + .ok(); + (resolved, inputs.reads.into_inner()) +} + +pub(crate) fn source( + source: (bool, Option<&str>, Option<&str>), + target: Option, +) -> Option, Option)>> { + let (managed, database, schema) = source; + managed_source( + &SourceFacts { + name: "orders".to_owned(), + managed, + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + }, + target.map(namespace_target).as_ref(), + &template_inputs(&variables()), + ) + .ok() +} + +pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)], dialect: &str) -> Option { + syntax_valid( + &SyntaxCheck { + sql: sql.to_owned(), + placeholders: placeholders + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(), + }, + dialect, + ) + .ok() +} + +pub(crate) fn namespace(expected: ExpectedNamespace) -> Namespace { + let (database, schema, logical_database, logical_schema) = expected; + Namespace { + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + logical_database: logical_database.map(str::to_owned), + logical_schema: logical_schema.map(str::to_owned), + } +} + +fn reference(spec: &(&str, &str, Option<&str>)) -> Reference { + let (kind, name, package) = spec; + Reference { + kind: (*kind).to_owned(), + name: (*name).to_owned(), + package: package.map(str::to_owned), + } +} + +fn namespace_target(spec: TargetSpec) -> TargetNamespace { + let (database, schema, loader_schema) = spec; + TargetNamespace { + database: database.map(str::to_owned), + schema: schema.map(str::to_owned), + loader_schema: loader_schema.map(str::to_owned), + } +} + +fn template_inputs(variables: &[(String, Variable)]) -> TemplateInputs<'_> { + TemplateInputs { + variables, + environment: &[], + reads: RefCell::new(Vec::new()), + } +} + +fn variables() -> Vec<(String, Variable)> { + vec![ + ( + "env_name".to_owned(), + Variable::Scalar(Scalar::Text("prod".to_owned())), + ), + ( + "n".to_owned(), + Variable::Scalar(Scalar::Text("3".to_owned())), + ), + ("flag".to_owned(), Variable::Scalar(Scalar::Bool(true))), + ("tags".to_owned(), Variable::Unsupported), + ] +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs new file mode 100644 index 0000000000..e7a7b02ccc --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs @@ -0,0 +1,289 @@ +use crate::assembly::project::models::InputRead; +use crate::assembly::project::tests::helpers::{deps, namespace, seed, seed_reads, source, valid}; +use crate::assembly::project::tests::test_types::{ + DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxTestCase, keys, +}; + +#[test] +fn given_references_when_deriving_deps_then_matches_python_keys() { + let test_cases = [ + DepsTestCase { + description: "each kind maps to its resource and repeats keep the first key", + references: &[ + ("ref", "orders", None), + ("source", "raw_orders", None), + ("ref", "orders", None), + ("dbt_ref", "customers", Some("upstream")), + ("dbt_ref", "products", None), + ("udf", "tax", None), + ("table_fn", "series", None), + ("seed", "countries", None), + ], + attached: Some(("model", "orders")), + expected_deps: Some(&[ + ("model", "orders"), + ("source", "raw_orders"), + ("dbt_ref", "upstream.customers"), + ("dbt_ref", "products"), + ("udf", "tax"), + ("table_fn", "series"), + ("seed", "countries"), + ]), + }, + DepsTestCase { + description: "an attached target joins the references", + references: &[("ref", "orders", None)], + attached: Some(("seed", "countries")), + expected_deps: Some(&[("model", "orders"), ("seed", "countries")]), + }, + DepsTestCase { + description: "an attached kind Python rejects", + references: &[], + attached: Some(("function", "tax")), + expected_deps: None, + }, + ]; + for test_case in test_cases { + let derived = deps(test_case.references, test_case.attached); + + assert_eq!( + derived, + test_case.expected_deps.map(keys), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_seeds_when_resolving_targets_then_matches_python_or_defers() { + let test_cases = [ + SeedTestCase { + description: "defaults and the seed's templates read variables and updated context", + seed: ( + "countries", + Some("${n}"), + Some("seeds_${env_name}_${CTX:model.database}"), + ), + defaults: (Some("db_${env_name}"), None, None, Some("raw")), + target: None, + expected_namespace: Some(( + Some("3"), + Some("seeds_prod_3"), + Some("3"), + Some("seeds_prod_3"), + )), + }, + SeedTestCase { + description: "a target overrides the schema and preserves the database", + seed: ("countries", None, None), + defaults: (Some("analytics"), Some("main"), None, None), + target: Some((Some("preserve"), Some("dev_${env_name}"), None)), + expected_namespace: Some(( + Some("analytics"), + Some("dev_prod"), + Some("analytics"), + Some("main"), + )), + }, + SeedTestCase { + description: "the qualified destination context", + seed: ("countries", None, Some("${CTX:destination.qualified}_x")), + defaults: (Some("analytics"), Some("main"), None, None), + target: None, + expected_namespace: Some(( + Some("analytics"), + Some("analytics.main.countries_x"), + Some("analytics"), + Some("analytics.main.countries_x"), + )), + }, + SeedTestCase { + description: "a whole-template boolean renders as Python's str", + seed: ("countries", None, Some("${flag}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: Some((None, Some("True"), None, Some("True"))), + }, + SeedTestCase { + description: "a preserved schema the seed does not own raises in Python", + seed: ("countries", None, None), + defaults: (None, None, None, None), + target: Some((None, Some("preserve"), None)), + expected_namespace: None, + }, + SeedTestCase { + description: "environment reads stay with Python", + seed: ("countries", None, Some("${ENV:SEED_SCHEMA}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: None, + }, + SeedTestCase { + description: "a variable only Python renders", + seed: ("countries", None, Some("${tags}")), + defaults: (None, None, None, None), + target: None, + expected_namespace: None, + }, + ]; + for test_case in test_cases { + let resolved = seed(test_case.seed, test_case.defaults, test_case.target); + + assert_eq!( + resolved, + test_case.expected_namespace.map(namespace), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sources_when_resolving_managed_targets_then_matches_python() { + let test_cases = [ + SourceTestCase { + description: "an empty loader schema falls back to the target schema", + source: (true, None, None), + target: Some((Some("wh_${env_name}"), Some("main"), Some(""))), + expected_namespace: Some(Some((Some("wh_prod"), Some("main")))), + }, + SourceTestCase { + description: "an authored schema wins over the loader schema", + source: (true, None, Some("landing")), + target: Some((Some("wh"), None, Some("loaders"))), + expected_namespace: Some(Some((Some("wh"), Some("landing")))), + }, + SourceTestCase { + description: "an unmanaged source stays as authored", + source: (false, None, None), + target: Some((Some("wh"), Some("main"), None)), + expected_namespace: Some(None), + }, + ]; + for test_case in test_cases { + let resolved = source(test_case.source, test_case.target); + + assert_eq!( + resolved, + test_case.expected_namespace.map(|namespace| { + namespace.map(|(database, schema)| { + (database.map(str::to_owned), schema.map(str::to_owned)) + }) + }), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_sql_when_validating_syntax_then_matches_python_or_defers() { + let test_cases = [ + SyntaxTestCase { + dialect: "duckdb", + description: "markers and placeholder defaults parse", + sql: "SELECT @@@x AS a FROM __ref(\"orders\")", + placeholders: &[("x", "1")], + expected_valid: Some(true), + }, + SyntaxTestCase { + dialect: "duckdb", + description: "a syntax error", + sql: "SELECT FROM WHERE (", + placeholders: &[], + expected_valid: Some(false), + }, + SyntaxTestCase { + dialect: "duckdb", + description: "a placeholder run retried one character later, as re.sub does", + sql: "SELECT @@@@@abc AS a", + placeholders: &[("abc", "version")], + expected_valid: Some(true), + }, + SyntaxTestCase { + dialect: "duckdb", + description: "a placeholder followed by text Python's \\w may extend", + sql: "SELECT @@@x\u{e9} AS a", + placeholders: &[("x", "1")], + expected_valid: None, + }, + SyntaxTestCase { + dialect: "trino", + description: "a dialect this parser build does not carry", + sql: "SELECT 1 AS a FROM t QUALIFY a = 1", + placeholders: &[], + expected_valid: None, + }, + ]; + for test_case in test_cases { + let checked = valid(test_case.sql, test_case.placeholders, test_case.dialect); + + assert_eq!( + checked, test_case.expected_valid, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_environment_templates_when_resolving_seeds_then_matches_python_values_and_reads() { + let test_cases = [ + EnvironmentTestCase { + description: "a coalesced unset variable, a set variable and a context key", + seed: ( + "countries", + None, + Some( + "${coalesce(ENV:SQB_UNSET_SCHEMA, 'main')}_${ENV:SQB_SET_SCHEMA}_${CTX:model.name}", + ), + ), + environment: &[ + ("SQB_UNSET_SCHEMA", None), + ("SQB_SET_SCHEMA", Some("landing")), + ], + expected_schema: Some("main_landing_countries"), + expected_reads: vec![ + InputRead::Environment("SQB_UNSET_SCHEMA".to_owned()), + InputRead::Environment("SQB_SET_SCHEMA".to_owned()), + InputRead::Context("model.name".to_owned()), + ], + }, + EnvironmentTestCase { + description: "an unset variable Python raises for", + seed: ("countries", None, Some("${ENV:SQB_UNSET_SCHEMA}")), + environment: &[("SQB_UNSET_SCHEMA", None)], + expected_schema: None, + expected_reads: vec![InputRead::Environment("SQB_UNSET_SCHEMA".to_owned())], + }, + EnvironmentTestCase { + description: "a name Python did not resolve stays with Python", + seed: ( + "countries", + None, + Some("${coalesce(ENV:app_schema, 'main')}"), + ), + environment: &[("APP_SCHEMA", Some("landing"))], + expected_schema: None, + expected_reads: vec![InputRead::Environment("app_schema".to_owned())], + }, + ]; + for test_case in test_cases { + let environment: Vec<(String, Option)> = test_case + .environment + .iter() + .map(|(name, value)| ((*name).to_owned(), value.map(str::to_owned))) + .collect(); + + let (resolved, reads) = seed_reads(test_case.seed, &environment); + + assert_eq!( + resolved.and_then(|namespace| namespace.schema), + test_case.expected_schema.map(str::to_owned), + "{}", + test_case.description + ); + assert_eq!(reads, test_case.expected_reads, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs new file mode 100644 index 0000000000..f39fb71bfd --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -0,0 +1,75 @@ +use crate::assembly::project::models::InputRead; +use crate::assembly::project::types::ObjectKey; + +/// `(name, database, schema)` of one authored seed. +pub(crate) type SeedSpec = (&'static str, Option<&'static str>, Option<&'static str>); +/// `(database, schema, logical database, logical schema)`. +pub(crate) type ExpectedNamespace = ( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, +); + +pub(crate) struct DepsTestCase { + pub(crate) description: &'static str, + /// `(kind, name, package)` references. + pub(crate) references: &'static [(&'static str, &'static str, Option<&'static str>)], + pub(crate) attached: Option<(&'static str, &'static str)>, + /// None where Python rejects the audit. + pub(crate) expected_deps: Option<&'static [(&'static str, &'static str)]>, +} + +pub(crate) struct SeedTestCase { + pub(crate) description: &'static str, + pub(crate) seed: SeedSpec, + /// `(database, schema, seed database, seed schema)` defaults. + pub(crate) defaults: ExpectedNamespace, + /// `(database, schema, loader schema)` of the selected target. + pub(crate) target: Option<( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + )>, + /// None where Python raises or expands the target itself. + pub(crate) expected_namespace: Option, +} + +pub(crate) struct SourceTestCase { + pub(crate) description: &'static str, + /// `(managed, database, schema)` of the authored source. + pub(crate) source: (bool, Option<&'static str>, Option<&'static str>), + pub(crate) target: Option<( + Option<&'static str>, + Option<&'static str>, + Option<&'static str>, + )>, + /// Outer None where Python raises, inner None where the entry stays as authored. + pub(crate) expected_namespace: Option, Option<&'static str>)>>, +} + +pub(crate) struct SyntaxTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) placeholders: &'static [(&'static str, &'static str)], + /// None where native defers to Python. + pub(crate) expected_valid: Option, +} + +pub(crate) fn keys(values: &[(&str, &str)]) -> Vec { + values + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect() +} + +pub(crate) struct EnvironmentTestCase { + pub(crate) description: &'static str, + pub(crate) seed: SeedSpec, + /// Python's `os.environ.get` of each name. + pub(crate) environment: &'static [(&'static str, Option<&'static str>)], + /// None where Python raises. + pub(crate) expected_schema: Option<&'static str>, + pub(crate) expected_reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/types.rs b/crates/sqlbuild-analysis/src/assembly/project/types.rs new file mode 100644 index 0000000000..8d39093007 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/types.rs @@ -0,0 +1,7 @@ +//! Project assembly key and result aliases. + +/// One step's result, or the deferral kind Python must assemble the project for. +pub(crate) type Fact = Result; + +/// A `CompiledObjectKey`: `(resource type, name)`. +pub type ObjectKey = (String, String); diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs index ad66de574c..55e9557c5f 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs @@ -26,25 +26,6 @@ pub(crate) fn proxy_parse_options() -> Result { }) } -/// Python's `parse_one(dialect=dialect or "generic")` dialect, if this parser build carries it. -pub(crate) fn lineage_dialect(name: Option<&str>) -> Option { - let name = name.filter(|name| !name.is_empty()).unwrap_or("generic"); - let Ok(dialect) = name.parse::() else { - return None; - }; - matches!( - dialect, - DialectType::Generic - | DialectType::PostgreSQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::DuckDB - | DialectType::TSQL - | DialectType::Databricks - ) - .then_some(dialect) -} - pub(crate) struct ParsedModel<'a> { pub(crate) query_sql: &'a str, pub(crate) inferred_names: &'a [String], diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index f1266f2eff..4eeb8c3c90 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -8,10 +8,11 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use crate::lineage::_helpers::parsed_lineage::{ - ParsedModel, lineage_dialect, parsed_model_lineage, proxy_parse_options, + ParsedModel, parsed_model_lineage, proxy_parse_options, }; use crate::lineage::_helpers::references::physical_resources; use crate::lineage::_helpers::stars::{schema_mapping, star_lineage}; +use crate::lineage::main::parser_dialect::parser_dialect; use crate::lineage::models::{ FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageDeferral, }; @@ -29,7 +30,7 @@ pub fn build_fast_lineage( /// A parser panic defers only its own model. fn model_outcomes(request: &FastLineageRequest) -> Vec { let schema = schema_mapping(&request.schema); - let dialect = lineage_dialect(request.dialect.as_deref()); + let dialect = parser_dialect(request.dialect.as_deref()); let options: Result = proxy_parse_options(); request .models diff --git a/crates/sqlbuild-analysis/src/lineage/main/mod.rs b/crates/sqlbuild-analysis/src/lineage/main/mod.rs index f1d5cd565b..6b75bd22ea 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/mod.rs @@ -1 +1,2 @@ pub mod build_fast_lineage; +pub mod parser_dialect; diff --git a/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs b/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs new file mode 100644 index 0000000000..61a7005ee7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/main/parser_dialect.rs @@ -0,0 +1,22 @@ +//! The polyglot dialect Python's `dialect or "generic"` names, if this parser build carries it. + +use polyglot_sql::DialectType; + +/// Python's `parse_one(dialect=dialect or "generic")` dialect, if this parser build carries it. +pub fn parser_dialect(name: Option<&str>) -> Option { + let name = name.filter(|name| !name.is_empty()).unwrap_or("generic"); + let Ok(dialect) = name.parse::() else { + return None; + }; + matches!( + dialect, + DialectType::Generic + | DialectType::PostgreSQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::DuckDB + | DialectType::TSQL + | DialectType::Databricks + ) + .then_some(dialect) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 7444bd0374..79eb3404a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1 +1,2 @@ pub(crate) mod registration; +pub(crate) mod resources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 63be94a863..67e7f084fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -2,6 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { - Ok(()) +use crate::bindings::_helpers::project_assembly::resources; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + resources::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs new file mode 100644 index 0000000000..d2fbcc1350 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -0,0 +1,207 @@ +//! Python's resource assembly facts for one project, computed natively for the preview engine. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::models::{ + AuditFacts, InputRead, ModelFacts, Namespace, ProjectRequest, ProjectResources, Reference, + SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, +}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_model_config::templates::models::Scalar; + +use crate::bindings::types::CompilerDetach; + +const TEXT_VARIABLE: &str = "text"; +const BOOLEAN_VARIABLE: &str = "bool"; +const NONE_VARIABLE: &str = "none"; +const TRUE_TEXT: &str = "1"; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; + +/// `(kind, name, package)`. +type ReferenceRow = (String, String, Option); +/// `(database, schema, loader schema)`. +type TargetRow = (Option, Option, Option); +/// `(database, schema, seed database, seed schema)`. +type DefaultsRow = ( + Option, + Option, + Option, + Option, +); +/// `(name, kind, text)`; kind `text` (also exact ints), `bool`, `none` or anything else. +type VariableRow = (String, String, String); +/// `(references, [(sql, placeholders)])`. +type ModelRow = (Vec, Vec<(String, Pairs)>); +/// `(name, managed, database, schema)`. +type SourceRow = (String, bool, Option, Option); +/// `(name, database, schema)`. +type SeedRow = (String, Option, Option); +/// `(references, attached (kind, name))`. +type AuditRow = (Vec, Option<(String, String)>); +/// The dialect, target, defaults, variables, environment and resources one project assembles. +type RequestRow = ( + String, + Option, + DefaultsRow, + Vec, + Vec<(String, Option)>, + Vec, + Vec, + Vec, + Vec>, + Vec, +); +/// `(database, schema, logical database, logical schema)`. +type NamespaceRow = ( + Option, + Option, + Option, + Option, +); +/// Deps, managed source and seed namespaces, and the `(env or ctx, name)` template reads. +type ResourcesRow = ( + Vec>, + Vec, Option)>>, + Vec, + Vec>, + Vec>, + Vec<(&'static str, String)>, +); + +/// The project's resource facts, or None with the reason Python must assemble it. +#[pyfunction] +fn assemble_project_resource_facts( + py: Python<'_>, + request: RequestRow, +) -> (Option, Option) { + let request: ProjectRequest = project_request(request); + match py.compiler_detach(|| assemble_project_resources(&request)) { + Ok(resources) => (Some(resources_row(resources)), None), + Err(reason) => (None, Some(reason)), + } +} + +fn project_request(request: RequestRow) -> ProjectRequest { + let ( + dialect, + target, + defaults, + variables, + environment, + models, + sources, + seeds, + functions, + audits, + ) = request; + let (database, schema, seed_database, seed_schema) = defaults; + ProjectRequest { + dialect, + target: target.map(|(database, schema, loader_schema)| TargetNamespace { + database, + schema, + loader_schema, + }), + defaults: SeedDefaults { + database, + schema, + seed_database, + seed_schema, + }, + variables: variables.into_iter().map(variable).collect(), + environment, + models: models.into_iter().map(model).collect(), + sources: sources + .into_iter() + .map(|(name, managed, database, schema)| SourceFacts { + name, + managed, + database, + schema, + }) + .collect(), + seeds: seeds + .into_iter() + .map(|(name, database, schema)| SeedFacts { + name, + database, + schema, + }) + .collect(), + functions: functions.into_iter().map(references).collect(), + audits: audits + .into_iter() + .map(|(rows, attached)| AuditFacts { + references: references(rows), + attached, + }) + .collect(), + } +} + +fn variable(row: VariableRow) -> (String, Variable) { + let (name, kind, text) = row; + let value: Variable = match kind.as_str() { + TEXT_VARIABLE => Variable::Scalar(Scalar::Text(text)), + BOOLEAN_VARIABLE => Variable::Scalar(Scalar::Bool(text == TRUE_TEXT)), + NONE_VARIABLE => Variable::Scalar(Scalar::Null), + _ => Variable::Unsupported, + }; + (name, value) +} + +fn model(row: ModelRow) -> ModelFacts { + let (rows, checks) = row; + ModelFacts { + references: references(rows), + syntax_checks: checks + .into_iter() + .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) + .collect(), + } +} + +fn references(rows: Vec) -> Vec { + rows.into_iter() + .map(|(kind, name, package)| Reference { + kind, + name, + package, + }) + .collect() +} + +fn resources_row(resources: ProjectResources) -> ResourcesRow { + ( + resources.model_deps, + resources.sources, + resources.seeds.into_iter().map(namespace_row).collect(), + resources.function_deps, + resources.audit_deps, + resources.reads.into_iter().map(read_row).collect(), + ) +} + +fn read_row(read: InputRead) -> (&'static str, String) { + match read { + InputRead::Environment(name) => (ENVIRONMENT_READ, name), + InputRead::Context(name) => (CONTEXT_READ, name), + } +} + +fn namespace_row(namespace: Namespace) -> NamespaceRow { + ( + namespace.database, + namespace.schema, + namespace.logical_database, + namespace.logical_schema, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; + Ok(()) +} diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py index 7cb668e3bb..e4a1f2475d 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py @@ -1,9 +1,45 @@ """Failure cases owned by the native project assembly and target validation lane.""" +from scripts.compiler_differential._helpers.corpus.case_builder import failure_case +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, +) from scripts.compiler_differential.models import FailureCase +_TRINO_ADAPTER_PATH: str = "adapters/warehouse/duckdb_trino.py" +_TRINO_ADAPTER: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class DuckDbTrinoAdapter(DuckDbAdapter):\n" + " adapter_name = 'duckdb_trino'\n" + " sql_analysis_dialect_name = 'trino'\n" +) +_QUALIFY_HOOK_MART: str = ( + "MODEL (\n" + ' description "Customer totals",\n' + ' post_hooks [inline_sql("CREATE TABLE audit_copy AS SELECT 1 AS a, 2 AS b ' + 'FROM (SELECT 1) AS t QUALIFY row_number() OVER (PARTITION BY a ORDER BY b) = 1")]\n' + ");\n\n" + 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" +) + def assembly_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" - return () + return ( + failure_case( + name="assembly-hook-syntax-unbuilt-dialect", + expected_code="P001", + expected_message="post_hooks", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace( + 'adapter = "duckdb"', 'adapter = "duckdb_trino"' + ), + _TRINO_ADAPTER_PATH: _TRINO_ADAPTER, + FAILURE_MART_PATH: _QUALIFY_HOOK_MART, + }, + ), + ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b98b504587..5d194a0bf1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -622,6 +622,32 @@ def plan_compiled_sql_tests( def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. +def assemble_project_resource_facts( + request: tuple[ + str, + tuple[str | None, str | None, str | None] | None, + tuple[str | None, str | None, str | None, str | None], + list[tuple[str, str, str]], + list[tuple[str, str | None]], + list[tuple[list[tuple[str, str, str | None]], list[tuple[str, list[tuple[str, str]]]]]], + list[tuple[str, bool, str | None, str | None]], + list[tuple[str, str | None, str | None]], + list[list[tuple[str, str, str | None]]], + list[tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]], + ], + /, +) -> tuple[ + tuple[ + list[list[tuple[str, str]]], + list[tuple[str | None, str | None] | None], + list[tuple[str | None, str | None, str | None, str | None]], + list[list[tuple[str, str]]], + list[list[tuple[str, str]]], + list[tuple[str, str]], + ] + | None, + str | None, +]: ... # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py new file mode 100644 index 0000000000..260ee5f815 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py @@ -0,0 +1,65 @@ +"""Model placeholders and the model and hook SQL Python's assembly validates.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile._helpers.analysis.validation import hook_sql_statements +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import CompileModelInput + +_HOOK_NAMES: tuple[str, ...] = ("pre_hooks", "post_hooks") + + +def model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: + """Return the model's `@@@name` placeholder defaults as strings, or None.""" + + raw_placeholders: object | None = model_input.config.values.get("placeholders") + return ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + + +def model_syntax_checks( + *, + model_inputs: tuple[CompileModelInput, ...], + analysis_model_names: frozenset[str] | None, + analysis_succeeded: frozenset[str], +) -> tuple[tuple[tuple[str, dict[str, str] | None], ...], ...]: + """Return each model's `(sql, placeholders)` that assembly would pass to syntax validation.""" + + return tuple( + _syntax_checks( + model_input=model_input, + analysis_model_names=analysis_model_names, + analysis_succeeded=analysis_succeeded, + ) + for model_input in model_inputs + ) + + +def _syntax_checks( + *, + model_input: CompileModelInput, + analysis_model_names: frozenset[str] | None, + analysis_succeeded: frozenset[str], +) -> tuple[tuple[str, dict[str, str] | None], ...]: + model_name: str = model_input.model_file.file_path.stem + if not model_input.sql_validation_enabled or ( + analysis_model_names is not None and model_name not in analysis_model_names + ): + return () + placeholders: dict[str, str] | None = model_placeholders(model_input) + statements: list[str] = [] + for hook_name in _HOOK_NAMES: + statements.extend(hook_sql_statements(model_input.config.values.get(hook_name))) + if model_name not in analysis_succeeded: + statements.append( + cursor_intrinsics_analysis_sql( + sql=model_input.query_sql, + cursor_type=model_input.config.values.get("cursor_type"), + ) + ) + return tuple((statement, placeholders) for statement in statements) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py index dbb1e9d2b0..8ef57f338b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py @@ -144,6 +144,20 @@ def validate_hook_sql_syntax( ) +def hook_sql_statements(value: object) -> tuple[str, ...]: + """Return the hook SQL strings `validate_hook_sql_syntax` validates, in its order.""" + + if isinstance(value, str): + return (value,) + if isinstance(value, SqlHookEntry): + return hook_sql_statements(value.statement) + statements: list[str] = [] + if isinstance(value, list | tuple): + for item in value: + statements.extend(hook_sql_statements(item)) + return tuple(statements) + + def validate_source_expression_syntax( *, expression: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index b4360ccdd3..87bf84e5ab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -34,6 +34,9 @@ from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( analyze_dynamic_column_contract, ) +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( + model_placeholders as _model_placeholders, +) from sqlbuild.compiler.compile._helpers.analysis.validation import ( validate_hook_sql_syntax, validate_sql_syntax, @@ -81,9 +84,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( analyze_model_sql_by_engine, - assemble_project_by_engine, dynamic_column_contract_by_engine, expression_source_shapes_by_engine, + project_resources_by_engine, ) from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( assemble_sql_tests_by_engine, @@ -163,6 +166,7 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main.record import record_compile_timing +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.models import ScopeIndex @@ -215,16 +219,6 @@ def assemble_compiled_project( ) -> CompiledProject: """Convert attached compile inputs into the planner-ready project view.""" - native_project: CompiledProject | None = assemble_project_by_engine( - inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, - ) - if native_project is not None: - return native_project sql_analysis_enabled: bool = ( inputs.effective_settings.sql_analysis and not skip_column_inference ) @@ -360,6 +354,16 @@ def assemble_compiled_project( python_analysis=python_analysis, dynamic_families_by_table=dynamic_families_by_table, ) + native: NativeProjectResources | None = project_resources_by_engine( + inputs=inputs, + dialect=profile.sql_analysis_dialect, + analysis_model_names=analysis_model_names, + analysis_succeeded=frozenset( + name + for name, analysis in model_sql_analysis_by_name.items() + if analysis.polyglot_analysis.analysis_succeeded + ), + ) scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -412,7 +416,7 @@ def assemble_compiled_project( sql_validation_enabled=( analysis_model_names is None or _model_name(model_input) in analysis_model_names ), - seed_names=seed_names, + native_deps=native.model_deps[index] if native else None, column_nullability_by_table=column_nullability_by_table, column_types_by_table=column_types_by_table, dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, @@ -420,15 +424,16 @@ def assemble_compiled_project( sql_analysis=model_sql_analysis_by_name.get(model_input.model_file.file_path.stem), allow_compact_analysis=allow_compact_analysis, ) - for model_input in inputs.model_inputs + for index, model_input in enumerate(inputs.model_inputs) ), sources=tuple( _assemble_compiled_source( source_input=source_input, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, + native_entry=native.source_entries[index] if native else None, ) - for source_input in inputs.source_inputs + for index, source_input in enumerate(inputs.source_inputs) ), seeds=tuple( _assemble_compiled_seed( @@ -436,14 +441,25 @@ def assemble_compiled_project( defaults=inputs.project_config.defaults, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, + native_destination=native.seed_destinations[index] if native else None, ) - for seed_input in inputs.seed_inputs + for index, seed_input in enumerate(inputs.seed_inputs) ), functions=tuple( - _assemble_compiled_function(function_input=function_input, seed_names=seed_names) - for function_input in inputs.sql_function_inputs + _assemble_compiled_function( + function_input=function_input, + seed_names=seed_names, + native_deps=native.function_deps[index] if native else None, + ) + for index, function_input in enumerate(inputs.sql_function_inputs) + ), + audits=tuple( + _assemble_compiled_audit( + audit_input=audit_input, + native_scope_deps=native.audit_scope_deps[index] if native else None, + ) + for index, audit_input in enumerate(inputs.audit_inputs) ), - audits=tuple(_assemble_compiled_audit(audit_input) for audit_input in inputs.audit_inputs), sql_tests=assemble_sql_tests_by_engine( inputs=inputs, assemble_python_test=lambda test_input: _assemble_compiled_sql_test( @@ -503,7 +519,7 @@ def _assemble_compiled_model( model_input: CompileModelInput, sql_analysis_enabled: bool, sql_validation_enabled: bool = True, - seed_names: frozenset[str] = frozenset(), + native_deps: tuple[CompiledObjectKey, ...] | None = None, column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, column_types_by_table: dict[str, dict[str, str]] | None = None, dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, @@ -512,6 +528,7 @@ def _assemble_compiled_model( allow_compact_analysis: bool = False, ) -> CompiledModel: model_name: str = model_input.model_file.file_path.stem + syntax_validated: bool = native_deps is not None profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() analysis_query_sql: str = cursor_intrinsics_analysis_sql( sql=model_input.query_sql, @@ -524,7 +541,7 @@ def _assemble_compiled_model( placeholders: dict[str, str] | None = ( sql_analysis.placeholders if sql_analysis is not None else _model_placeholders(model_input) ) - if sql_validation_enabled and model_input.sql_validation_enabled: + if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: for hook_name in ("pre_hooks", "post_hooks"): validate_hook_sql_syntax( value=model_input.config.values.get(hook_name), @@ -554,7 +571,7 @@ def _assemble_compiled_model( fast_lineage_has_star = polyglot_analysis.has_star fast_lineage_star_resolved = polyglot_analysis.star_resolved else: - if model_input.sql_validation_enabled: + if model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, model_name=model_name, @@ -568,7 +585,7 @@ def _assemble_compiled_model( column_nullability_by_table=column_nullability_by_table, inference_profile=profile, ) - elif sql_validation_enabled and model_input.sql_validation_enabled: + elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, model_name=model_name, @@ -613,7 +630,11 @@ def _assemble_compiled_model( fast_lineage_has_star = True return CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=model_name), - deps=model_build_deps(references=model_input.references, seed_names=seed_names), + deps=( + native_deps + if native_deps is not None + else model_build_deps(references=model_input.references) + ), name=model_name, relative_path=model_input.model_file.relative_path, query_sql=model_input.query_sql, @@ -1367,15 +1388,6 @@ def _model_sql_analysis_request( ) -def _model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: - raw_placeholders: object | None = model_input.config.values.get("placeholders") - return ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - - def _should_recover_cte_facts(model_input: CompileModelInput) -> bool: return model_input.config.values.get("contract") == ContractPolicy.ENFORCED or ( model_input.schema_entry is not None and bool(model_input.schema_entry.type_enforcement) @@ -1496,11 +1508,16 @@ def _assemble_compiled_source( source_input: CompileSourceInput, target_config: TargetConfig | None, effective_vars: dict[str, object], + native_entry: SourceEntry | None = None, ) -> CompiledSource: - source_entry: SourceEntry = _build_source_relation_entry( - source_entry=source_input.source_entry, - target_config=target_config, - effective_vars=effective_vars, + source_entry: SourceEntry = ( + native_entry + if native_entry is not None + else _build_source_relation_entry( + source_entry=source_input.source_entry, + target_config=target_config, + effective_vars=effective_vars, + ) ) return CompiledSource( key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=source_entry.name), @@ -1572,12 +1589,17 @@ def _assemble_compiled_seed( defaults: DefaultsConfig, target_config: TargetConfig | None, effective_vars: dict[str, object], + native_destination: CompiledRelationLocation | None = None, ) -> CompiledSeed: - target: CompiledRelationLocation = build_seed_relation_target( - seed_entry=seed_input.schema_entry, - defaults=defaults, - target_config=target_config, - effective_vars=effective_vars, + target: CompiledRelationLocation = ( + native_destination + if native_destination is not None + else build_seed_relation_target( + seed_entry=seed_input.schema_entry, + defaults=defaults, + target_config=target_config, + effective_vars=effective_vars, + ) ) return CompiledSeed( key=CompiledObjectKey( @@ -1596,6 +1618,7 @@ def _assemble_compiled_function( *, function_input: CompileSqlFunctionInput, seed_names: frozenset[str] = frozenset(), + native_deps: tuple[CompiledObjectKey, ...] | None = None, ) -> CompiledFunction: return CompiledFunction( key=CompiledObjectKey( @@ -1604,7 +1627,11 @@ def _assemble_compiled_function( ), name=function_input.name, ), - deps=function_build_deps(references=function_input.references, seed_names=seed_names), + deps=( + native_deps + if native_deps is not None + else function_build_deps(references=function_input.references, seed_names=seed_names) + ), name=function_input.name, relative_path=function_input.function_file.relative_path, arguments=function_input.arguments, @@ -1638,17 +1665,25 @@ def _assemble_compiled_function( ) -def _assemble_compiled_audit(audit_input: CompileAuditInput) -> CompiledAudit: +def _assemble_compiled_audit( + *, + audit_input: CompileAuditInput, + native_scope_deps: tuple[CompiledObjectKey, ...] | None = None, +) -> CompiledAudit: audit_name: str = _resolve_audit_name(audit_input) normalized_target_kind: AttachedAuditTargetKind | None = None if audit_input.attached_target_kind is not None: normalized_target_kind = AttachedAuditTargetKind(audit_input.attached_target_kind) return CompiledAudit( key=CompiledObjectKey(resource_type=CompiledResourceType.AUDIT, name=audit_name), - scope_deps=audit_scope_deps( - references=audit_input.references, - attached_target_kind=audit_input.attached_target_kind, - attached_target_name=audit_input.attached_target_name, + scope_deps=( + native_scope_deps + if native_scope_deps is not None + else audit_scope_deps( + references=audit_input.references, + attached_target_kind=audit_input.attached_target_kind, + attached_target_name=audit_input.attached_target_name, + ) ), name=audit_name, definition_name=(audit_input.audit_block.name or audit_input.audit_file.file_path.stem), diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 145745939d..2c1f1bc21d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -3,7 +3,6 @@ from __future__ import annotations from functools import partial -from pathlib import Path from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( @@ -16,44 +15,43 @@ prove_native_dynamic_contract, ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_syntax_checks from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, ) from sqlbuild.compiler.compile.models import ( - CompiledProject, CompileProjectInputs, DynamicColumnContractProof, ModelSqlAnalysis, ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.project_assembly.main._assemble_native_project import ( - assemble_native_project, +from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( + assemble_native_project_resources, ) +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily -def assemble_project_by_engine( +def project_resources_by_engine( *, inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile | None, - skip_column_inference: bool, - column_lineage_mode: ColumnLineageMode, - analysis_cache_dir: Path | None, + dialect: str | None, analysis_model_names: frozenset[str] | None, -) -> CompiledProject | None: - """Return the natively assembled project, or None where Python must assemble it.""" + analysis_succeeded: frozenset[str], +) -> NativeProjectResources | None: + """Return the natively assembled resource facts, or None where Python must assemble them.""" if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): return None - return assemble_native_project( + return assemble_native_project_resources( inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, + dialect=dialect, + syntax_checks=model_syntax_checks( + model_inputs=inputs.model_inputs, + analysis_model_names=analysis_model_names, + analysis_succeeded=analysis_succeeded, + ), ) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py new file mode 100644 index 0000000000..2a1bd2a676 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native project assembly helpers.""" diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py b/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py new file mode 100644 index 0000000000..296c742539 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/deferrals.py @@ -0,0 +1,24 @@ +"""Debug-only record of the project assembly work native hands back to Python.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from sqlbuild.compiler.lineage.constants import ANALYSIS_DEFERRAL_RECORD_PREFIX +from sqlbuild.compiler.project_assembly.constants import PROJECT_ASSEMBLY_DEFERRAL_SITE +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR + + +def record_project_assembly_deferral(*, kind: str) -> None: + """Append one `{"kind", "site"}` line when the analysis record directory is set.""" + + directory: str | None = os.environ.get(ANALYSIS_RECORD_DIR_ENV_VAR) + if not directory: + return + target: Path = Path(directory) + target.mkdir(parents=True, exist_ok=True) + record: str = json.dumps({"kind": kind, "site": PROJECT_ASSEMBLY_DEFERRAL_SITE}) + with open(target / f"{ANALYSIS_DEFERRAL_RECORD_PREFIX}{os.getpid()}.jsonl", "a") as handle: + _ = handle.write(record + "\n") diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py new file mode 100644 index 0000000000..145c5a8964 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/request.py @@ -0,0 +1,141 @@ +"""Build the native project assembly request from Python's compile inputs.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Iterable + +from sqlbuild.compiler.compile.models import CompileProjectInputs, CompileSqlReference +from sqlbuild.compiler.project_assembly.constants import ( + BOOLEAN_VARIABLE, + ENVIRONMENT_NAME_PATTERN, + FALSE_TEXT, + NATIVE_INT_MAX, + NATIVE_INT_MIN, + NONE_VARIABLE, + TEXT_VARIABLE, + TRUE_TEXT, + UNSUPPORTED_VARIABLE, +) +from sqlbuild.compiler.project_assembly.types import ( + AuditRow, + ProjectRequestRow, + ReferenceRow, + SyntaxCheckRow, + VariableRow, +) +from sqlbuild.spec.contracts.models import DefaultsConfig, TargetConfig + + +def project_request( + *, + inputs: CompileProjectInputs, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> ProjectRequestRow: + """Return the request row; `syntax_checks` holds each model's SQL Python would validate.""" + + target: TargetConfig | None = inputs.effective_target + defaults: DefaultsConfig = inputs.project_config.defaults + target_values: tuple[str | None, ...] = ( + (target.database, target.schema, target.loader_schema) if target is not None else () + ) + default_values: tuple[str | None, ...] = ( + defaults.database, + defaults.schema, + defaults.seed_database, + defaults.seed_schema, + ) + seed_values: list[str | None] = [] + for seed_input in inputs.seed_inputs: + seed_values.extend((seed_input.schema_entry.database, seed_input.schema_entry.schema)) + return ( + dialect or "generic", + (target.database, target.schema, target.loader_schema) if target is not None else None, + (defaults.database, defaults.schema, defaults.seed_database, defaults.seed_schema), + [_variable_row(name=name, value=value) for name, value in inputs.effective_vars.items()], + _environment_rows((*target_values, *default_values, *seed_values)), + [ + (_reference_rows(model_input.references), _syntax_check_rows(checks)) + for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) + ], + [ + ( + source_input.source_entry.name, + source_input.source_entry.loader is not None, + source_input.source_entry.database, + source_input.source_entry.schema, + ) + for source_input in inputs.source_inputs + ], + [ + ( + seed_input.schema_entry.name, + seed_input.schema_entry.database, + seed_input.schema_entry.schema, + ) + for seed_input in inputs.seed_inputs + ], + [ + _reference_rows(function_input.references) + for function_input in inputs.sql_function_inputs + ], + [ + _audit_row( + references=audit_input.references, + attached_kind=audit_input.attached_target_kind, + attached_name=audit_input.attached_target_name, + ) + for audit_input in inputs.audit_inputs + ], + ) + + +def _environment_rows(values: Iterable[str | None]) -> list[tuple[str, str | None]]: + """Python's own `os.environ` lookup of every name an `ENV:` template may read.""" + + names: dict[str, None] = {} + for value in values: + if value is not None: + names.update(dict.fromkeys(re.findall(ENVIRONMENT_NAME_PATTERN, value))) + return [(name, os.environ.get(name)) for name in names] + + +def _variable_row(*, name: str, value: object) -> VariableRow: + if value is None: + return (name, NONE_VARIABLE, "") + if type(value) is bool: + return (name, BOOLEAN_VARIABLE, TRUE_TEXT if value else FALSE_TEXT) + if type(value) is str: + return (name, TEXT_VARIABLE, value) + if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: + return (name, TEXT_VARIABLE, str(value)) + return (name, UNSUPPORTED_VARIABLE, "") + + +def _reference_rows(references: tuple[CompileSqlReference, ...]) -> list[ReferenceRow]: + return [ + (str(reference.ref_kind), reference.ref_name, reference.ref_package) + for reference in references + ] + + +def _syntax_check_rows( + checks: tuple[tuple[str, dict[str, str] | None], ...], +) -> list[SyntaxCheckRow]: + return [(sql, list((placeholders or {}).items())) for sql, placeholders in checks] + + +def _audit_row( + *, + references: tuple[CompileSqlReference, ...], + attached_kind: object, + attached_name: str | None, +) -> AuditRow: + attached: tuple[str, str] | None = ( + (str(attached_kind), attached_name) + if attached_kind is not None and attached_name is not None + else None + ) + return (_reference_rows(references), attached) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py new file mode 100644 index 0000000000..4475507e1e --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py @@ -0,0 +1,81 @@ +"""Materialise the native project assembly result as the objects Python's assembly builds.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, + CompileProjectInputs, + CompileSeedInput, + CompileSourceInput, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.project_assembly.constants import ENVIRONMENT_READ +from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.types import ( + NamespaceRow, + ObjectKeyRow, + ProjectResourcesRow, +) +from sqlbuild.spec.contracts.models import SourceEntry + + +def project_resources( + *, inputs: CompileProjectInputs, row: ProjectResourcesRow +) -> NativeProjectResources: + """Return the resource facts the native row describes, in input order.""" + + model_deps, source_namespaces, seed_namespaces, function_deps, audit_deps, reads = row + _ = [_replay_read(kind=kind, name=name) for kind, name in reads] + return NativeProjectResources( + model_deps=tuple(_keys(keys) for keys in model_deps), + source_entries=tuple( + _source_entry(source_input=source_input, namespace=namespace) + for source_input, namespace in zip(inputs.source_inputs, source_namespaces, strict=True) + ), + seed_destinations=tuple( + _seed_destination(seed_input=seed_input, namespace=namespace) + for seed_input, namespace in zip(inputs.seed_inputs, seed_namespaces, strict=True) + ), + function_deps=tuple(_keys(keys) for keys in function_deps), + audit_scope_deps=tuple(_keys(keys) for keys in audit_deps), + ) + + +def _replay_read(*, kind: str, name: str) -> None: + if kind == ENVIRONMENT_READ: + COMPILE_INPUT_READS.environment_read(name) + else: + COMPILE_INPUT_READS.context_read(name) + + +def _keys(rows: list[ObjectKeyRow]) -> tuple[CompiledObjectKey, ...]: + return tuple( + CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) + for kind, name in rows + ) + + +def _source_entry( + *, source_input: CompileSourceInput, namespace: tuple[str | None, str | None] | None +) -> SourceEntry: + if namespace is None: + return source_input.source_entry + return replace(source_input.source_entry, database=namespace[0], schema=namespace[1]) + + +def _seed_destination( + *, seed_input: CompileSeedInput, namespace: NamespaceRow +) -> CompiledRelationLocation: + database, schema, logical_database, logical_schema = namespace + return CompiledRelationLocation( + database=database, + schema=schema, + name=seed_input.schema_entry.name, + qualified_name=None, + logical_schema=logical_schema, + logical_database=logical_database, + ) diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py new file mode 100644 index 0000000000..9b809a1ecd --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/constants.py @@ -0,0 +1,13 @@ +"""Native project assembly request kinds and deferral record site.""" + +PROJECT_ASSEMBLY_DEFERRAL_SITE: str = "project_assembly" +TEXT_VARIABLE: str = "text" +BOOLEAN_VARIABLE: str = "bool" +NONE_VARIABLE: str = "none" +UNSUPPORTED_VARIABLE: str = "unsupported" +TRUE_TEXT: str = "1" +FALSE_TEXT: str = "0" +NATIVE_INT_MIN: int = -(2**63) +NATIVE_INT_MAX: int = 2**63 - 1 +ENVIRONMENT_READ: str = "env" +ENVIRONMENT_NAME_PATTERN: str = r"ENV:([^\s(),'\"}]+)" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py deleted file mode 100644 index 74ba3e90fb..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Assemble the compiled project natively for the preview compiler engine.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs -from sqlbuild.compiler.lineage.types import ColumnLineageMode - - -def assemble_native_project( - *, - inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile | None, - skip_column_inference: bool, - column_lineage_mode: ColumnLineageMode, - analysis_cache_dir: Path | None, - analysis_model_names: frozenset[str] | None, -) -> CompiledProject | None: - """Return the compiled project, or None where Python must assemble it.""" - - return None diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py new file mode 100644 index 0000000000..fd3b5d63d0 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py @@ -0,0 +1,30 @@ +"""Assemble the compiled project's resource facts natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.project_assembly._helpers.deferrals import record_project_assembly_deferral +from sqlbuild.compiler.project_assembly._helpers.request import project_request +from sqlbuild.compiler.project_assembly._helpers.resources import project_resources +from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.types import ProjectResourcesRow + + +def assemble_native_project_resources( + *, + inputs: CompileProjectInputs, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> NativeProjectResources | None: + """Return the resource facts, or None with a deferral record where Python must assemble.""" + + row: ProjectResourcesRow | None + deferral: str | None + row, deferral = _native.assemble_project_resource_facts( + project_request(inputs=inputs, dialect=dialect, syntax_checks=syntax_checks) + ) + if row is None: + _ = record_project_assembly_deferral(kind=deferral or "unknown") + return None + return project_resources(inputs=inputs, row=row) diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py new file mode 100644 index 0000000000..8c6631c2cf --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -0,0 +1,19 @@ +"""Resource facts the native project assembly returns, as Python's assembly builds them.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.spec.contracts.models import SourceEntry + + +@dataclass(frozen=True, slots=True) +class NativeProjectResources: + """Per-resource facts in input order; every requested SQL string passed syntax validation.""" + + model_deps: tuple[tuple[CompiledObjectKey, ...], ...] + source_entries: tuple[SourceEntry, ...] + seed_destinations: tuple[CompiledRelationLocation, ...] + function_deps: tuple[tuple[CompiledObjectKey, ...], ...] + audit_scope_deps: tuple[tuple[CompiledObjectKey, ...], ...] diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py new file mode 100644 index 0000000000..a3e8ccf62b --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/types.py @@ -0,0 +1,35 @@ +"""Plain-data rows exchanged with the native project assembly.""" + +from __future__ import annotations + +type ReferenceRow = tuple[str, str, str | None] +type SyntaxCheckRow = tuple[str, list[tuple[str, str]]] +type VariableRow = tuple[str, str, str] +type ModelRow = tuple[list[ReferenceRow], list[SyntaxCheckRow]] +type SourceRow = tuple[str, bool, str | None, str | None] +type SeedRow = tuple[str, str | None, str | None] +type AuditRow = tuple[list[ReferenceRow], tuple[str, str] | None] +type TargetRow = tuple[str | None, str | None, str | None] +type DefaultsRow = tuple[str | None, str | None, str | None, str | None] +type ProjectRequestRow = tuple[ + str, + TargetRow | None, + DefaultsRow, + list[VariableRow], + list[tuple[str, str | None]], + list[ModelRow], + list[SourceRow], + list[SeedRow], + list[list[ReferenceRow]], + list[AuditRow], +] +type ObjectKeyRow = tuple[str, str] +type NamespaceRow = tuple[str | None, str | None, str | None, str | None] +type ProjectResourcesRow = tuple[ + list[list[ObjectKeyRow]], + list[tuple[str | None, str | None] | None], + list[NamespaceRow], + list[list[ObjectKeyRow]], + list[list[ObjectKeyRow]], + list[tuple[str, str]], +] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 0d3a7c4dc8..c66fd2bc8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4882,7 +4882,7 @@ def engine_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_stages, "assemble_native_project"), + (native_stages, "assemble_native_project_resources"), (native_stages, "infer_native_expression_source_shapes"), (native_stages, "analyze_native_model_sql"), (native_sql_test_stage, "assemble_native_sql_tests"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index cf88cb5277..bd27e5c2d5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, ) @@ -32,7 +33,7 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_preview_returns={ - "assemble_native_project": [None], + "assemble_native_project_resources": [ANY], "infer_native_expression_source_shapes": [ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [None], @@ -79,6 +80,7 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ test_case.expected_preview_returns, ) assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) + assert isinstance(preview_seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/__init__.py b/tests/integration/src/sqlbuild/compiler/project_assembly/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py new file mode 100644 index 0000000000..680d93254c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedAssemblyParityTestCase: + """Seeded generated projects whose resource assembly both engines must agree on.""" + + description: str + seed: int + count: int + model_count: int + environment: dict[str, str] + expected_minimum_python_calls: int + expected_minimum_environment_reads: int + + +@dataclass(frozen=True) +class AssemblyDeferralTestCase: + """A project native hands back to Python, recording why, with Python's exact outcome.""" + + description: str + files: dict[str, str] + expected_kind: str + expected_error: str + + +@dataclass(frozen=True) +class DeferredAssemblyTestCase: + """A project native hands back to Python, which assembles it without error.""" + + description: str + effective_vars: dict[str, object] + seed_schema: str + expected_schema: str + expected_kind: str + + +@dataclass(frozen=True) +class WindowsEnvironmentTestCase: + """A seed schema reading ENV through a mapping that upper-cases keys, as Windows does.""" + + description: str + environment: dict[str, str] + seed_schema: str + expected_schema: str diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py new file mode 100644 index 0000000000..05d2a1188c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -0,0 +1,238 @@ +"""Generated projects and engine views for native project assembly parity.""" + +from __future__ import annotations + +import os +import random +from collections import Counter +from collections.abc import Callable +from operator import methodcaller +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.assembly import project as project_module +from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, + CompiledProject, + CompileProjectInputs, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import deferral_kinds + +_PYTHON_RESOURCE_FUNCTIONS: tuple[str, ...] = ( + "_build_source_relation_entry", + "build_seed_relation_target", + "validate_sql_syntax", + "validate_hook_sql_syntax", + "model_build_deps", + "function_build_deps", + "audit_scope_deps", +) +_ASSEMBLY_RECORD_PREFIX: str = "project_assembly:" +_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, +) +_DEFAULTS: tuple[str, ...] = ( + "", + 'seed_schema = "raw"\n', + 'database = "db_${env_name}"\nschema = "main"\n', + 'seed_database = "${n}"\nseed_schema = "seeds_${env_name}"\n', +) +_TARGETS: tuple[str, ...] = ( + 'schema = "dev_${env_name}"\nloader_schema = "loaders"\n', + 'database = "wh_${n}"\nschema = "main"\n', + 'schema = "dev"\n', + 'database = "wh"\nschema = "analytics"\nloader_schema = "raw_${flag}"\n', + "schema = \"${coalesce(ENV:SQB_ASSEMBLY_UNSET, 'main')}\"\n" + 'loader_schema = "${ENV:SQB_ASSEMBLY_LOADERS}"\n', +) +_SEED_NAMESPACES: tuple[str, ...] = ( + "", + " schema: seeds_${env_name}\n", + ' schema: "${CTX:model.name}_s"\n', + ' database: "${n}"\n schema: "${CTX:model.database}_x"\n', + ' schema: "${flag}"\n', + ' schema: "${ENV:SQB_ASSEMBLY_LOADERS}_${CTX:model.alias}"\n', +) +_SOURCE_NAMESPACES: tuple[str, ...] = ("", " schema: landing\n", " database: lake\n") +_HOOKS: tuple[str, ...] = ( + "", + ' pre_hooks [inline_sql("SELECT 0 AS n")],\n', + " post_hooks [inline_sql('SELECT 1'), inline_sql('SELECT 2')],\n", + " pre_hooks [inline_sql('SELECT 3')],\n post_hooks [inline_sql('SELECT 4')],\n", +) +_AUDITS: tuple[str, ...] = ("", " columns (n (audits [not_null])),\n") + + +def generated_assembly_files(*, rng: random.Random, model_count: int) -> dict[str, str]: + """Return a project with managed sources, templated seeds, hooks, audits and a function.""" + + references: list[str] = [ + '__source("raw_orders")', + '__source("landing_orders")', + '__seed("countries")', + '__seed("regions")', + *[f'__ref("orders_{index}")' for index in range(model_count - 1)], + ] + models: dict[str, str] = { + f"models/orders_{index}.sql": ( + f'MODEL (\n description "Generated orders model {index}",\n' + f"{rng.choice(_HOOKS)}{rng.choice(_AUDITS)});\n\n" + f"SELECT 1 AS n FROM {rng.choice(references[: 4 + index])}\n" + ) + for index in range(model_count) + } + return { + "sqlbuild_project.toml": ( + 'name = "assembly"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n' + '[vars]\nenv_name = "prod"\nn = "3"\nflag = "true"\n\n' + f"[defaults]\n{rng.choice(_DEFAULTS)}\n[targets.dev]\n{rng.choice(_TARGETS)}" + ), + "seeds/lookups.yml": ( + "seeds:\n" + f" - name: countries\n description: Countries.\n{rng.choice(_SEED_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + f" - name: regions\n description: Regions.\n{rng.choice(_SEED_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + ), + "seeds/countries.csv": "n\n1\n", + "seeds/regions.csv": "n\n2\n", + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Raw orders.\n managed: true\n" + f" write_strategy: table\n{rng.choice(_SOURCE_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + " - name: landing_orders\n description: Landing orders.\n" + f"{rng.choice(_SOURCE_NAMESPACES)}" + " columns:\n - name: n\n type: INTEGER\n" + ), + "python/loaders/raw.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\ndef raw_orders(ctx):\n '''Load raw orders.'''\n return [{'n': 1}]\n" + ), + "functions/sql/order_numbers.sql": ( + 'FUNCTION (\n description "Order numbers.",\n arguments (p_n INTEGER),\n' + " returns table (n INTEGER)\n);\n\n" + 'SELECT n FROM __ref("orders_0") WHERE n = p_n\n' + ), + **models, + } + + +def windows_environ(environment: dict[str, str]) -> os._Environ[str]: + """Return this process's environment plus `environment` with Windows' upper-cased keys.""" + + data: dict[str, str] = { + name.upper(): value for name, value in {**os.environ, **environment}.items() + } + return os._Environ(data, str.upper, str, str, str) + + +def seed_yml(schema: str) -> str: + """Return a one-seed declaration whose schema is `schema`.""" + + return ( + "seeds:\n - name: countries\n description: Countries.\n" + f' schema: "{schema}"\n columns:\n - name: n\n type: INTEGER\n' + ) + + +def project_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileProjectInputs: + """Write `files` and attach their compile inputs as the compile pipeline does.""" + + _ = [ + _write(path=project_dir / relative_path, contents=contents) + for relative_path, contents in files.items() + ] + return build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=_ADAPTER_CONTEXT, + run_id="integration_run", + defer_model_sql_validation=True, + ) + + +def assemble_with( + *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> CompiledProject: + """Assemble `inputs` with `engine`.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return assemble_compiled_project(inputs=inputs) + + +def recorded_assembly( + *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> tuple[object, tuple[tuple[str, ...], bool]]: + """Assemble `inputs` with `engine`; return its assembly view and the input reads recorded.""" + + with COMPILE_INPUT_READS.recording() as reads: + project: CompiledProject = assemble_with( + inputs=inputs, engine=engine, monkeypatch=monkeypatch + ) + return assembly_view(project), (reads.environment_names, reads.read_run_id) + + +def assembly_view(project: CompiledProject) -> object: + """Return every resource fact native project assembly derives, with the diagnostics.""" + + return ( + tuple((model.name, model.deps, model.destination) for model in project.models), + tuple(source.source_entry for source in project.sources), + tuple(seed.destination for seed in project.seeds), + tuple((function.name, function.deps) for function in project.functions), + tuple((audit.name, audit.scope_deps) for audit in project.audits), + project.diagnostics, + ) + + +def assembly_deferrals(record_dir: Path) -> Counter[str]: + """Count the project assembly `site:kind` deferral records below `record_dir`.""" + + return Counter( + filter( + methodcaller("startswith", _ASSEMBLY_RECORD_PREFIX), + deferral_kinds(record_dir).elements(), + ) + ) + + +def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count calls to Python's resource-fact functions from project assembly.""" + + calls: Counter[str] = Counter() + _ = [ + monkeypatch.setattr( + project_module, + name, + _counted(name=name, function=getattr(project_module, name), calls=calls), + ) + for name in _PYTHON_RESOURCE_FUNCTIONS + ] + return calls + + +def _write(*, path: Path, contents: str) -> int: + path.parent.mkdir(parents=True, exist_ok=True) + return path.write_text(contents, encoding="utf-8") + + +def _counted(*, name: str, function: Callable[..., Any], calls: Counter[str]) -> Callable[..., Any]: + def counted(*args: Any, **kwargs: Any) -> Any: + calls[name] += 1 + return function(*args, **kwargs) + + return counted diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py new file mode 100644 index 0000000000..5b9ef5783e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -0,0 +1,263 @@ +"""Native project assembly equals Python's resource assembly, or hands the project back.""" + +from __future__ import annotations + +import os +import random +from collections import Counter +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.project_assembly._test_types import ( + AssemblyDeferralTestCase, + DeferredAssemblyTestCase, + GeneratedAssemblyParityTestCase, + WindowsEnvironmentTestCase, +) +from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( + assemble_with, + assembly_deferrals, + assembly_view, + generated_assembly_files, + project_inputs, + python_resource_calls, + recorded_assembly, + seed_yml, + windows_environ, +) + +_PROJECT_TOML: str = ( + 'name = "assembly"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n\n' +) +_ORDERS_MODEL: str = 'MODEL (description "Orders");\n\nSELECT 1 AS n\n' +_SEED_FILES: dict[str, str] = { + "seeds/countries.csv": "n\n1\n", + "models/orders.sql": _ORDERS_MODEL, +} + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedAssemblyParityTestCase( + description="managed sources, templated seeds and targets, hooks, audits, a function", + seed=20261009, + count=8, + model_count=12, + environment={"SQB_ASSEMBLY_LOADERS": "loaders"}, + expected_minimum_python_calls=200, + expected_minimum_environment_reads=4, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_projects_when_assembling_natively_then_matches_python( + test_case: GeneratedAssemblyParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + _ = [monkeypatch.setenv(name, value) for name, value in test_case.environment.items()] + calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) + rng: random.Random = random.Random(test_case.seed) + inputs: list[CompileProjectInputs] = [ + project_inputs( + project_dir=tmp_path / f"project_{index}", + files=generated_assembly_files(rng=rng, model_count=test_case.model_count), + ) + for index in range(test_case.count) + ] + python: list[tuple[object, tuple[tuple[str, ...], bool]]] = [ + recorded_assembly(inputs=item, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + for item in inputs + ] + python_calls: int = calls.total() + native: list[tuple[object, tuple[tuple[str, ...], bool]]] = [ + recorded_assembly( + inputs=item, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + for item in inputs + ] + + assert ( + mismatches( + inputs=list(range(test_case.count)), + expected=list[object](python), + actual=list[object](native), + ) + == [] + ) + assert sum(len(reads[0]) for _, reads in native) >= test_case.expected_minimum_environment_reads + assert python_calls >= test_case.expected_minimum_python_calls + assert calls.total() == python_calls + assert assembly_deferrals(record_dir) == Counter() + + +@pytest.mark.parametrize( + "test_case", + [ + AssemblyDeferralTestCase( + description="model SQL the parser rejects", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": 'MODEL (description "Orders");\n\nSELECT FROM WHERE (\n', + }, + expected_kind="syntax_error", + expected_error="SQL syntax error in model 'orders'", + ), + AssemblyDeferralTestCase( + description="hook SQL the parser rejects", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + 'MODEL (\n description "Orders",\n pre_hooks [inline_sql("SELECT FROM (")],\n' + ");\n\nSELECT 1 AS n\n" + ), + }, + expected_kind="syntax_error", + expected_error="Polyglot could not parse model 'orders' pre_hooks[0]", + ), + AssemblyDeferralTestCase( + description="a seed schema reading an unset environment variable", + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml("${ENV:SQB_UNSET_SEED_SCHEMA}"), + **_SEED_FILES, + }, + expected_kind="target_template", + expected_error="references missing ENV variable 'SQB_UNSET_SEED_SCHEMA'", + ), + AssemblyDeferralTestCase( + description="a preserved target schema the seed does not own", + files={ + "sqlbuild_project.toml": ( + 'name = "assembly"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "preserve"\n' + ), + "seeds/lookups.yml": ( + "seeds:\n - name: countries\n description: Countries.\n" + " columns:\n - name: n\n type: INTEGER\n" + ), + "seeds/countries.csv": "n\n1\n", + "models/orders.sql": ( + 'MODEL (description "Orders", schema "orders");\n\nSELECT 1 AS n\n' + ), + }, + expected_kind="preserved_namespace", + expected_error="Seed 'countries' has no logical schema", + ), + ], + ids=lambda case: case.description, +) +def test_given_projects_python_rejects_when_assembling_natively_then_defers_with_python_error( + test_case: AssemblyDeferralTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = project_inputs(project_dir=tmp_path, files=test_case.files) + + with pytest.raises(CompileInputError) as python_error: + _ = assemble_with(inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + with pytest.raises(CompileInputError) as native_error: + _ = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert test_case.expected_error in str(python_error.value) + assert str(native_error.value) == str(python_error.value) + assert assembly_deferrals(record_dir) == Counter( + {f"project_assembly:{test_case.expected_kind}": 1} + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeferredAssemblyTestCase( + description="a seed schema reading a list variable", + effective_vars={"tags": ["daily"]}, + seed_schema="${tags}", + expected_schema="['daily']", + expected_kind="target_template", + ) + ], + ids=lambda case: case.description, +) +def test_given_a_variable_only_python_renders_when_assembling_natively_then_python_assembles( + test_case: DeferredAssemblyTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = replace( + project_inputs( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml(test_case.seed_schema), + **_SEED_FILES, + }, + ), + effective_vars=test_case.effective_vars, + ) + + python: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + native: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert assembly_view(native) == assembly_view(python) + assert native.seeds[0].destination.schema == test_case.expected_schema + assert assembly_deferrals(record_dir) == Counter( + {f"project_assembly:{test_case.expected_kind}": 1} + ) + + +@pytest.mark.parametrize( + "test_case", + [ + WindowsEnvironmentTestCase( + description="a lower-case ENV name Windows resolves to an upper-case variable", + environment={"APP_SCHEMA": "landing"}, + seed_schema="${coalesce(ENV:app_schema, 'main')}", + expected_schema="landing", + ) + ], + ids=lambda case: case.description, +) +def test_given_windows_environment_when_assembling_natively_then_env_lookups_match_python( + test_case: WindowsEnvironmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setattr(os, "environ", windows_environ(test_case.environment)) + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + inputs: CompileProjectInputs = project_inputs( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/lookups.yml": seed_yml(test_case.seed_schema), + **_SEED_FILES, + }, + ) + + python: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + native: CompiledProject = assemble_with( + inputs=inputs, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + + assert python.seeds[0].destination.schema == test_case.expected_schema + assert assembly_view(native) == assembly_view(python) + assert assembly_deferrals(record_dir) == Counter()