diff --git a/Cargo.lock b/Cargo.lock index 04152783bb..2d7ab1ff76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1295,6 +1295,7 @@ version = "0.164.1" dependencies = [ "junction", "rayon", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-model-config", diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 3b1132a79f..0cd082a7fc 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-model-config.workspace = true diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs index 7c99418bd0..5b2e5261fc 100644 --- a/crates/sqlbuild-discovery/src/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -4,5 +4,6 @@ pub(crate) mod locations; pub(crate) mod pool; pub(crate) mod reading; pub(crate) mod scoped_paths; +pub(crate) mod sorted_keys; pub(crate) mod statement_headers; pub(crate) mod suggestions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs b/crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs rename to crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs index f377167577..4f04b9db33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs +++ b/crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs @@ -1,4 +1,4 @@ -//! Header key sets as the sorted lists native discovery options take. +//! Header key sets as the sorted lists the discovery options take. /// A header key set as owned keys, sorted by code point as Python sorts them. pub(crate) fn sorted_keys(keys: &[&str]) -> Vec { diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index f1bb0e5c90..e415a42148 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -47,3 +47,124 @@ pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; /// The singular audit directory below an audit role. pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; +/// Header keys a `MODEL` statement accepts, sorted by code point. +pub const SQL_MODEL_HEADER_KEYS: [&str; 73] = [ + "alias", + "append_cursor_inclusive", + "audit_factories", + "audits", + "batch_concurrency", + "batch_size", + "check_columns", + "columns", + "config", + "constants", + "contract", + "cursor", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_grain", + "cursor_inputs", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_type", + "cursor_watermark_mode", + "database", + "description", + "dynamic_columns", + "enabled", + "engine", + "enums", + "full_refresh", + "historical_input", + "incremental_mode", + "incremental_strategy", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "max_duration", + "max_microbatches", + "merge_exclude_columns", + "microbatch_limit", + "microbatch_strategy", + "migrate_force", + "migrate_from", + "model_schema", + "observed_at", + "old_name_view", + "on_schema_change", + "order_by", + "partition_by", + "placeholders", + "post_hooks", + "pre_hooks", + "primary_key", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "sql_analysis", + "sql_validation", + "table_settings", + "table_type", + "tags", + "time_travel_retention", + "unaccounted_partition_policy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +/// `MODEL` header keys that were removed and fail with a dedicated message. +pub const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"]; +/// Header keys a `FUNCTION` statement accepts. +pub const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [ + "arguments", + "database", + "description", + "returns", + "schema", + "tags", +]; +/// Header keys an `AUDIT` statement accepts. +pub const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [ + "always_run", + "evaluation", + "minimum_samples", + "name", + "run_scope", + "sample_count", + "sample_unit", + "severity", + "sql_analysis", + "thresholds", + "value", +]; +/// Header keys a `HOOK` statement accepts. +pub const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"]; +/// Header keys a `TEST` statement accepts. +pub const SQL_TEST_HEADER_KEYS: [&str; 8] = [ + "cases", + "cursor_end", + "cursor_start", + "mode", + "name", + "parameters", + "sql_analysis", + "timeout", +]; +/// Header keys a `SCENARIO` statement accepts. +pub const SQL_SCENARIO_HEADER_KEYS: [&str; 4] = + ["description", "expected_columns", "tags", "timeout"]; +/// `SCENARIO (expected_columns ...)` values: compare listed columns, or require every column. +pub const SCENARIO_EXPECTED_COLUMNS_KEY: &str = "expected_columns"; +pub const SCENARIO_EXPECTED_COLUMNS_VALUES: [&str; 2] = ["listed", "all"]; +pub const SCENARIO_EXPECTED_COLUMNS_ALL: &str = "all"; diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs b/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs new file mode 100644 index 0000000000..c677c9cc6a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs @@ -0,0 +1,15 @@ +//! The options declaration discovery parses headers with. + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{SQL_AUDIT_HEADER_KEYS, SQL_FUNCTION_HEADER_KEYS, SQL_HOOK_HEADER_KEYS}; +use crate::declaration_files::models::DeclarationFileOptions; + +/// The header keys `FUNCTION`, `AUDIT` and `HOOK` statements accept. +#[must_use] +pub fn declaration_file_options() -> DeclarationFileOptions { + DeclarationFileOptions { + function_keys: sorted_keys(&SQL_FUNCTION_HEADER_KEYS), + audit_keys: sorted_keys(&SQL_AUDIT_HEADER_KEYS), + hook_keys: sorted_keys(&SQL_HOOK_HEADER_KEYS), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs index 81483be916..443692bd2e 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod bare_number_failure; +pub mod declaration_file_options; pub mod discover_declaration_collection; pub mod parse_declaration_text; pub mod python_function_files; diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs new file mode 100644 index 0000000000..76aeb91265 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod references; +pub(crate) mod value_encoding; diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs new file mode 100644 index 0000000000..0d218d21c8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs @@ -0,0 +1,39 @@ +//! Whether an edited query still references only what the stored model read. + +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::header_changes::constants::{ + LINE_COMMENT, MODEL_KIND, REF_CALL, SEED_CALL, SEED_KIND, SOURCE_CALL, SOURCE_KIND, +}; + +/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. +pub(crate) fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT.to_owned()], + ..LexicalSyntax::default() + }; + let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { + return false; + }; + scan.references.iter().all(|reference| { + let kind: Option<&str> = match reference.kind { + REF_CALL => Some(MODEL_KIND), + SOURCE_CALL => Some(SOURCE_KIND), + SEED_CALL => Some(SEED_KIND), + _ => None, + }; + match (kind, &reference.package) { + (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), + _ => true, + } + }) +} + +/// Whether `upstream` holds the `kind` resource `name`. +fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { + upstream + .iter() + .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs new file mode 100644 index 0000000000..6d28d570ad --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs @@ -0,0 +1,79 @@ +//! Header values as short digests: an unambiguous byte encoding, hashed and truncated. + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::constants::{HEX, KEPT_DIGEST_BYTES}; + +/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. +#[derive(Default)] +pub(crate) struct ValueEncoding { + bytes: Vec, +} + +impl ValueEncoding { + fn text(&mut self, tag: u8, text: &str) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(text.len() as u64).to_le_bytes()); + self.bytes.extend_from_slice(text.as_bytes()); + } + + fn list(&mut self, tag: u8, items: &[AuthoredValue]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for item in items { + self.value(item); + } + } + + fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for (key, item) in items { + self.text(b'k', key); + self.value(item); + } + } + + pub(crate) fn value(&mut self, value: &AuthoredValue) { + match value { + AuthoredValue::Null => self.bytes.push(b'n'), + AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), + AuthoredValue::BareWord(word) => self.text(b'w', word), + AuthoredValue::String(string) => self.text(b's', string), + AuthoredValue::List(items) => self.list(b'l', items), + AuthoredValue::Map(items) => self.pairs(b'm', items), + AuthoredValue::Set(items) => self.list(b'e', items), + AuthoredValue::Tuple(items) => self.list(b't', items), + AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), + AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), + AuthoredValue::NamedSqlHook(name, items) => { + self.text(b'h', name); + self.pairs(b'a', items); + } + AuthoredValue::PythonHook(name, items) => { + self.text(b'p', name); + self.pairs(b'a', items); + } + } + } +} + +pub(crate) fn short_hex(digest: &[u8]) -> String { + let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); + for byte in &digest[..KEPT_DIGEST_BYTES] { + hex.push(char::from(HEX[usize::from(byte >> 4)])); + hex.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + hex +} + +/// The short digest of one header value. +pub(crate) fn value_digest(value: &AuthoredValue) -> String { + let mut encoding: ValueEncoding = ValueEncoding::default(); + encoding.value(value); + short_hex(&bytes_digest(&encoding.bytes)) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/constants.rs b/crates/sqlbuild-discovery/src/header_changes/constants.rs new file mode 100644 index 0000000000..fcea7e085b --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/constants.rs @@ -0,0 +1,27 @@ +//! The header keys and values a delta compile can recompile a model for. + +/// Header keys a delta compile recompiles the model for; any other changed key falls back. +pub(crate) const PATCHED_KEYS: [&str; 6] = [ + "description", + "columns", + "dynamic_columns", + "model_schema", + "audits", + "contract", +]; +/// `materialized` values a delta compile handles; others name materializations it cannot. +pub(crate) const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +/// Plain `--` comments, which every adapter's SQL reads as comments. +pub(crate) const LINE_COMMENT: &str = "--"; +/// Reference call kinds and the upstream kinds they resolve to. +pub(crate) const REF_CALL: &str = "ref"; +pub(crate) const SOURCE_CALL: &str = "source"; +pub(crate) const SEED_CALL: &str = "seed"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const SEED_KIND: &str = "seed"; + +/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. +pub(crate) const KEPT_DIGEST_BYTES: usize = 16; +pub(crate) const HEX: &[u8; 16] = b"0123456789abcdef"; diff --git a/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs b/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs new file mode 100644 index 0000000000..e833b04d0c --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs @@ -0,0 +1,36 @@ +//! Whether a custom rule reads what changed in a model header. + +use std::collections::BTreeSet; + +/// Whether a custom rule reads a fact of this model that the changed header keys decide. +#[must_use] +pub fn custom_rule_reads( + fact_digests: &[(Vec, String)], + model: (&str, &str), + changed: &BTreeSet, +) -> Option { + let (path, name) = model; + for (key, _) in fact_digests { + let Some((fact, arguments)) = key.split_first() else { + continue; + }; + if !arguments + .iter() + .any(|argument| argument == path || argument == name) + { + continue; + } + let reads: &[&str] = match fact.as_str() { + "contracts.enforced" | "contract" => &["contract"], + "contracts.grain" | "grain" => &["contract", "columns"], + "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], + "materialized" => &["materialized"], + "tests.for_model" | "audits.for_model" => &["audits", "columns"], + _ => &[], + }; + if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { + return Some(format!("a custom rule reads {name}'s {key}")); + } + } + None +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs b/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs new file mode 100644 index 0000000000..10dc646bca --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs @@ -0,0 +1,81 @@ +//! How an edit changed one model file's header. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::_helpers::references::references_read; +use crate::header_changes::constants::{MATERIALIZED_KEY, PATCHED_KEYS, PATCHED_MATERIALIZATIONS}; +use crate::header_changes::main::header_key_digests::header_key_digests; +use crate::header_changes::models::HeaderChange; +use crate::header_changes::types::HeaderKeys; +use crate::model_files::main::model_file_options::model_file_options; +use crate::model_files::main::parse_model_text::parse_model_text; + +/// How the header of the model file at `path` changed against its recorded keys. +#[must_use] +pub fn header_change( + project_dir: &Path, + relative_path: &str, + (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), +) -> HeaderChange { + let Some(stored) = stored else { + return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); + }; + let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { + return HeaderChange::Unparsed; + }; + let file_path: String = project_dir + .join(relative_path) + .to_string_lossy() + .into_owned(); + let Ok(parsed) = parse_model_text( + &file_path, + contents, + &model_file_options((false, false), None), + ) else { + return HeaderChange::Unparsed; + }; + if !references_read(&parsed.query_sql, upstream) { + return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); + } + let current: HeaderKeys = header_key_digests(&parsed.header_values); + let stored_map: BTreeMap<&str, &str> = stored + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let current_map: BTreeMap<&str, &str> = current + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let changed: BTreeSet = stored_map + .keys() + .chain(current_map.keys()) + .filter(|key| stored_map.get(*key) != current_map.get(*key)) + .map(|key| (*key).to_owned()) + .collect(); + if changed.is_empty() { + return HeaderChange::Unchanged; + } + for key in &changed { + if key == MATERIALIZED_KEY { + let value = parsed.header_values.iter().find(|(name, _)| name == key); + let supported: bool = matches!( + value, + Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) + if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) + ); + if !supported { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its materialization beyond views and tables" + )); + } + } else if !PATCHED_KEYS.contains(&key.as_str()) { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its {key} header key" + )); + } + } + HeaderChange::Patched(changed) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs b/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs new file mode 100644 index 0000000000..02ff9c77ab --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs @@ -0,0 +1,17 @@ +//! A model header's keys as short value digests. + +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::_helpers::value_encoding::value_digest; +use crate::header_changes::types::HeaderKeys; + +/// Each header key with its value's short digest, sorted by key. +#[must_use] +pub fn header_key_digests(values: &[(String, AuthoredValue)]) -> HeaderKeys { + let mut keys: HeaderKeys = values + .iter() + .map(|(key, value)| (key.clone(), value_digest(value))) + .collect(); + keys.sort(); + keys +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/mod.rs b/crates/sqlbuild-discovery/src/header_changes/main/mod.rs new file mode 100644 index 0000000000..9d20ee7ce1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/mod.rs @@ -0,0 +1,3 @@ +pub mod custom_rule_reads; +pub mod header_change; +pub mod header_key_digests; diff --git a/crates/sqlbuild-discovery/src/header_changes/mod.rs b/crates/sqlbuild-discovery/src/header_changes/mod.rs new file mode 100644 index 0000000000..15398bd447 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/mod.rs @@ -0,0 +1,10 @@ +//! Model header changes: each header's keys as digests, and which keys an edit changed. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/header_changes/models.rs b/crates/sqlbuild-discovery/src/header_changes/models.rs new file mode 100644 index 0000000000..a8c12f79dd --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/models.rs @@ -0,0 +1,16 @@ +//! What an edit did to a model header. + +use std::collections::BTreeSet; + +/// What an edit did to one model's header. +#[derive(Debug, PartialEq, Eq)] +pub enum HeaderChange { + /// The header's keys and values are unchanged. + Unchanged, + /// These keys changed, every one of which the delta recompiles. + Patched(BTreeSet), + /// Why the delta cannot recompile this header change. + Unsupported(String), + /// The file does not parse; the delta reports its error as a full compile would. + Unparsed, +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests.rs b/crates/sqlbuild-discovery/src/header_changes/tests.rs new file mode 100644 index 0000000000..412e8eb6d9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_custom_rule_reads.rs"] +mod custom_rule_reads; +#[path = "tests/test_header_change.rs"] +mod header_change; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs b/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs new file mode 100644 index 0000000000..a3b65c38e5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs @@ -0,0 +1,42 @@ +use std::collections::BTreeSet; + +use crate::header_changes::main::header_change::header_change; +use crate::header_changes::main::header_key_digests::header_key_digests; +use crate::header_changes::models::HeaderChange; +use crate::header_changes::tests::test_types::HeaderChangeTestCase; +use crate::model_files::main::model_file_options::model_file_options; +use crate::model_files::main::parse_model_text::parse_model_text; + +const RELATIVE_PATH: &str = "models/orders.sql"; +const UPSTREAM: [(&str, &str); 1] = [("model", "customers")]; + +/// The change of a recorded model after its file is rewritten, or deleted when none is given. +pub(super) fn edited_change(test_case: &HeaderChangeTestCase) -> HeaderChange { + let project = tempfile::tempdir().expect("project directory"); + let path = project.path().join(RELATIVE_PATH); + std::fs::create_dir_all(path.parent().expect("models directory")).expect("models directory"); + let stored = parse_model_text( + &path.to_string_lossy(), + test_case.stored_contents.to_owned(), + &model_file_options((false, false), None), + ) + .expect("stored model parses"); + let keys = header_key_digests(&stored.header_values); + let upstream: Vec<(String, String)> = UPSTREAM + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(); + let _ = test_case + .edited_contents + .map(|contents| std::fs::write(&path, contents).expect("edited model")); + header_change( + project.path(), + RELATIVE_PATH, + (test_case.recorded.then_some(&keys), &upstream), + ) +} + +/// The set of changed keys a header change names. +pub(super) fn keys(names: &[&str]) -> BTreeSet { + names.iter().map(|name| (*name).to_owned()).collect() +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs new file mode 100644 index 0000000000..c64d0166e6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs @@ -0,0 +1,48 @@ +use crate::header_changes::main::custom_rule_reads::custom_rule_reads; +use crate::header_changes::tests::helpers::keys; +use crate::header_changes::tests::test_types::CustomRuleReadsTestCase; + +#[test] +fn given_custom_rule_facts_when_a_header_changes_then_reads_of_changed_keys_are_named() { + let test_cases = [ + CustomRuleReadsTestCase { + description: "a fact of the model deciding a changed key is named", + fact_key: vec!["columns.names", "orders"], + changed: vec!["columns"], + expected_reason: Some("a custom rule reads orders's columns"), + }, + CustomRuleReadsTestCase { + description: "a fact of another model is ignored", + fact_key: vec!["columns.names", "customers"], + changed: vec!["columns"], + expected_reason: None, + }, + CustomRuleReadsTestCase { + description: "a fact no changed key decides is ignored", + fact_key: vec!["materialized", "models/orders.sql"], + changed: vec!["description"], + expected_reason: None, + }, + ]; + for test_case in test_cases { + let facts = vec![( + test_case + .fact_key + .iter() + .map(|part| (*part).to_owned()) + .collect::>(), + "digest".to_owned(), + )]; + assert_eq!( + custom_rule_reads( + &facts, + ("models/orders.sql", "orders"), + &keys(&test_case.changed) + ) + .as_deref(), + test_case.expected_reason, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs new file mode 100644 index 0000000000..a919ad92fd --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs @@ -0,0 +1,85 @@ +use crate::header_changes::models::HeaderChange; +use crate::header_changes::tests::helpers::{edited_change, keys}; +use crate::header_changes::tests::test_types::HeaderChangeTestCase; + +const STORED: &str = + "MODEL (materialized view, description 'orders');\nSELECT id FROM __ref(\"customers\")"; + +#[test] +fn given_an_edited_model_header_when_compared_then_the_change_is_classified() { + let test_cases = [ + HeaderChangeTestCase { + description: "an unrecorded model is unsupported", + stored_contents: STORED, + edited_contents: Some(STORED), + recorded: false, + expected_change: HeaderChange::Unsupported( + "models/orders.sql has no recorded header".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "an unreadable file is unparsed", + stored_contents: STORED, + edited_contents: None, + recorded: true, + expected_change: HeaderChange::Unparsed, + }, + HeaderChangeTestCase { + description: "the same header is unchanged", + stored_contents: STORED, + edited_contents: Some(STORED), + recorded: true, + expected_change: HeaderChange::Unchanged, + }, + HeaderChangeTestCase { + description: "a changed description and a table materialization are patched", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized table, description 'all orders');\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Patched(keys(&["description", "materialized"])), + }, + HeaderChangeTestCase { + description: "an incremental materialization is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized incremental, description 'orders');\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed its materialization beyond views and tables".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "another changed key is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized view, description 'orders', unique_key id);\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed its unique_key header key".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "a new reference is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized view, description 'orders');\nSELECT id FROM __ref(\"products\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed what it references".to_owned(), + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + edited_change(&test_case), + test_case.expected_change, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs new file mode 100644 index 0000000000..7efd8e2582 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::header_changes::models::HeaderChange; + +pub(super) struct HeaderChangeTestCase { + pub(super) description: &'static str, + pub(super) stored_contents: &'static str, + pub(super) edited_contents: Option<&'static str>, + pub(super) recorded: bool, + pub(super) expected_change: HeaderChange, +} + +pub(super) struct CustomRuleReadsTestCase { + pub(super) description: &'static str, + pub(super) fact_key: Vec<&'static str>, + pub(super) changed: Vec<&'static str>, + pub(super) expected_reason: Option<&'static str>, +} diff --git a/crates/sqlbuild-discovery/src/header_changes/types.rs b/crates/sqlbuild-discovery/src/header_changes/types.rs new file mode 100644 index 0000000000..817f5bc85d --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/types.rs @@ -0,0 +1,4 @@ +//! The recorded header of one model. + +/// Each header key with its value's short digest, sorted by key. +pub type HeaderKeys = Vec<(String, String)>; diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 9226c314a6..0f9a9b8350 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -6,6 +6,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod declaration_files; pub mod declarations; +pub mod header_changes; pub mod key_checks; pub mod model_files; pub mod models; diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index eb6f515e35..4d936ccc0a 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,4 +1,5 @@ pub mod discover_model_files; pub mod header_facts; +pub mod model_file_options; pub mod model_file_stem; pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs b/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs new file mode 100644 index 0000000000..a0e201f27f --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs @@ -0,0 +1,24 @@ +//! The options model discovery parses headers with. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS}; +use crate::model_files::models::ModelFileOptions; + +/// Model header options; `extraction` is `(implicit alias columns, output locations)`. +#[must_use] +pub fn model_file_options( + extraction: (bool, bool), + path_filter: Option>>, +) -> ModelFileOptions { + let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; + ModelFileOptions { + supported_keys: sorted_keys(&SQL_MODEL_HEADER_KEYS), + removed_keys: sorted_keys(&REMOVED_SQL_MODEL_HEADER_KEYS), + extract_implicit_alias_columns, + extract_output_column_locations, + path_filter, + } +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs index 032628c3f1..87039a540f 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs @@ -2,3 +2,4 @@ pub mod discover_scenario_files; pub mod discover_sql_test_files; pub mod parse_scenario_text; pub mod parse_sql_test_text; +pub mod sql_test_file_options; diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs b/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs new file mode 100644 index 0000000000..d6cae443d8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs @@ -0,0 +1,18 @@ +//! The options SQL test and scenario discovery parses headers with. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{SQL_SCENARIO_HEADER_KEYS, SQL_TEST_HEADER_KEYS}; +use crate::sql_tests::models::SqlTestFileOptions; + +/// The header keys `TEST` and `SCENARIO` statements accept, discovering only `path_filter`. +#[must_use] +pub fn sql_test_file_options(path_filter: Option>>) -> SqlTestFileOptions { + SqlTestFileOptions { + test_keys: sorted_keys(&SQL_TEST_HEADER_KEYS), + scenario_keys: sorted_keys(&SQL_SCENARIO_HEADER_KEYS), + path_filter, + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs new file mode 100644 index 0000000000..f1c51ec966 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs @@ -0,0 +1,146 @@ +//! Config values read as authored values, as `PyNode` reads the Python objects they mirror. + +use crate::config_values::_helpers::reprs::{python_repr, python_str}; +use crate::config_values::models::{ConfigNode, HostChildren}; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +impl<'a, H: HostObject> AuthoredNode for &'a ConfigNode { + fn kind(&self) -> NodeKind { + match *self { + ConfigNode::Null => NodeKind::Null, + ConfigNode::Bool(flag) => NodeKind::Bool(*flag), + ConfigNode::Int(number) => NodeKind::Int { + negative: *number < 0, + }, + ConfigNode::Float(_) => NodeKind::Other, + ConfigNode::Str(_) => NodeKind::Str, + ConfigNode::List(_) => NodeKind::List, + ConfigNode::Tuple(_) => NodeKind::Tuple, + ConfigNode::Map(_) => NodeKind::Map, + ConfigNode::Host(host) => host.facts.kind, + } + } + + fn integer(&self) -> Option { + match *self { + ConfigNode::Int(number) => Some(*number), + ConfigNode::Host(host) => host.facts.integer, + _ => None, + } + } + + /// Python's `float(int)`, which rounds the decimal value to the nearest float, as parsing does. + fn number(&self) -> Option { + match *self { + ConfigNode::Int(number) => Some(*number as f64), + ConfigNode::Float(number) => Some(*number), + ConfigNode::Host(host) => host.facts.number, + _ => None, + } + } + + fn text(&self) -> Option { + self.with_text(str::to_owned) + } + + fn is_text(&self, text: &str) -> bool { + self.with_text(|value| value == text).unwrap_or(false) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + match *self { + ConfigNode::Str(text) => Some(read(text)), + ConfigNode::Host(host) => host.facts.text.as_deref().map(read), + _ => None, + } + } + + fn python_str(&self) -> String { + python_str(self) + } + + fn python_repr(&self) -> String { + python_repr(self) + } + + fn is_date_like(&self) -> bool { + matches!(self, ConfigNode::Host(host) if host.handle.is_date_like()) + } + + fn items(&self) -> Vec { + let node: &'a ConfigNode = self; + match node { + ConfigNode::List(items) | ConfigNode::Tuple(items) => items.iter().collect(), + ConfigNode::Host(host) => match &host.children { + HostChildren::Items(items) => items.iter().collect(), + _ => Vec::new(), + }, + _ => Vec::new(), + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + let node: &'a ConfigNode = self; + match node { + ConfigNode::Map(entries) => entries.iter().map(|(key, value)| (key, value)).collect(), + ConfigNode::Host(host) => match &host.children { + HostChildren::Entries(entries) => { + entries.iter().map(|(key, value)| (key, value)).collect() + } + _ => Vec::new(), + }, + _ => Vec::new(), + } + } +} + +impl AuthoredNode for ConfigNode { + fn kind(&self) -> NodeKind { + (&self).kind() + } + + fn integer(&self) -> Option { + (&self).integer() + } + + fn number(&self) -> Option { + (&self).number() + } + + fn text(&self) -> Option { + (&self).text() + } + + fn is_text(&self, text: &str) -> bool { + (&self).is_text(text) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + (&self).with_text(read) + } + + fn python_str(&self) -> String { + python_str(self) + } + + fn python_repr(&self) -> String { + python_repr(self) + } + + fn is_date_like(&self) -> bool { + (&self).is_date_like() + } + + fn items(&self) -> Vec { + (&self).items().into_iter().cloned().collect() + } + + fn entries(&self) -> Vec<(Self, Self)> { + (&self) + .entries() + .into_iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs new file mode 100644 index 0000000000..98fb958caf --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs @@ -0,0 +1,22 @@ +//! The authored value of a config key and the type error a reader raises for it. + +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; + +/// `values.get(key)` unless it is `None`. +pub(crate) fn authored(value: Option<&ConfigNode>) -> Option<&ConfigNode> { + value.filter(|value| !matches!(value, ConfigNode::Null)) +} + +/// The error naming `key`, the `expected` kind and the value found. +pub(crate) fn wrong_type<'a, H>( + key: &str, + expected: &'static str, + value: &'a ConfigNode, +) -> ConfigTypeError<'a, H> { + ConfigTypeError { + key: key.to_owned(), + expected, + value, + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs new file mode 100644 index 0000000000..3ece308db8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs @@ -0,0 +1,75 @@ +//! A config mapping read and written as a Python `dict`. + +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// The exact text of a string key, borrowed. +fn key_text(node: &ConfigNode) -> Option<&str> { + match node { + ConfigNode::Str(text) => Some(text), + ConfigNode::Host(host) if host.facts.kind == NodeKind::Str && host.facts.strict_text => { + host.facts.text.as_deref() + } + _ => None, + } +} + +/// Python's `==` between two dict keys: equal text, equal numbers, or equal tuples. +pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { + match (left, right) { + (ConfigNode::Null, ConfigNode::Null) => true, + (ConfigNode::Tuple(left), ConfigNode::Tuple(right)) => { + left.len() == right.len() && left.iter().zip(right).all(|(a, b)| same_key(a, b)) + } + _ => match (key_text(left), key_text(right)) { + (Some(left), Some(right)) => left == right, + (None, None) => match (numeric(left), numeric(right)) { + (Some(left), Some(right)) => left == right, + _ => false, + }, + _ => false, + }, + } +} + +/// A boolean, integer or float key's numeric value, as Python compares keys. +fn numeric(node: &ConfigNode) -> Option { + match node { + ConfigNode::Bool(flag) => Some(f64::from(u8::from(*flag))), + ConfigNode::Int(_) | ConfigNode::Float(_) => node.number(), + _ => None, + } +} + +/// `mapping.get(key)` for a string key. +pub(crate) fn entry<'a, H: HostObject>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + entries + .iter() + .find(|(name, _)| key_text(name) == Some(key)) + .map(|(_, value)| value) +} + +/// `mapping.get(key)` unless absent or `None`. +pub(crate) fn set_entry<'a, H: HostObject>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + entry(entries, key).filter(|value| !matches!(value, ConfigNode::Null)) +} + +/// `entries` after `mapping[key] = value`: an equal key keeps its place and its key, else the entry is appended. +pub(crate) fn inserted( + mut entries: Vec<(ConfigNode, ConfigNode)>, + key: ConfigNode, + value: ConfigNode, +) -> Vec<(ConfigNode, ConfigNode)> { + match entries.iter().position(|(name, _)| same_key(name, &key)) { + Some(position) => entries[position].1 = value, + None => entries.push((key, value)), + } + entries +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs new file mode 100644 index 0000000000..42fe2ead92 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod authored; +pub(crate) mod checks; +pub(crate) mod entries; +pub(crate) mod reprs; +pub(crate) mod unions; diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs new file mode 100644 index 0000000000..edabcf2939 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs @@ -0,0 +1,30 @@ +//! Python's `str()` and `repr()` of config values: plain scalars in Rust, the rest by Python. + +use sqlbuild_core::json::main::python_float_repr::python_float_repr; + +use crate::config_values::constants::{FALSE_TEXT, NONE_TEXT, TRUE_TEXT}; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; + +/// Python's exact `str()` (or `repr()`) of a plain scalar, or `None` for any other value. +pub(crate) fn scalar_text(value: &ConfigNode, repr: bool) -> Option { + match value { + ConfigNode::Null => Some(NONE_TEXT.to_owned()), + ConfigNode::Bool(true) => Some(TRUE_TEXT.to_owned()), + ConfigNode::Bool(false) => Some(FALSE_TEXT.to_owned()), + ConfigNode::Int(number) => Some(number.to_string()), + ConfigNode::Float(number) => Some(python_float_repr(*number)), + ConfigNode::Str(text) if !repr => Some(text.clone()), + _ => None, + } +} + +/// Python's `repr(value)`. +pub(crate) fn python_repr(value: &ConfigNode) -> String { + scalar_text(value, true).unwrap_or_else(|| H::display(value, true)) +} + +/// Python's `str(value)`. +pub(crate) fn python_str(value: &ConfigNode) -> String { + scalar_text(value, false).unwrap_or_else(|| H::display(value, false)) +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs new file mode 100644 index 0000000000..d1b29d2747 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs @@ -0,0 +1,79 @@ +//! The unions `_merged_with_tag_union` makes of string sequences and row-diff tolerances. + +use crate::config_values::_helpers::entries::{inserted, set_entry}; +use crate::config_values::_helpers::reprs::scalar_text; +use crate::config_values::constants::AUDIT_OVERRIDE_SECTIONS; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::{HostObject, ItemText}; +use crate::types::{AuthoredNode, NodeKind}; + +/// `_as_string_list(value)`: `str()` of each list or tuple item. +fn string_items( + value: &ConfigNode, + text: ItemText<'_, H, E>, +) -> Result, E> { + match value.kind() { + NodeKind::List | NodeKind::Tuple => (&value) + .items() + .into_iter() + .map(|item| scalar_text(item, false).map_or_else(|| text(item), Ok)) + .collect(), + _ => Ok(Vec::new()), + } +} + +/// The strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. +pub(crate) fn merged_strings( + base: &ConfigNode, + overlay: &ConfigNode, + text: ItemText<'_, H, E>, +) -> Result>, E> { + let mut merged: Vec = string_items(base, text)?; + for item in string_items(overlay, text)? { + if !merged.contains(&item) { + merged.push(item); + } + } + Ok(merged.into_iter().map(ConfigNode::Str).collect()) +} + +/// Row-diff tolerances deep-merged by section, as `_merge_row_diff_tolerances_mapping`. +pub(crate) fn merged_tolerances( + base: &ConfigNode, + overlay: &ConfigNode, +) -> ConfigNode { + if base.kind() != NodeKind::Map || overlay.kind() != NodeKind::Map { + return overlay.clone(); + } + let (base, overlay) = (base.entries(), overlay.entries()); + let mut merged = base.clone(); + for section in AUDIT_OVERRIDE_SECTIONS { + let Some(overlay_section) = set_entry(&overlay, section) else { + continue; + }; + let value: ConfigNode = match set_entry(&base, section) { + Some(base_section) + if base_section.kind() == NodeKind::Map + && overlay_section.kind() == NodeKind::Map => + { + let mut values = base_section.entries(); + for (key, item) in overlay_section.entries() { + values = inserted(values, key, item); + } + ConfigNode::Map(values) + } + _ => overlay_section.clone(), + }; + merged = inserted(merged, ConfigNode::Str(section.to_owned()), value); + } + for (key, item) in &overlay { + if key.kind() == NodeKind::Str + && !AUDIT_OVERRIDE_SECTIONS + .iter() + .any(|section| key.is_text(section)) + { + merged = inserted(merged, key.clone(), item.clone()); + } + } + ConfigNode::Map(merged) +} diff --git a/crates/sqlbuild-model-config/src/config_values/constants.rs b/crates/sqlbuild-model-config/src/config_values/constants.rs new file mode 100644 index 0000000000..7eff286ac1 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/constants.rs @@ -0,0 +1,15 @@ +//! Python's literal spellings of the plain config values. + +pub(crate) const NONE_TEXT: &str = "None"; +pub(crate) const TRUE_TEXT: &str = "True"; +pub(crate) const FALSE_TEXT: &str = "False"; +pub(crate) const TAGS_KEY: &str = "tags"; +pub(crate) const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; +pub(crate) const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; +/// The `row_diff_tolerances` sections merged one level deeper than the rest. +pub(crate) const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; +pub(crate) const EXPECTED_TEXT: &str = "a string"; +pub(crate) const EXPECTED_FLAG: &str = "a boolean"; +pub(crate) const EXPECTED_TEXTS: &str = "a list or tuple of strings"; +pub(crate) const EXPECTED_INTEGER: &str = "an integer"; +pub(crate) const EXPECTED_CURSOR_BOUND: &str = "a string, integer, date, or datetime"; diff --git a/crates/sqlbuild-model-config/src/config_values/errors.rs b/crates/sqlbuild-model-config/src/config_values/errors.rs new file mode 100644 index 0000000000..389606e98b --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/errors.rs @@ -0,0 +1,11 @@ +//! The error a typed config read returns. + +use crate::config_values::models::ConfigNode; + +/// A config value of the wrong type: its key, the kind expected, and the value found. +#[derive(Clone, Debug, PartialEq)] +pub struct ConfigTypeError<'a, H> { + pub key: String, + pub expected: &'static str, + pub value: &'a ConfigNode, +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs new file mode 100644 index 0000000000..3527ee3580 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs @@ -0,0 +1,25 @@ +//! A config cursor bound read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_CURSOR_BOUND; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::{ConfigNode, CursorBound}; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// How `key` reads as a cursor bound: a string, a non-boolean integer, a date or a datetime. +pub fn checked_cursor_bound<'a, H: HostObject>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + if value.is_date_like() { + return Ok(Some(CursorBound::Dated(value))); + } + match value.kind() { + NodeKind::Str | NodeKind::Int { .. } => Ok(Some(CursorBound::Stringified(value))), + _ => Err(wrong_type(key, EXPECTED_CURSOR_BOUND, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs new file mode 100644 index 0000000000..af12ea2733 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs @@ -0,0 +1,22 @@ +//! A config boolean read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_FLAG; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// The boolean value of `key`, or `None` when absent or `None`. +pub fn checked_flag<'a, H: HostObject>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Bool(flag) => Ok(Some(flag)), + _ => Err(wrong_type(key, EXPECTED_FLAG, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs new file mode 100644 index 0000000000..577a321f84 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs @@ -0,0 +1,22 @@ +//! A config integer read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_INTEGER; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// The value of `key` when it is an integer and not a boolean, or `None` when absent or `None`. +pub fn checked_integer<'a, H: HostObject>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Int { .. } => Ok(Some(value)), + _ => Err(wrong_type(key, EXPECTED_INTEGER, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs new file mode 100644 index 0000000000..81b9a3e85f --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs @@ -0,0 +1,22 @@ +//! A config string read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_TEXT; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// The value of `key` when it is a string (a present empty string included), or `None` when absent or `None`. +pub fn checked_text<'a, H: HostObject>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Str => Ok(Some(value)), + _ => Err(wrong_type(key, EXPECTED_TEXT, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs new file mode 100644 index 0000000000..bcbbe3368d --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs @@ -0,0 +1,27 @@ +//! A config sequence of strings read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_TEXTS; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::types::{AuthoredNode, NodeKind}; + +/// The string items of `key` when it is a list or tuple of strings, or `None` when absent or `None`. +pub fn checked_texts<'a, H: HostObject>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + let items: Vec<&ConfigNode> = match value.kind() { + NodeKind::List | NodeKind::Tuple => (&value).items(), + _ => return Err(wrong_type(key, EXPECTED_TEXTS, value)), + }; + if items.iter().all(|item| item.kind() == NodeKind::Str) { + Ok(Some(items)) + } else { + Err(wrong_type(key, EXPECTED_TEXTS, value)) + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs new file mode 100644 index 0000000000..f22b150cc8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs @@ -0,0 +1,14 @@ +//! One entry of a config mapping. + +use crate::config_values::_helpers::entries::entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; + +/// Python's `mapping.get(key)` for a string `key`. +#[must_use] +pub fn config_entry<'a, H: HostObject>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + entry(entries, key) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs new file mode 100644 index 0000000000..f1ede71f58 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs @@ -0,0 +1,16 @@ +//! A config mapping with one entry set. + +use crate::config_values::_helpers::entries::inserted; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; + +/// `entries` after `mapping[key] = value`: an equal key keeps its place, else the entry is appended. +#[must_use] +pub fn inserted_entry( + entries: ConfigEntries, + key: ConfigNode, + value: ConfigNode, +) -> ConfigEntries { + inserted(entries, key, value) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs new file mode 100644 index 0000000000..d02967215f --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs @@ -0,0 +1,54 @@ +//! One config layer merged over another, as `_merged_with_tag_union` merges them. + +use crate::config_values::_helpers::entries::{inserted, set_entry}; +use crate::config_values::_helpers::unions::{merged_strings, merged_tolerances}; +use crate::config_values::constants::{ + ROW_DIFF_EXCLUDE_COLUMNS_KEY, ROW_DIFF_TOLERANCES_KEY, TAGS_KEY, +}; +use crate::config_values::models::{ConfigNode, LayeredConfig}; +use crate::config_values::types::{HostObject, ItemText}; +use crate::types::{AuthoredNode, NodeKind}; + +/// `overlay` over a copy of `base`, unioning tags, row-diff exclusions and tolerances; `text` prints non-scalar items. +pub fn layered_config( + base: &[(ConfigNode, ConfigNode)], + overlay: &[(ConfigNode, ConfigNode)], + text: ItemText<'_, H, E>, +) -> Result, E> { + let mut entries = base.to_vec(); + for (key, value) in overlay { + entries = inserted(entries, key.clone(), value.clone()); + } + let mut unioned_keys: Vec<&'static str> = Vec::new(); + for (key, tuple) in [(TAGS_KEY, false), (ROW_DIFF_EXCLUDE_COLUMNS_KEY, true)] { + if let (Some(base_items), Some(overlay_items)) = + (set_entry(base, key), set_entry(overlay, key)) + { + let items: Vec> = merged_strings(base_items, overlay_items, text)?; + let merged: ConfigNode = if tuple { + ConfigNode::Tuple(items) + } else { + ConfigNode::List(items) + }; + entries = inserted(entries, ConfigNode::Str(key.to_owned()), merged); + unioned_keys.push(key); + } + } + if let (Some(base_tolerances), Some(overlay_tolerances)) = ( + set_entry(base, ROW_DIFF_TOLERANCES_KEY), + set_entry(overlay, ROW_DIFF_TOLERANCES_KEY), + ) { + entries = inserted( + entries, + ConfigNode::Str(ROW_DIFF_TOLERANCES_KEY.to_owned()), + merged_tolerances(base_tolerances, overlay_tolerances), + ); + if base_tolerances.kind() == NodeKind::Map && overlay_tolerances.kind() == NodeKind::Map { + unioned_keys.push(ROW_DIFF_TOLERANCES_KEY); + } + } + Ok(LayeredConfig { + entries, + unioned_keys, + }) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/mod.rs b/crates/sqlbuild-model-config/src/config_values/main/mod.rs new file mode 100644 index 0000000000..a16884acfe --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/mod.rs @@ -0,0 +1,11 @@ +pub mod checked_cursor_bound; +pub mod checked_flag; +pub mod checked_integer; +pub mod checked_text; +pub mod checked_texts; +pub mod config_entry; +pub mod inserted_entry; +pub mod layered_config; +pub mod plain_scalar_text; +pub mod same_config_key; +pub mod set_config_entry; diff --git a/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs b/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs new file mode 100644 index 0000000000..4c78b6425d --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs @@ -0,0 +1,10 @@ +//! Python's text of a plain scalar config value. + +use crate::config_values::_helpers::reprs::scalar_text; +use crate::config_values::models::ConfigNode; + +/// Python's exact `str()` (or `repr()`) of `None`, a boolean, an integer, a float or a string's `str()`; else `None`. +#[must_use] +pub fn plain_scalar_text(value: &ConfigNode, repr: bool) -> Option { + scalar_text(value, repr) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs new file mode 100644 index 0000000000..012d815e0f --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs @@ -0,0 +1,11 @@ +//! Python's equality between two config mapping keys. + +use crate::config_values::_helpers::entries::same_key; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; + +/// Whether two dict keys are equal: equal text, equal numbers, or equal tuples. +#[must_use] +pub fn same_config_key(left: &ConfigNode, right: &ConfigNode) -> bool { + same_key(left, right) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs new file mode 100644 index 0000000000..3d0d68c948 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs @@ -0,0 +1,14 @@ +//! One set entry of a config mapping. + +use crate::config_values::_helpers::entries::set_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; + +/// Python's `mapping.get(key)` for a string `key`, unless it is absent or `None`. +#[must_use] +pub fn set_config_entry<'a, H: HostObject>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + set_entry(entries, key) +} diff --git a/crates/sqlbuild-model-config/src/config_values/mod.rs b/crates/sqlbuild-model-config/src/config_values/mod.rs new file mode 100644 index 0000000000..b326b8b35a --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/mod.rs @@ -0,0 +1,11 @@ +//! Model config values as Rust data: plain Python builtins typed, every other value an opaque host value. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/config_values/models.rs b/crates/sqlbuild-model-config/src/config_values/models.rs new file mode 100644 index 0000000000..1d1f6df825 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/models.rs @@ -0,0 +1,64 @@ +//! One model config value. + +use crate::types::NodeKind; + +/// A config value: the builtin Python types it is made of, or a value only its host understands. +#[derive(Clone, Debug, PartialEq)] +pub enum ConfigNode { + Null, + Bool(bool), + /// An `int` within `i64`. + Int(i64), + Float(f64), + /// A `str` that is valid UTF-8. + Str(String), + List(Vec), + Tuple(Vec), + /// A `dict`'s entries in insertion order. + Map(Vec<(Self, Self)>), + /// Any other value, carried by its host handle and read through the facts taken from it. + Host(Box>), +} + +/// An opaque value: its host handle, what it reads as, and its items when it is a container. +#[derive(Clone, Debug, PartialEq)] +pub struct HostValue { + pub handle: H, + pub facts: HostFacts, + pub children: HostChildren, +} + +/// The cheap facts read of an opaque value when it is converted; anything else is asked of its handle. +#[derive(Clone, Debug, PartialEq)] +pub struct HostFacts { + pub kind: NodeKind, + /// The text of a string value, with lone surrogates read as U+FFFD. + pub text: Option, + /// Whether `text` is the value's exact text. + pub strict_text: bool, + pub integer: Option, + pub number: Option, +} + +/// The converted items of an opaque list, tuple or mapping, which config logic walks. +#[derive(Clone, Debug, PartialEq)] +pub enum HostChildren { + None, + Items(Vec>), + Entries(Vec<(ConfigNode, ConfigNode)>), +} + +/// One layer merged over another, and which keys a union replaced rather than overrode. +#[derive(Clone, Debug, PartialEq)] +pub struct LayeredConfig { + pub entries: Vec<(ConfigNode, ConfigNode)>, + /// The keys both layers set whose merged value is a union of the two. + pub unioned_keys: Vec<&'static str>, +} + +/// How a cursor bound reads as text: its `str()`, or its `isoformat()` for a date or datetime. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum CursorBound<'a, H> { + Stringified(&'a ConfigNode), + Dated(&'a ConfigNode), +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests.rs b/crates/sqlbuild-model-config/src/config_values/tests.rs new file mode 100644 index 0000000000..8613a52d31 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/test_checked_reads.rs"] +mod checked_reads; +#[path = "tests/test_config_entry.rs"] +mod config_entry; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_layered_config.rs"] +mod layered_config; +#[path = "tests/test_python_text.rs"] +mod python_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs new file mode 100644 index 0000000000..8d2b3921ac --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs @@ -0,0 +1,75 @@ +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::{ConfigNode, CursorBound, HostChildren, HostFacts, HostValue}; +use crate::config_values::tests::test_types::Node; +use crate::types::NodeKind; + +/// A string node. +pub(super) fn text(value: &str) -> Node { + ConfigNode::Str(value.to_owned()) +} + +/// A `StrEnum`-like host value whose text is `value`. +pub(super) fn host_text(value: &str) -> Node { + ConfigNode::Host(Box::new(HostValue { + handle: 1, + facts: HostFacts { + kind: NodeKind::Str, + text: Some(value.to_owned()), + strict_text: true, + integer: None, + number: None, + }, + children: HostChildren::None, + })) +} + +/// A list of string nodes. +pub(super) fn texts(values: &[&str]) -> Node { + ConfigNode::List(values.iter().map(|value| text(value)).collect()) +} + +/// A `datetime.date` host value. +pub(super) fn host_date() -> Node { + ConfigNode::Host(Box::new(HostValue { + handle: 2, + facts: HostFacts { + kind: NodeKind::Other, + text: None, + strict_text: false, + integer: None, + number: None, + }, + children: HostChildren::None, + })) +} + +/// `value`, `none`, or the expected kind of a typed read's outcome. +pub(super) fn outcome(result: Result, ConfigTypeError<'_, u8>>) -> String { + result.map_or_else( + |error| error.expected.to_owned(), + |value| value.map_or("none", |_| "value").to_owned(), + ) +} + +/// `dated`, `stringified`, `none`, or the expected kind of a cursor bound read. +pub(super) fn cursor_outcome( + result: Result>, ConfigTypeError<'_, u8>>, +) -> String { + result.map_or_else( + |error| error.expected.to_owned(), + |bound| { + let kinds = [ + ("dated", matches!(bound, Some(CursorBound::Dated(_)))), + ( + "stringified", + matches!(bound, Some(CursorBound::Stringified(_))), + ), + ]; + kinds + .iter() + .find(|(_, found)| *found) + .map_or("none", |(kind, _)| kind) + .to_owned() + }, + ) +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs new file mode 100644 index 0000000000..7839c18087 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs @@ -0,0 +1,118 @@ +use crate::config_values::main::checked_cursor_bound::checked_cursor_bound; +use crate::config_values::main::checked_flag::checked_flag; +use crate::config_values::main::checked_integer::checked_integer; +use crate::config_values::main::checked_text::checked_text; +use crate::config_values::main::checked_texts::checked_texts; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{ + cursor_outcome, host_date, host_text, outcome, text, texts, +}; +use crate::config_values::tests::test_types::CheckedReadTestCase; + +#[test] +fn given_config_values_when_read_as_typed_values_then_types_are_checked_as_python_does() { + let test_cases = [ + CheckedReadTestCase { + description: "an absent or null value reads as none", + value: Some(ConfigNode::Null), + expected_text: "none", + expected_flag: "none", + expected_integer: "none", + expected_texts: "none", + expected_cursor_bound: "none", + }, + CheckedReadTestCase { + description: "a present empty string is a string", + value: Some(text("")), + expected_text: "value", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a str-enum member is a string", + value: Some(host_text("daily")), + expected_text: "value", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a boolean is not an integer or a cursor bound", + value: Some(ConfigNode::Bool(true)), + expected_text: "a string", + expected_flag: "value", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "an integer reads as an integer and a stringified bound", + value: Some(ConfigNode::Int(-4)), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "value", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a date is a dated cursor bound only", + value: Some(host_date()), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "dated", + }, + CheckedReadTestCase { + description: "a tuple of strings is a string sequence", + value: Some(ConfigNode::Tuple(vec![text("a"), host_text("b")])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "value", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "a list holding a non-string is not a string sequence", + value: Some(ConfigNode::List(vec![text("a"), ConfigNode::Int(1)])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "a list of strings is a string sequence", + value: Some(texts(&["a"])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "value", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + ]; + for test_case in test_cases { + let value = test_case.value.as_ref(); + assert_eq!( + [ + outcome(checked_text("key", value)), + outcome(checked_flag("key", value)), + outcome(checked_integer("key", value)), + outcome(checked_texts("key", value)), + cursor_outcome(checked_cursor_bound("key", value)), + ], + [ + test_case.expected_text, + test_case.expected_flag, + test_case.expected_integer, + test_case.expected_texts, + test_case.expected_cursor_bound, + ], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs new file mode 100644 index 0000000000..3b9239df64 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs @@ -0,0 +1,36 @@ +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{host_text, text}; +use crate::config_values::tests::test_types::ConfigEntryTestCase; + +#[test] +fn given_config_entries_when_read_by_string_key_then_python_dict_equality_applies() { + let test_cases = [ + ConfigEntryTestCase { + description: "a string key is found", + entries: vec![(text("schema"), text("analytics"))], + key: "schema", + expected_value: Some(text("analytics")), + }, + ConfigEntryTestCase { + description: "a str-enum key equals its text", + entries: vec![(host_text("schema"), ConfigNode::Int(1))], + key: "schema", + expected_value: Some(ConfigNode::Int(1)), + }, + ConfigEntryTestCase { + description: "a number key never equals a string", + entries: vec![(ConfigNode::Int(1), text("one"))], + key: "1", + expected_value: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + config_entry(&test_case.entries, test_case.key).cloned(), + test_case.expected_value, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs new file mode 100644 index 0000000000..feba8f1201 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs @@ -0,0 +1,104 @@ +use crate::config_values::main::layered_config::layered_config; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{text, texts}; +use crate::config_values::tests::test_types::{LayeredConfigTestCase, Node}; + +#[test] +fn given_two_layers_when_merged_then_overrides_keep_order_and_unions_merge() { + let test_cases = [ + LayeredConfigTestCase { + description: "an override keeps the key's place; a new key is appended", + base: vec![ + (text("materialized"), text("view")), + (text("schema"), text("raw")), + ], + overlay: vec![ + (text("alias"), text("orders")), + (text("materialized"), text("table")), + ], + expected_entries: vec![ + (text("materialized"), text("table")), + (text("schema"), text("raw")), + (text("alias"), text("orders")), + ], + expected_unioned_keys: vec![], + }, + LayeredConfigTestCase { + description: "tags and row-diff exclusions union as list and tuple", + base: vec![ + (text("tags"), texts(&["daily", "core"])), + ( + text("row_diff_exclude_columns"), + ConfigNode::Tuple(vec![text("updated_at")]), + ), + ], + overlay: vec![ + (text("tags"), texts(&["core", "finance"])), + (text("row_diff_exclude_columns"), texts(&["loaded_at"])), + ], + expected_entries: vec![ + (text("tags"), texts(&["daily", "core", "finance"])), + ( + text("row_diff_exclude_columns"), + ConfigNode::Tuple(vec![text("updated_at"), text("loaded_at")]), + ), + ], + expected_unioned_keys: vec!["tags", "row_diff_exclude_columns"], + }, + LayeredConfigTestCase { + description: "a null overlay tag list overrides without a union", + base: vec![(text("tags"), texts(&["daily"]))], + overlay: vec![(text("tags"), ConfigNode::Null)], + expected_entries: vec![(text("tags"), ConfigNode::Null)], + expected_unioned_keys: vec![], + }, + LayeredConfigTestCase { + description: "tolerance sections merge one level deeper", + base: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![(text("float"), ConfigNode::Float(0.1))]), + ), + (text("default"), ConfigNode::Int(0)), + ]), + )], + overlay: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![(text("decimal"), ConfigNode::Int(1))]), + ), + (text("default"), ConfigNode::Int(2)), + ]), + )], + expected_entries: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![ + (text("float"), ConfigNode::Float(0.1)), + (text("decimal"), ConfigNode::Int(1)), + ]), + ), + (text("default"), ConfigNode::Int(2)), + ]), + )], + expected_unioned_keys: vec!["row_diff_tolerances"], + }, + ]; + for test_case in test_cases { + let text = |_: &Node| -> Result { Err(()) }; + let layered = layered_config(&test_case.base, &test_case.overlay, &text) + .expect("plain scalar unions need no host text"); + assert_eq!( + (layered.entries, layered.unioned_keys), + (test_case.expected_entries, test_case.expected_unioned_keys), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs new file mode 100644 index 0000000000..74ee21a803 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs @@ -0,0 +1,51 @@ +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{host_text, text}; +use crate::config_values::tests::test_types::PythonTextTestCase; +use crate::types::AuthoredNode; + +#[test] +fn given_config_values_when_printed_then_scalars_print_natively_and_the_rest_by_python() { + let test_cases = [ + PythonTextTestCase { + description: "plain scalars print as Python literals", + value: ConfigNode::Float(1e16), + expected_str: "1e+16", + expected_repr: "1e+16", + }, + PythonTextTestCase { + description: "a boolean and None print as Python spells them", + value: ConfigNode::Bool(false), + expected_str: "False", + expected_repr: "False", + }, + PythonTextTestCase { + description: "a string prints itself; its repr is Python's", + value: text("it's\u{200b}"), + expected_str: "it's\u{200b}", + expected_repr: "0", + }, + PythonTextTestCase { + description: "a container prints by Python, never approximated in Rust", + value: ConfigNode::List(vec![host_text("warn"), text("x")]), + expected_str: "0", + expected_repr: "0", + }, + PythonTextTestCase { + description: "a host value prints by Python", + value: host_text("warn"), + expected_str: "1", + expected_repr: "1", + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.value.python_str(), test_case.value.python_repr()), + ( + test_case.expected_str.to_owned(), + test_case.expected_repr.to_owned() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs new file mode 100644 index 0000000000..112ea25acc --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs @@ -0,0 +1,58 @@ +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; + +/// A config node in tests; host handles are plain numbers. +pub(super) type Node = ConfigNode; + +pub(super) struct PythonTextTestCase { + pub(super) description: &'static str, + pub(super) value: Node, + pub(super) expected_str: &'static str, + pub(super) expected_repr: &'static str, +} + +pub(super) struct LayeredConfigTestCase { + pub(super) description: &'static str, + pub(super) base: Vec<(Node, Node)>, + pub(super) overlay: Vec<(Node, Node)>, + pub(super) expected_entries: Vec<(Node, Node)>, + pub(super) expected_unioned_keys: Vec<&'static str>, +} + +pub(super) struct ConfigEntryTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec<(Node, Node)>, + pub(super) key: &'static str, + pub(super) expected_value: Option, +} + +/// One config value and how each typed reader reads it: `value`, `none`, or the expected kind. +pub(super) struct CheckedReadTestCase { + pub(super) description: &'static str, + pub(super) value: Option, + pub(super) expected_text: &'static str, + pub(super) expected_flag: &'static str, + pub(super) expected_integer: &'static str, + pub(super) expected_texts: &'static str, + pub(super) expected_cursor_bound: &'static str, +} + +/// Test handles: `2` is a date; other values print as `` or `` with their handle. +impl HostObject for u8 { + fn is_date_like(&self) -> bool { + *self == 2 + } + + fn display(node: &ConfigNode, repr: bool) -> String { + let kind: &str = ["", ""][usize::from(repr)]; + format!("{kind}{}", node_handle(node)) + } +} + +/// The handle of a host node, or `0` for a value the crate printed nothing of. +fn node_handle(node: &Node) -> u8 { + let ConfigNode::Host(host) = node else { + return 0; + }; + host.handle +} diff --git a/crates/sqlbuild-model-config/src/config_values/types.rs b/crates/sqlbuild-model-config/src/config_values/types.rs new file mode 100644 index 0000000000..1da941f764 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/types.rs @@ -0,0 +1,17 @@ +//! A config mapping's entries. + +use crate::config_values::models::ConfigNode; + +/// A mapping's `(key, value)` entries in insertion order, as a Python `dict` keeps them. +pub type ConfigEntries = Vec<(ConfigNode, ConfigNode)>; + +/// What config logic asks of an opaque value's handle, read only when it is needed. +pub trait HostObject: Clone { + /// Whether the value is a `datetime.date` or `datetime.datetime`. + fn is_date_like(&self) -> bool; + /// Python's `str(node)`, or `repr(node)`, of a value the crate does not print exactly itself. + fn display(node: &ConfigNode, repr: bool) -> String; +} + +/// Python's `str()` of a value that is not a plain scalar, or the error it raises. +pub type ItemText<'t, H, E> = &'t dyn Fn(&ConfigNode) -> Result; diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs index 130e9d67fa..c991f8c080 100644 --- a/crates/sqlbuild-model-config/src/lib.rs +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -3,10 +3,12 @@ #![forbid(unsafe_code)] pub mod config_presence; +pub mod config_values; pub mod declarations; pub mod errors; pub mod execution_limits; pub mod header_metadata; +pub mod model_config_build; pub mod model_validation; pub mod name_patterns; pub mod path_defaults; diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs new file mode 100644 index 0000000000..821b0fad03 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs @@ -0,0 +1,130 @@ +//! The header, hook and macro checks of one model's config build. + +use crate::config_presence::main::entry_macro_path::entry_macro_path; +use crate::config_presence::models::MacroPath; +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::set_config_entry::set_config_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::errors::ConfigError; +use crate::model_config_build::constants::{HOOK_KEYS, SORTED_HOOK_KEYS, TAGS_KEY}; +use crate::model_config_build::models::{ConfigBuildHalt, HeaderEntryFacts}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt, HostNode}; +use crate::types::{AuthoredNode, NodeKind}; + +/// The `CompileInputError` `model '' `. +pub(crate) fn model_error(model_name: &str, text: &str) -> ConfigError { + ConfigError::compile(format!("model '{model_name}' {text}")) +} + +/// `_validate_model_header_tags`. +pub(crate) fn check_header_tags( + header: &[(ConfigNode, ConfigNode)], + model_name: &str, +) -> Result<(), ConfigError> { + let Some(tags) = set_config_entry(header, TAGS_KEY) else { + return Ok(()); + }; + if tags.kind() != NodeKind::List { + return Err(model_error(model_name, "tags must be a list")); + } + if tags.items().iter().all(|tag| tag.kind() == NodeKind::Str) { + Ok(()) + } else { + Err(model_error(model_name, "tags entries must be strings")) + } +} + +/// Whether every item of a hook list is a typed hook entry, checked in order. +fn all_hook_entries( + host: &B, + value: &HostNode, +) -> Result> { + for item in (&value).items() { + let typed: bool = match item { + ConfigNode::Host(_) => host.typed_hook_entry(item).map_err(ConfigBuildHalt::Host)?, + _ => false, + }; + if !typed { + return Ok(false); + } + } + Ok(true) +} + +/// `validate_model_hook_config`: every hook list holds typed hook entries only. +pub(crate) fn check_hooks( + host: &B, + values: &[(HostNode, HostNode)], + model_name: &str, +) -> Result<(), HostHalt> { + let config = ConfigBuildHalt::Config; + for key in SORTED_HOOK_KEYS { + let Some(value) = config_entry(values, key) else { + continue; + }; + if !matches!(value.kind(), NodeKind::List | NodeKind::Tuple) { + return Err(config(model_error( + model_name, + &format!("{key} must be a list of typed hook entries"), + ))); + } + if !all_hook_entries(host, value)? { + return Err(config(model_error( + model_name, + &format!( + "{key} entries must use typed inline_sql(...), sql(...), or python(...) hook syntax" + ), + ))); + } + } + Ok(()) +} + +/// The facts of `key` when its value is still the header's own. +pub(crate) fn own_facts<'a>( + facts: Option<&'a [(String, HeaderEntryFacts)]>, + own: &[String], + key: &str, +) -> Option<&'a HeaderEntryFacts> { + if !own.iter().any(|name| name == key) { + return None; + } + facts? + .iter() + .find(|(name, _)| name == key) + .map(|(_, facts)| facts) +} + +/// The exact text of a string key the header facts are found by. +pub(crate) fn key_text(key: &ConfigNode) -> Option { + match key { + ConfigNode::Str(text) => Some(text.clone()), + ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.clone(), + _ => None, + } +} + +/// The first config field outside hooks holding a macro call, as Python's walk reports it. +pub(crate) fn check_no_config_macros( + values: &[(ConfigNode, ConfigNode)], + facts: Option<&[(String, HeaderEntryFacts)]>, + own: &[String], +) -> Result<(), ConfigError> { + for (key, value) in values { + if HOOK_KEYS.iter().any(|name| key.is_text(name)) { + continue; + } + let found: MacroPath = match key_text(key).and_then(|name| own_facts(facts, own, &name)) { + Some(facts) => facts.macro_path.clone(), + None => entry_macro_path(&key, &value), + }; + if let MacroPath::Found(path) = found { + return Err(ConfigError::compile(format!( + "model config field '{}' does not allow macros", + path.join(".") + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs new file mode 100644 index 0000000000..fac426ef08 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs @@ -0,0 +1,84 @@ +//! The run and model context values templates read through `${ctx.*}`. + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; +use crate::model_config_build::constants::{ + ALIAS_KEY, DATABASE_KEY, DESTINATION_DATABASE_CONTEXT, DESTINATION_QUALIFIED_CONTEXT, + DESTINATION_SCHEMA_CONTEXT, DESTINATION_TABLE_CONTEXT, MODEL_ALIAS_CONTEXT, + MODEL_DATABASE_CONTEXT, MODEL_NAME_CONTEXT, MODEL_SCHEMA_CONTEXT, RUN_ID_CONTEXT, + RUN_TARGET_CONTEXT, SCHEMA_KEY, +}; +use crate::model_config_build::models::ConfigBuildInputs; +use crate::types::{AuthoredNode, NodeKind}; + +/// Context entries. +type Entries = ConfigEntries; + +fn named(name: &str, value: ConfigNode) -> (ConfigNode, ConfigNode) { + (ConfigNode::Str(name.to_owned()), value) +} + +/// The value of `key` when it is a string, as `isinstance(raw, str)` keeps it. +pub(crate) fn string_entry( + values: &[(ConfigNode, ConfigNode)], + key: &str, +) -> Option> { + config_entry(values, key) + .filter(|value| value.kind() == NodeKind::Str) + .cloned() +} + +/// `build_run_context_values`: the run's id and target. +pub(crate) fn run_context(inputs: &ConfigBuildInputs) -> Entries { + vec![ + named(RUN_ID_CONTEXT, inputs.run_id.clone()), + named(RUN_TARGET_CONTEXT, inputs.target_name.clone()), + ] +} + +fn text_of(value: &ConfigNode) -> String { + value.text().unwrap_or_default() +} + +/// `build_model_context_values` for a model's config values. +pub(crate) fn model_context( + inputs: &ConfigBuildInputs, + values: &[(ConfigNode, ConfigNode)], + model_name: &str, + include_target_values: bool, +) -> Entries { + let database: Option> = string_entry(values, DATABASE_KEY); + let schema: Option> = string_entry(values, SCHEMA_KEY); + let alias: ConfigNode = + string_entry(values, ALIAS_KEY).unwrap_or_else(|| ConfigNode::Str(model_name.to_owned())); + let or_null = |value: &Option>| value.clone().unwrap_or(ConfigNode::Null); + let mut context: Entries = run_context(inputs); + context.extend([ + named(MODEL_NAME_CONTEXT, ConfigNode::Str(model_name.to_owned())), + named(MODEL_DATABASE_CONTEXT, or_null(&database)), + named(MODEL_SCHEMA_CONTEXT, or_null(&schema)), + named(MODEL_ALIAS_CONTEXT, alias.clone()), + ]); + if !include_target_values { + return context; + } + let qualified: ConfigNode = match (&database, &schema) { + (Some(database), Some(schema)) => ConfigNode::Str(format!( + "{}.{}.{}", + text_of(database), + text_of(schema), + text_of(&alias) + )), + (None, Some(schema)) => ConfigNode::Str(format!("{}.{}", text_of(schema), text_of(&alias))), + _ => ConfigNode::Null, + }; + context.extend([ + named(DESTINATION_DATABASE_CONTEXT, or_null(&database)), + named(DESTINATION_SCHEMA_CONTEXT, or_null(&schema)), + named(DESTINATION_TABLE_CONTEXT, alias), + named(DESTINATION_QUALIFIED_CONTEXT, qualified), + ]); + context +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs new file mode 100644 index 0000000000..485beb7354 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs @@ -0,0 +1,261 @@ +//! `${...}` templates in config values expanded over the build's variables, environment and context. + +use std::cell::RefCell; + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::plain_scalar_text::plain_scalar_text; +use crate::config_values::models::{ConfigNode, HostChildren}; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_config_build::constants::{CONTEXT_READ, ENVIRONMENT_READ, TEMPLATE_OPEN_TOKEN}; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; +use crate::templates::main::expand_template_string::expand_template_string; +use crate::templates::main::template_error_message::template_error_message; +use crate::templates::models::{ + ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, +}; +use crate::templates::types::TemplateHost; +use crate::types::{AuthoredNode, NodeKind}; + +/// What one expansion reads: variables, the caller's environment and context values. +pub(crate) struct ExpansionSources<'a, B: ConfigBuildHost> { + pub(crate) host: &'a B, + pub(crate) variables: &'a [(HostNode, HostNode)], + pub(crate) context: ConfigEntries, +} + +/// One expansion's template host, holding the first caller error and recording reads. +struct BuildTemplateHost<'a, B: ConfigBuildHost> { + sources: ExpansionSources<'a, B>, + label: &'a str, + reads: RefCell>, + raised: RefCell>, +} + +impl BuildTemplateHost<'_, B> { + fn hold(&self, error: B::Error) -> TemplateFailure { + let _ = self.raised.borrow_mut().get_or_insert(error); + TemplateFailure::Host + } + + /// The exact text of a string value, else the caller's error. + fn exact_text<'v>(&self, value: &'v ConfigNode) -> Result<&'v str, TemplateFailure> { + match value { + ConfigNode::Str(text) => Ok(text), + ConfigNode::Host(host) if host.facts.strict_text => { + host.facts.text.as_deref().ok_or(TemplateFailure::Host) + } + _ => Err(self.hold(self.sources.host.text_error(value))), + } + } + + fn halt(&self, failure: TemplateFailure) -> HostHalt { + match failure { + TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { + ConfigBuildHalt::Config(ConfigError::compile(template_error_message( + &error, self.label, + ))) + } + TemplateFailure::Host => match self.raised.borrow_mut().take() { + Some(error) => ConfigBuildHalt::Host(error), + None => ConfigBuildHalt::Config(ConfigError::compile( + "template expansion stopped without an error".to_owned(), + )), + }, + } + } +} + +impl TemplateHost for BuildTemplateHost<'_, B> { + type Value = ConfigNode; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + Ok(config_entry(self.sources.variables, name).cloned()) + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((ENVIRONMENT_READ, name.to_owned())); + self.sources + .host + .environment(name) + .map_err(|error| self.hold(error)) + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((CONTEXT_READ, name.to_owned())); + Ok(match config_entry(&self.sources.context, name) { + None => ContextValue::Unknown, + Some(ConfigNode::Null) => ContextValue::Unavailable, + Some(value) => ContextValue::Value(value.clone()), + }) + } + + fn text(&self, text: &str) -> Result { + Ok(ConfigNode::Str(text.to_owned())) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(ConfigNode::Bool(flag)) + } + + fn null(&self) -> Self::Value { + ConfigNode::Null + } + + fn scalar(&self, value: &Self::Value) -> Result { + match value { + ConfigNode::Null => Ok(Scalar::Null), + ConfigNode::Bool(flag) => Ok(Scalar::Bool(*flag)), + _ => match plain_scalar_text(value, false) { + Some(text) => Ok(Scalar::Text(text)), + None => self + .sources + .host + .python_str(value, true) + .map(Scalar::Text) + .map_err(|error| self.hold(error)), + }, + } + } + + fn render(&self, value: &Self::Value, label: &str) -> Result { + if let ConfigNode::Str(text) = value { + return Ok(text.clone()); + } + match self.sources.host.render(value, label) { + Ok(Ok(text)) => Ok(text), + Ok(Err(message)) => Err(TemplateFailure::Invalid( + crate::templates::errors::TemplateError::Message(message), + )), + Err(error) => Err(self.hold(error)), + } + } + + fn label(&self) -> &str { + self.label + } +} + +/// Each item expanded. +fn expanded_items( + host: &BuildTemplateHost<'_, B>, + values: &[HostNode], + options: TemplateOptions, +) -> Result>, TemplateFailure> { + values + .iter() + .map(|item| expanded(host, item, options)) + .collect() +} + +/// Each entry's value expanded under its key. +fn expanded_entries( + host: &BuildTemplateHost<'_, B>, + mapping: &[(HostNode, HostNode)], + options: TemplateOptions, +) -> Result, TemplateFailure> { + let mut expanded_mapping: HostEntries = Vec::with_capacity(mapping.len()); + for (key, item) in mapping { + expanded_mapping.push((key.clone(), expanded(host, item, options)?)); + } + Ok(expanded_mapping) +} + +/// Every template in `value` and the containers inside it expanded, as `expanded()` does. +fn expanded( + host: &BuildTemplateHost<'_, B>, + value: &ConfigNode, + options: TemplateOptions, +) -> Result, TemplateFailure> { + let items = |values: &[HostNode]| expanded_items(host, values, options); + let entries = |mapping: &[(HostNode, HostNode)]| expanded_entries(host, mapping, options); + match (value, value.kind()) { + (_, NodeKind::Str) => { + let text: &str = host.exact_text(value)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + Ok(match expand_template_string(host, text, options)? { + StringExpansion::Unchanged => ConfigNode::Str(text.to_owned()), + StringExpansion::Value(result) => result, + StringExpansion::Text(result) => ConfigNode::Str(result), + }) + } + (ConfigNode::Map(mapping), _) => Ok(ConfigNode::Map(entries(mapping)?)), + (ConfigNode::List(values), _) => Ok(ConfigNode::List(items(values)?)), + (ConfigNode::Tuple(values), _) => Ok(ConfigNode::Tuple(items(values)?)), + (ConfigNode::Host(opaque), kind) => match (&opaque.children, kind) { + (HostChildren::Entries(mapping), NodeKind::Map) => { + Ok(ConfigNode::Map(entries(mapping)?)) + } + (HostChildren::Items(values), NodeKind::List) => Ok(ConfigNode::List(items(values)?)), + (HostChildren::Items(values), NodeKind::Tuple) => Ok(ConfigNode::Tuple(items(values)?)), + _ => Ok(value.clone()), + }, + _ => Ok(value.clone()), + } +} + +/// An expansion's result and the `(kind, name)` reads it made, in order. +pub(crate) type Expanded = ( + Result::Handle>, HostHalt>, + Vec<(&'static str, String)>, +); + +/// A result and the `(kind, name)` reads made on the way to it. +type Tracked = (Result>, Vec<(&'static str, String)>); + +/// `expand` run with a template host over `sources`, with the reads it made. +fn with_host( + sources: ExpansionSources<'_, B>, + scope: (bool, &str), + expand: impl FnOnce(&BuildTemplateHost<'_, B>, TemplateOptions) -> Result, +) -> Tracked { + let (preserve_unknown_context, label) = scope; + let host: BuildTemplateHost<'_, B> = BuildTemplateHost { + sources, + label, + reads: RefCell::new(Vec::new()), + raised: RefCell::new(None), + }; + let options: TemplateOptions = TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context, + }; + let result = expand(&host, options).map_err(|failure| host.halt(failure)); + (result, host.reads.into_inner()) +} + +/// `value` expanded, with its reads; `scope` is `preserve_unknown_context` and the context label. +pub(crate) fn expand_value( + sources: ExpansionSources<'_, B>, + value: &ConfigNode, + scope: (bool, &str), +) -> Expanded { + with_host(sources, scope, |host, options| { + expanded(host, value, options) + }) +} + +/// Expanded entries and the `(kind, name)` reads the expansion made. +pub(crate) type ExpandedEntries = ( + Result, HostHalt>, + Vec<(&'static str, String)>, +); + +/// A mapping's entries expanded, with the reads. +pub(crate) fn expand_entries( + sources: ExpansionSources<'_, B>, + entries: &[(HostNode, HostNode)], + scope: (bool, &str), +) -> ExpandedEntries { + with_host(sources, scope, |host, options| { + expanded_entries(host, entries, options) + }) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs new file mode 100644 index 0000000000..5560f3db39 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs @@ -0,0 +1,83 @@ +//! A model's layers merged, with which header entries are still the header's own values. + +use crate::config_values::main::layered_config::layered_config; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; +use crate::model_config_build::_helpers::checks::key_text; +use crate::model_config_build::constants::{TAGS_KEY, TEMPLATE_OPEN_TOKEN}; +use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; +use crate::types::{AuthoredNode, NodeKind}; + +/// Config entries. +pub(crate) type Entries = ConfigEntries; + +/// A fresh copy of the project defaults, with its own `tags` list. +pub(crate) fn project_defaults(inputs: &ConfigBuildInputs) -> Entries { + inputs + .defaults + .iter() + .map(|(key, value)| match value { + ConfigNode::List(items) if key.is_text(TAGS_KEY) => { + (key.clone(), ConfigNode::List(items.clone())) + } + _ => (key.clone(), value.clone()), + }) + .collect() +} + +/// The path default `key`'s entries. +pub(crate) fn path_default_values<'a, H: HostObject, E>( + inputs: &'a ConfigBuildInputs, + key: &str, +) -> Result<&'a Entries, ConfigBuildHalt> { + inputs + .path_defaults + .iter() + .find(|(name, _)| name == key) + .map(|(_, values)| values) + .ok_or_else(|| ConfigBuildHalt::MissingPathDefault(key.to_owned())) +} + +/// `overlay` over `base`, and the overlay's keys whose merged value is still the overlay's own. +pub(crate) fn layered_own( + host: &B, + base: &HostEntries, + overlay: &[(HostNode, HostNode)], +) -> Result<(HostEntries, Vec), HostHalt> { + let text = |item: &HostNode| host.python_str(item, false); + let layered = layered_config(base, overlay, &text).map_err(ConfigBuildHalt::Host)?; + let own: Vec = overlay + .iter() + .filter_map(|(key, _)| key_text(key)) + .filter(|name| !layered.unioned_keys.contains(&name.as_str())) + .collect(); + Ok((layered.entries, own)) +} + +/// Whether expansion keeps `value` as the same object: a string without a template, or a non-container. +pub(crate) fn kept_by_expansion(value: &ConfigNode) -> bool { + match value.kind() { + NodeKind::Str => !value + .with_text(|text| text.contains(TEMPLATE_OPEN_TOKEN)) + .unwrap_or(false), + NodeKind::Map | NodeKind::List | NodeKind::Tuple => false, + _ => true, + } +} + +/// Whether the entry named `name` in `values` keeps its value through an expansion. +fn kept_entry(values: &Entries, name: &str) -> bool { + values + .iter() + .find(|(key, _)| key_text(key).as_deref() == Some(name)) + .is_some_and(|(_, value)| kept_by_expansion(value)) +} + +/// The own keys whose values an expansion of `values` keeps. +pub(crate) fn kept_own(values: &Entries, own: Vec) -> Vec { + own.into_iter() + .filter(|name| kept_entry(values, name)) + .collect() +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs new file mode 100644 index 0000000000..7250eb231f --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod checks; +pub(crate) mod context; +pub(crate) mod expansion; +pub(crate) mod layers; +pub(crate) mod namespace; +pub(crate) mod phases; +pub(crate) mod storage; diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs new file mode 100644 index 0000000000..858bbd444e --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs @@ -0,0 +1,91 @@ +//! The target namespace applied over a model's values, as `apply_environment_database_schema_overrides`. + +use crate::config_values::main::inserted_entry::inserted_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_config_build::_helpers::context::model_context; +use crate::model_config_build::_helpers::expansion::{ExpansionSources, expand_value}; +use crate::model_config_build::constants::{ + DATABASE_KEY, ENVIRONMENT_DATABASE_LABEL, ENVIRONMENT_SCHEMA_LABEL, PRESERVE_TARGET_VALUE, + SCHEMA_KEY, +}; +use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt}; +use crate::types::AuthoredNode; + +/// Config entries. +type Entries = ConfigEntries; + +/// The namespaced values, the keys the target replaced, and the reads its expansions made. +pub(crate) struct Namespaced { + pub(crate) values: Entries, + pub(crate) replaced: Vec<&'static str>, + pub(crate) reads: Vec<(&'static str, String)>, +} + +/// One model's values with the target's database and schema expanded over them. +pub(crate) fn namespaced_values( + (host, inputs): (&B, &ConfigBuildInputs), + values: &Entries, + logical: (bool, bool), + model_name: &str, +) -> Result, HostHalt> { + let mut namespaced = Namespaced { + values: values.clone(), + replaced: Vec::new(), + reads: Vec::new(), + }; + let Some((database, schema)) = &inputs.target_namespace else { + return Ok(namespaced); + }; + let (logical_database, logical_schema) = logical; + let namespace = [ + ( + DATABASE_KEY, + database, + logical_database, + ENVIRONMENT_DATABASE_LABEL, + ), + (SCHEMA_KEY, schema, logical_schema, ENVIRONMENT_SCHEMA_LABEL), + ]; + let preserved = |value: &ConfigNode| value.is_text(PRESERVE_TARGET_VALUE); + let missing: Vec<&str> = namespace + .iter() + .filter(|(_, value, logical, _)| preserved(value) && !logical) + .map(|(key, _, _, _)| *key) + .collect(); + if !missing.is_empty() { + let dimensions: String = missing.join(" and "); + return Err(ConfigBuildHalt::Config( + ConfigError::compile(format!( + "Model '{model_name}' has no logical {dimensions}, but the selected target sets \ + {dimensions} to 'preserve'" + )) + .with_help(format!( + "Set {dimensions} on the resource or its defaults, or configure a literal \ + target {dimensions}." + )), + )); + } + let context: Entries = model_context(inputs, values, model_name, false); + for (key, value, _, label) in namespace { + if matches!(value, ConfigNode::Null) || preserved(value) { + continue; + } + let sources: ExpansionSources<'_, B> = ExpansionSources { + host, + variables: &inputs.variables, + context: context.clone(), + }; + let (expanded, reads) = expand_value(sources, value, (false, label)); + namespaced.reads.extend(reads); + namespaced.values = inserted_entry( + namespaced.values, + ConfigNode::Str(key.to_owned()), + expanded?, + ); + namespaced.replaced.push(key); + } + Ok(namespaced) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs new file mode 100644 index 0000000000..87ea793f80 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs @@ -0,0 +1,228 @@ +//! The phases of one model's config build: layering, model templates, target namespace, result. + +use crate::config_presence::main::contains_template::contains_template; +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::inserted_entry::inserted_entry; +use crate::config_values::models::ConfigNode; +use crate::model_config_build::_helpers::checks::{ + check_hooks, check_no_config_macros, key_text, own_facts, +}; +use crate::model_config_build::_helpers::context::{model_context, run_context, string_entry}; +use crate::model_config_build::_helpers::expansion::{ExpansionSources, expand_entries}; +use crate::model_config_build::_helpers::layers::{ + Entries, kept_own, layered_own, path_default_values, project_defaults, +}; +use crate::model_config_build::_helpers::namespace::{Namespaced, namespaced_values}; +use crate::model_config_build::_helpers::storage::{sorted_header_keys, storage_free_values}; +use crate::model_config_build::constants::{ + DATABASE_KEY, HOOK_KEYS, MODEL_CONFIG_LABEL, RETENTION_KEY, SCHEMA_KEY, TABLE_TYPE_KEY, +}; +use crate::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, ModelHeader, +}; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; +use crate::model_validation::main::retention_override::retention_override; +use crate::model_validation::main::table_type_override::table_type_override; +use crate::types::AuthoredNode; + +/// What one model's build reads: its host and project inputs, its header and its name. +pub(crate) struct BuildModel<'a, B: ConfigBuildHost> { + pub(crate) host: &'a B, + pub(crate) inputs: &'a ConfigBuildInputs, + pub(crate) header: ModelHeader<'a, B::Handle>, + pub(crate) name: &'a str, +} + +/// The values so far, the header keys still holding the header's own values, and the reads made. +pub(crate) struct Phase { + pub(crate) values: Entries, + pub(crate) own: Vec, + pub(crate) reads: Vec<(&'static str, String)>, +} + +/// What layering sets aside until the end: hooks, which of them are the header's own, and the schema layer. +pub(crate) struct SetAside { + pub(crate) hooks: Entries, + pub(crate) own_hooks: Vec, + pub(crate) layer_schema_configured: bool, +} + +/// The layered values, what is set aside, and whether any value holds a template. +pub(crate) struct Layered { + pub(crate) phase: Phase, + pub(crate) set_aside: SetAside, + pub(crate) has_templates: bool, +} + +/// The namespaced values and the logical database and schema the model set itself. +pub(crate) struct Targeted { + pub(crate) phase: Phase, + pub(crate) logical_schema: Option>, + pub(crate) logical_database: Option>, +} + +/// The project defaults, the matched path default and the header merged, with hooks set aside. +pub(crate) fn layered( + model: &BuildModel<'_, B>, + matched_path_default: Option<&str>, +) -> Result, HostHalt> { + let path_values: Option<&Entries> = matched_path_default + .map(|key| path_default_values(model.inputs, key)) + .transpose()?; + let mut defaults: Entries = project_defaults(model.inputs); + if let Some(path_values) = path_values { + defaults = layered_own(model.host, &defaults, path_values)?.0; + } + let (mut values, own) = layered_own(model.host, &defaults, model.header.entries)?; + check_hooks(model.host, &values, model.name)?; + let mut hooks: Entries = Vec::new(); + for key in HOOK_KEYS { + if let Some(position) = values.iter().position(|(name, _)| name.is_text(key)) { + hooks.push(values.remove(position)); + } + } + let has_templates: bool = values.iter().any(|(key, value)| { + let facts = key_text(key).and_then(|name| own_facts(model.header.facts, &own, &name)); + facts.map_or_else(|| contains_template(&value), |facts| facts.template) + }); + let own_hooks: Vec = own + .iter() + .filter(|name| HOOK_KEYS.contains(&name.as_str())) + .cloned() + .collect(); + let layer_schema_configured: bool = config_entry(model.header.entries, SCHEMA_KEY).is_some() + || path_values.is_some_and(|values| config_entry(values, SCHEMA_KEY).is_some()); + Ok(Layered { + phase: Phase { + values, + own, + reads: Vec::new(), + }, + set_aside: SetAside { + hooks, + own_hooks, + layer_schema_configured, + }, + has_templates, + }) +} + +/// `phase`'s values expanded once over `sources`, its reads extended. +fn expanded_phase( + mut phase: Phase, + sources: ExpansionSources<'_, B>, + preserve_unknown_context: bool, +) -> Result, HostHalt> { + let scope = (preserve_unknown_context, MODEL_CONFIG_LABEL); + let (expanded, reads) = expand_entries(sources, &phase.values, scope); + phase.reads.extend(reads); + phase.values = expanded?; + Ok(phase) +} + +/// Sources of `model`'s host over `variables` and `context`. +fn sources<'a, B: ConfigBuildHost>( + model: &BuildModel<'a, B>, + variables: &'a [(HostNode, HostNode)], + context: HostEntries, +) -> ExpansionSources<'a, B> { + ExpansionSources { + host: model.host, + variables, + context, + } +} + +/// The model's templates expanded against the run, then twice against the model context. +pub(crate) fn model_templates( + model: &BuildModel<'_, B>, + mut phase: Phase, +) -> Result, HostHalt> { + phase.own = kept_own(&phase.values, phase.own); + let run = run_context(model.inputs); + phase = expanded_phase(phase, sources(model, &model.inputs.variables, run), true)?; + for _ in 0..2 { + let context = model_context(model.inputs, &phase.values, model.name, false); + phase = expanded_phase(phase, sources(model, &[], context), true)?; + } + Ok(phase) +} + +/// The target's namespace applied, then target templates expanded when any are left. +pub(crate) fn targeted( + model: &BuildModel<'_, B>, + mut phase: Phase, + has_templates: bool, +) -> Result, HostHalt> { + let logical_schema = string_entry(&phase.values, SCHEMA_KEY); + let logical_database = string_entry(&phase.values, DATABASE_KEY); + let logical = (logical_database.is_some(), logical_schema.is_some()); + let Namespaced { + values, + replaced, + reads, + } = namespaced_values( + (model.host, model.inputs), + &phase.values, + logical, + model.name, + )?; + phase.values = values; + phase.reads.extend(reads); + phase.own.retain(|name| !replaced.contains(&name.as_str())); + let needs_target_templates: bool = has_templates + || [DATABASE_KEY, SCHEMA_KEY].iter().any(|key| { + config_entry(&phase.values, key).is_some_and(|value| contains_template(&value)) + }); + if needs_target_templates { + phase.own = kept_own(&phase.values, phase.own); + let context = model_context(model.inputs, &phase.values, model.name, true); + phase = expanded_phase(phase, sources(model, &[], context), false)?; + } + Ok(Targeted { + phase, + logical_schema, + logical_database, + }) +} + +/// The built config: hooks restored, storage keys dropped, overrides read and macros rejected. +pub(crate) fn finished( + model: &BuildModel<'_, B>, + set_aside: SetAside, + targeted: Targeted, +) -> Result, HostHalt> { + let SetAside { + hooks, + own_hooks, + layer_schema_configured, + } = set_aside; + let (header, config) = (&model.header, ConfigBuildHalt::Config); + let mut resolved: Entries = targeted.phase.values; + for (key, value) in hooks { + resolved = inserted_entry(resolved, key, value); + } + let values: Entries = storage_free_values(resolved, header.entries); + let retention_value = config_entry(header.entries, RETENTION_KEY); + let retention = retention_override(retention_value, model.name).map_err(config)?; + let table_type_value = config_entry(header.entries, TABLE_TYPE_KEY); + let table_type = table_type_override(table_type_value, model.name).map_err(config)?; + check_no_config_macros(&values, header.facts, &targeted.phase.own).map_err(config)?; + let mut own_header_keys: Vec = targeted.phase.own; + let restored: Vec = own_hooks + .into_iter() + .filter(|name| !own_header_keys.contains(name)) + .collect(); + own_header_keys.extend(restored); + Ok(BuiltModelConfig { + values, + header_keys: sorted_header_keys(header.entries)?, + logical_schema: targeted.logical_schema, + layer_schema_configured, + logical_database: targeted.logical_database, + retention, + table_type, + reads: targeted.phase.reads, + own_header_keys, + }) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs new file mode 100644 index 0000000000..d2cb4fec06 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs @@ -0,0 +1,57 @@ +//! The built values without storage keys, and the header's keys. + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::same_config_key::same_config_key; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; +use crate::model_config_build::_helpers::checks::key_text; +use crate::model_config_build::constants::{ + FULL_REFRESH_KEY, INCREMENTAL_MATERIALIZATION, MATERIALIZED_KEY, RETENTION_KEY, TABLE_TYPE_KEY, +}; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::types::AuthoredNode; + +/// Config entries. +type Entries = ConfigEntries; + +/// Whether `key` names a storage setting, which the config values leave out. +fn storage_key(key: &ConfigNode) -> bool { + key.is_text(RETENTION_KEY) || key.is_text(TABLE_TYPE_KEY) +} + +/// Drop storage keys, then `full_refresh` where `build_model_config` drops it. +pub(crate) fn storage_free_values( + resolved: Entries, + header: &[(ConfigNode, ConfigNode)], +) -> Entries { + let mut values: Entries = resolved + .into_iter() + .filter(|(key, _)| !storage_key(key)) + .collect(); + let incremental: bool = config_entry(&values, MATERIALIZED_KEY) + .is_some_and(|value| value.is_text(INCREMENTAL_MATERIALIZATION)); + if config_entry(header, FULL_REFRESH_KEY).is_none() && !incremental { + let refresh = ConfigNode::Str(FULL_REFRESH_KEY.to_owned()); + values.retain(|(key, _)| !same_config_key(key, &refresh)); + } + values +} + +/// The header's keys sorted by code point; a key that is not a string halts. +pub(crate) fn sorted_header_keys( + header: &[(ConfigNode, ConfigNode)], +) -> Result, ConfigBuildHalt> { + let mut keys: Vec = Vec::with_capacity(header.len()); + for (key, _) in header { + match (key_text(key), key) { + (Some(text), _) => keys.push(text), + (None, ConfigNode::Host(host)) if host.facts.text.is_some() => { + keys.push(host.facts.text.clone().unwrap_or_default()); + } + (None, _) => return Err(ConfigBuildHalt::KeyNotText(key.clone())), + } + } + keys.sort(); + Ok(keys) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/constants.rs b/crates/sqlbuild-model-config/src/model_config_build/constants.rs new file mode 100644 index 0000000000..0260df4a21 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/constants.rs @@ -0,0 +1,31 @@ +//! Config keys, labels and context names the model config build reads and writes. + +pub(crate) const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +/// Hook keys in the sorted order `validate_model_hook_config` checks them. +pub(crate) const SORTED_HOOK_KEYS: [&str; 2] = ["post_hooks", "pre_hooks"]; +pub(crate) const MODEL_CONFIG_LABEL: &str = "model config"; +pub(crate) const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; +pub(crate) const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; +pub(crate) const TAGS_KEY: &str = "tags"; +pub(crate) const DATABASE_KEY: &str = "database"; +pub(crate) const SCHEMA_KEY: &str = "schema"; +pub(crate) const ALIAS_KEY: &str = "alias"; +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const FULL_REFRESH_KEY: &str = "full_refresh"; +pub(crate) const RETENTION_KEY: &str = "time_travel_retention"; +pub(crate) const TABLE_TYPE_KEY: &str = "table_type"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const PRESERVE_TARGET_VALUE: &str = "preserve"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const ENVIRONMENT_READ: &str = "env"; +pub(crate) const CONTEXT_READ: &str = "ctx"; +pub(crate) const RUN_ID_CONTEXT: &str = "run.id"; +pub(crate) const RUN_TARGET_CONTEXT: &str = "run.target"; +pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; +pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; +pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; +pub(crate) const MODEL_ALIAS_CONTEXT: &str = "model.alias"; +pub(crate) const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; +pub(crate) const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; +pub(crate) const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; +pub(crate) const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs new file mode 100644 index 0000000000..06d561416a --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs @@ -0,0 +1,36 @@ +//! One model's effective config built from its layers, as `build_model_config` builds it. + +use crate::model_config_build::_helpers::checks::check_header_tags; +use crate::model_config_build::_helpers::phases::{ + BuildModel, Layered, finished, layered, model_templates, targeted, +}; +use crate::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, ModelHeader, +}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt}; + +/// One model's effective config: its layers merged, then expanded for the model and its target. +pub fn build_model_config( + (host, inputs): (&B, &ConfigBuildInputs), + header: ModelHeader<'_, B::Handle>, + matched_path_default: Option<&str>, + name: &str, +) -> Result, HostHalt> { + check_header_tags(header.entries, name).map_err(ConfigBuildHalt::Config)?; + let model: BuildModel<'_, B> = BuildModel { + host, + inputs, + header, + name, + }; + let Layered { + mut phase, + set_aside, + has_templates, + } = layered(&model, matched_path_default)?; + if has_templates { + phase = model_templates(&model, phase)?; + } + let targeted = targeted(&model, phase, has_templates)?; + finished(&model, set_aside, targeted) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs new file mode 100644 index 0000000000..050a00fbe9 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs @@ -0,0 +1,19 @@ +//! What a path default's models inherit. + +use crate::model_config_build::_helpers::layers::{ + layered_own, path_default_values, project_defaults, +}; +use crate::model_config_build::models::ConfigBuildInputs; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt}; + +/// `build_layered_model_values` with an empty header: the defaults under the matched path default. +pub fn inherited_config( + (host, inputs): (&B, &ConfigBuildInputs), + matched_path_default: Option<&str>, +) -> Result, HostHalt> { + let mut layered = project_defaults(inputs); + if let Some(key) = matched_path_default { + layered = layered_own(host, &layered, path_default_values(inputs, key)?)?.0; + } + Ok(layered) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs new file mode 100644 index 0000000000..d48f05004a --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs @@ -0,0 +1,3 @@ +pub mod build_model_config; +pub mod inherited_config; +pub mod model_context_values; diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs new file mode 100644 index 0000000000..201b9afd8f --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs @@ -0,0 +1,18 @@ +//! The context values hook templates read for one model's built config. + +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; +use crate::model_config_build::_helpers::context::model_context; +use crate::model_config_build::models::ConfigBuildInputs; + +/// `build_model_context_values` for a model's config values. +#[must_use] +pub fn model_context_values( + inputs: &ConfigBuildInputs, + values: &[(ConfigNode, ConfigNode)], + model_name: &str, + include_target_values: bool, +) -> ConfigEntries { + model_context(inputs, values, model_name, include_target_values) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/mod.rs new file mode 100644 index 0000000000..39e299a85d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/mod.rs @@ -0,0 +1,10 @@ +//! A model's effective config: its layers merged, templated and checked, as `build_model_config` builds it. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/model_config_build/models.rs b/crates/sqlbuild-model-config/src/model_config_build/models.rs new file mode 100644 index 0000000000..a5860bfeba --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/models.rs @@ -0,0 +1,65 @@ +//! The inputs and result of one model's config build. + +use crate::config_presence::models::MacroPath; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_validation::models::{RetentionOverride, TableTypeOverride}; + +/// The project-wide inputs every model's config build reads, captured once per compile. +#[derive(Clone, Debug)] +pub struct ConfigBuildInputs { + /// `project_defaults_to_mapping`'s entries. + pub defaults: ConfigEntries, + /// Each path default's entries by its key. + pub path_defaults: Vec<(String, ConfigEntries)>, + /// The effective vars. + pub variables: ConfigEntries, + pub run_id: ConfigNode, + pub target_name: ConfigNode, + /// The active target's `(database, schema)`, without a target `None`. + pub target_namespace: Option<(ConfigNode, ConfigNode)>, +} + +/// What the native header parse found in one header entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderEntryFacts { + /// Whether a string in the value holds a `${...}` template. + pub template: bool, + /// Where the config macro walk finds a macro call. + pub macro_path: MacroPath, +} + +/// One model's authored header and, when it was parsed natively, each entry's facts by key. +#[derive(Clone, Copy, Debug)] +pub struct ModelHeader<'a, H> { + pub entries: &'a [(ConfigNode, ConfigNode)], + pub facts: Option<&'a [(String, HeaderEntryFacts)]>, +} + +/// One model's built config, before the caller wraps it in `CompileModelConfig`. +#[derive(Clone, Debug)] +pub struct BuiltModelConfig { + pub values: ConfigEntries, + pub header_keys: Vec, + pub logical_schema: Option>, + pub layer_schema_configured: bool, + pub logical_database: Option>, + pub retention: RetentionOverride, + pub table_type: TableTypeOverride, + /// The `(kind, name)` environment and context reads, in order. + pub reads: Vec<(&'static str, String)>, + /// The header keys whose built values are still the header's own values. + pub own_header_keys: Vec, +} + +/// Why a build stops: the config error it raises, a caller error, or a header key that is not text. +#[derive(Debug, PartialEq)] +pub enum ConfigBuildHalt { + Config(ConfigError), + Host(E), + /// A header key that is not a string, which Python fails to read as one. + KeyNotText(ConfigNode), + /// A path default the inputs do not hold. + MissingPathDefault(String), +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests.rs b/crates/sqlbuild-model-config/src/model_config_build/tests.rs new file mode 100644 index 0000000000..ce6598d186 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_model_config.rs"] +mod build_model_config; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs new file mode 100644 index 0000000000..4715d4af52 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs @@ -0,0 +1,41 @@ +use crate::config_values::models::ConfigNode; +use crate::model_config_build::main::build_model_config::build_model_config; +use crate::model_config_build::models::{ConfigBuildInputs, ModelHeader}; +use crate::model_config_build::tests::test_types::{BuildOutcome, BuildTestCase, FixedHost, Node}; + +/// A string node. +pub(super) fn text(value: &str) -> Node { + ConfigNode::Str(value.to_owned()) +} + +/// A string-keyed entry. +pub(super) fn named(key: &str, value: Node) -> (Node, Node) { + (text(key), value) +} + +/// The values and reads of building `orders` from the case's header, or the error text. +pub(super) fn built(test_case: &BuildTestCase) -> BuildOutcome { + let inputs: ConfigBuildInputs = ConfigBuildInputs { + defaults: vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + ], + path_defaults: Vec::new(), + variables: vec![ + named("region", text("north")), + named("count", ConfigNode::Int(3)), + ], + run_id: text("run_1"), + target_name: text("dev"), + target_namespace: test_case.target_namespace.clone(), + }; + let host = FixedHost { + environment: vec![("SCHEMA_SUFFIX", "eu")], + }; + let header = ModelHeader { + entries: &test_case.header, + facts: None, + }; + build_model_config((&host, &inputs), header, None, "orders") + .map(|built| (built.values, built.reads)) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs new file mode 100644 index 0000000000..04afb9130d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs @@ -0,0 +1,91 @@ +use crate::config_values::models::ConfigNode; +use crate::errors::ConfigError; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::tests::helpers::{built, named, text}; +use crate::model_config_build::tests::test_types::BuildTestCase; + +#[test] +fn given_model_headers_when_building_config_then_values_expand_and_layer_as_python_does() { + let test_cases = [ + BuildTestCase { + description: "the header overrides defaults and tags union in place", + header: vec![ + named("tags", ConfigNode::List(vec![text("orders")])), + named("materialized", text("table")), + ], + target_namespace: None, + expected_outcome: Ok(( + vec![ + named("materialized", text("table")), + named("tags", ConfigNode::List(vec![text("core"), text("orders")])), + ], + vec![], + )), + }, + BuildTestCase { + description: "templates read variables, the environment and the model context", + header: vec![named( + "alias", + text("${CTX:model.name}_${region}_${ENV:SCHEMA_SUFFIX}_${count}"), + )], + target_namespace: None, + expected_outcome: Ok(( + vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + named("alias", text("orders_north_eu_3")), + ], + vec![ + ("ctx", "model.name".to_owned()), + ("env", "SCHEMA_SUFFIX".to_owned()), + ("ctx", "model.name".to_owned()), + ], + )), + }, + BuildTestCase { + description: "the target schema expands with the model context", + header: vec![], + target_namespace: Some((ConfigNode::Null, text("${region}_${CTX:model.name}"))), + expected_outcome: Ok(( + vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + named("schema", text("north_orders")), + ], + vec![("ctx", "model.name".to_owned())], + )), + }, + BuildTestCase { + description: "a preserved target schema needs a logical schema", + header: vec![], + target_namespace: Some((ConfigNode::Null, text("preserve"))), + expected_outcome: Err(ConfigBuildHalt::Config( + ConfigError::compile( + "Model 'orders' has no logical schema, but the selected target sets schema \ + to 'preserve'" + .to_owned(), + ) + .with_help( + "Set schema on the resource or its defaults, or configure a literal target \ + schema.", + ), + )), + }, + BuildTestCase { + description: "a macro call in a config value is rejected", + header: vec![named("description", text("@note()"))], + target_namespace: None, + expected_outcome: Err(ConfigBuildHalt::Config(ConfigError::compile( + "model config field 'description' does not allow macros".to_owned(), + ))), + }, + ]; + for test_case in test_cases { + assert_eq!( + built(&test_case), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs new file mode 100644 index 0000000000..9cdb666bb2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs @@ -0,0 +1,54 @@ +use crate::config_values::models::ConfigNode; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::types::ConfigBuildHost; + +/// A config node in tests; host handles are plain numbers. +pub(super) type Node = ConfigNode; + +/// An environment of fixed variables; rendering takes `str()` of numbers. +pub(super) struct FixedHost { + pub(super) environment: Vec<(&'static str, &'static str)>, +} + +impl ConfigBuildHost for FixedHost { + type Handle = u8; + type Error = String; + + fn environment(&self, name: &str) -> Result, String> { + Ok(self + .environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| ConfigNode::Str((*value).to_owned()))) + } + + fn render(&self, value: &Node, _label: &str) -> Result, String> { + Ok(match value { + ConfigNode::Int(number) => Ok(number.to_string()), + _ => Err("cannot be interpolated".to_owned()), + }) + } + + fn text_error(&self, _value: &Node) -> String { + "not text".to_owned() + } + + fn python_str(&self, _value: &Node, _exact: bool) -> Result { + Err("no host text".to_owned()) + } + + fn typed_hook_entry(&self, _value: &Node) -> Result { + Ok(false) + } +} + +/// The built values and reads, or why the build stops. +pub(super) type BuildOutcome = + Result<(Vec<(Node, Node)>, Vec<(&'static str, String)>), ConfigBuildHalt>; + +pub(super) struct BuildTestCase { + pub(super) description: &'static str, + pub(super) header: Vec<(Node, Node)>, + pub(super) target_namespace: Option<(Node, Node)>, + pub(super) expected_outcome: BuildOutcome, +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/types.rs b/crates/sqlbuild-model-config/src/model_config_build/types.rs new file mode 100644 index 0000000000..2b1e7bf966 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/types.rs @@ -0,0 +1,40 @@ +//! What the model config build asks of its caller. + +use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; +use crate::model_config_build::models::ConfigBuildHalt; + +/// The process environment, project-var rendering and exact text errors, owned by the caller. +pub trait ConfigBuildHost { + /// The caller's handle of an opaque value. + type Handle: HostObject; + /// An error the caller raises, held until the build stops. + type Error; + + /// The environment variable `name`, or `None` when it is not set. + fn environment(&self, name: &str) -> Result>, Self::Error>; + /// A non-string value as interpolated text, or the message of a value that cannot be. + fn render( + &self, + value: &ConfigNode, + label: &str, + ) -> Result, Self::Error>; + /// The error reading `value` as exact text raises. + fn text_error(&self, value: &ConfigNode) -> Self::Error; + /// Python's `str(value)` of a value that is not a plain scalar; `exact` rejects lone surrogates. + fn python_str( + &self, + value: &ConfigNode, + exact: bool, + ) -> Result; + /// Whether `value` is a typed hook entry. + fn typed_hook_entry(&self, value: &ConfigNode) -> Result; +} + +/// A config value of the build host `B`. +pub type HostNode = ConfigNode<::Handle>; +/// Config entries of the build host `B`. +pub type HostEntries = Vec<(HostNode, HostNode)>; +/// Why a build over the host `B` stops. +pub type HostHalt = + ConfigBuildHalt<::Handle, ::Error>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs index 3b9826048f..03a24163f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs @@ -4,18 +4,15 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use pyo3::prelude::PyResult; - use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_rules::engine::main::retained_request::retained_request; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; /// Whether edits that report or raise errors compile as deltas; until then they fall back. pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; -/// One model's header keys with the digest of each authored value, sorted by key. -pub(crate) type HeaderKeys = Vec<(String, String)>; - /// What one model contributes to the partial-check reasons, apart from its dependencies. #[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub(crate) struct ModelCoverage { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 127756794c..df7b38b822 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -33,10 +33,10 @@ use crate::bindings::_helpers::compile_driver::command::{ PhasesOutcome, resolved_request, unrecorded_compile, }; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; -use crate::bindings::_helpers::discovery::header_keys::{ - HeaderChange, custom_rule_reads, header_change, -}; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; +use sqlbuild_discovery::header_changes::main::header_change::header_change; +use sqlbuild_discovery::header_changes::models::HeaderChange; /// Set to `1` to always compile the whole project. pub(crate) const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index aebdc05f91..24322ec018 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -7,13 +7,13 @@ use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; -use crate::bindings::_helpers::compile_context::delta::HeaderKeys; use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compile_context::delta::{BaseRules, PendingRules}; use crate::bindings::_helpers::compile_context::delta::{ CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, }; use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; +use sqlbuild_discovery::header_changes::types::HeaderKeys; /// Changes whenever the image's meaning changes, so an older image is never read. pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 074a44b231..cc89abc6c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -26,6 +26,12 @@ pub(crate) const FORMAT_LINT_FILES: &str = "format_lint_files"; pub(crate) const SQL_TEST_ARTIFACT_PLANS: &str = "plan_compiled_sql_test_artifacts"; pub(crate) const SQL_TEST_MODEL_CHAINS: &str = "sql_test_model_chain_names_by_key"; pub(crate) const BACKGROUND_SQL_TEST_PLANS: &str = "background_sql_test_plans"; +/// Models whose header columns are reused from the native header parse. +pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; +/// Models whose header columns are parsed again from their config values. +pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; +/// Config values converted into typed nodes for a key a reader asked for. +pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads"; /// Compile stage runs since the last reset, by stage. #[pyclass] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs index 6f14cf3cd6..ef9a10ebaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -1,119 +1,5 @@ -//! Header keys native discovery accepts, sorted by code point as Python sorts them. +//! The fields discovery reports for SQL hook entries. -pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 73] = [ - "alias", - "append_cursor_inclusive", - "audit_factories", - "audits", - "batch_concurrency", - "batch_size", - "check_columns", - "columns", - "config", - "constants", - "contract", - "cursor", - "cursor_end", - "cursor_future_action", - "cursor_future_max_distance", - "cursor_grain", - "cursor_inputs", - "cursor_start", - "cursor_start_max_action", - "cursor_start_max_ahead", - "cursor_type", - "cursor_watermark_mode", - "database", - "description", - "dynamic_columns", - "enabled", - "engine", - "enums", - "full_refresh", - "historical_input", - "incremental_mode", - "incremental_strategy", - "initial_valid_from", - "invalidate_hard_deletes", - "lookback", - "materialized", - "max_duration", - "max_microbatches", - "merge_exclude_columns", - "microbatch_limit", - "microbatch_strategy", - "migrate_force", - "migrate_from", - "model_schema", - "observed_at", - "old_name_view", - "on_schema_change", - "order_by", - "partition_by", - "placeholders", - "post_hooks", - "pre_hooks", - "primary_key", - "replay_on_change", - "row_diff_exclude_columns", - "row_diff_sample_rows", - "row_diff_sample_seed", - "row_diff_tolerances", - "schema", - "snapshot_full_refresh", - "snapshot_schema_change", - "snapshot_strategy", - "sql_analysis", - "sql_validation", - "table_settings", - "table_type", - "tags", - "time_travel_retention", - "unaccounted_partition_policy", - "unique_key", - "updated_at", - "valid_from_column", - "valid_to_column", -]; -pub(crate) const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"]; -pub(crate) const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [ - "arguments", - "database", - "description", - "returns", - "schema", - "tags", -]; -pub(crate) const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [ - "always_run", - "evaluation", - "minimum_samples", - "name", - "run_scope", - "sample_count", - "sample_unit", - "severity", - "sql_analysis", - "thresholds", - "value", -]; -pub(crate) const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"]; -pub(crate) const SQL_TEST_HEADER_KEYS: [&str; 8] = [ - "cases", - "cursor_end", - "cursor_start", - "mode", - "name", - "parameters", - "sql_analysis", - "timeout", -]; -pub(crate) const SQL_SCENARIO_HEADER_KEYS: [&str; 4] = - ["description", "expected_columns", "tags", "timeout"]; -/// `SCENARIO (expected_columns ...)` values: compare listed columns, or require every column. -pub(crate) const SCENARIO_EXPECTED_COLUMNS_KEY: &str = "expected_columns"; -pub(crate) const SCENARIO_EXPECTED_COLUMNS_VALUES: [&str; 2] = ["listed", "all"]; -pub(crate) const SCENARIO_EXPECTED_COLUMNS_ALL: &str = "all"; /// The fields a SQL hook entry reports, in output order. pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ "statement", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index c1415677ac..bc3b154b28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -7,8 +7,8 @@ use sqlbuild_discovery::declaration_files::main::discover_declaration_collection use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ AuditBlock, AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, - DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, - FunctionFile, HookFile, MacroFile, SchemaFile, ScopedFile, + DeclarationCollection, DiscoverySession, EnumFile, FileScope, FunctionFile, HookFile, + MacroFile, SchemaFile, ScopedFile, }; use sqlbuild_discovery::declaration_files::types::Collection; use sqlbuild_discovery::models::{FileOutcome, LineColumnSpan}; @@ -18,7 +18,6 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords}; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; use crate::bindings::_helpers::discovery::project_files::path_object; @@ -26,7 +25,6 @@ use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, st use crate::bindings::_helpers::discovery::snapshot::{ in_snapshot, project_root, project_tree, snapshot_memo, }; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::model_config::header_columns::schema_columns; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::{enum_from_value, new_record, new_record_with}; @@ -34,6 +32,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; +use sqlbuild_discovery::declaration_files::main::declaration_file_options::declaration_file_options; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; @@ -377,14 +376,7 @@ pub(crate) fn discovery_session<'py>( let created: Bound<'py, NativeDiscoverySession> = Bound::new( py, NativeDiscoverySession { - inner: DiscoverySession::new( - project_root(project_dir)?, - DeclarationFileOptions { - function_keys: sorted_keys(&constants::SQL_FUNCTION_HEADER_KEYS), - audit_keys: sorted_keys(&constants::SQL_AUDIT_HEADER_KEYS), - hook_keys: sorted_keys(&constants::SQL_HOOK_HEADER_KEYS), - }, - ), + inner: DiscoverySession::new(project_root(project_dir)?, declaration_file_options()), }, )?; memo.set_item(SESSION_MEMO_KEY, &created)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs index 5abec8d9f4..9fdbdf97be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs @@ -1,270 +1,19 @@ -//! Each model header's keys as digests, so a delta compile can tell which keys an edit changed. +//! Record each discovered model header's keys as digests, so a delta compile can tell which keys an edit changed. -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; - -use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; -use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; +use sqlbuild_discovery::header_changes::main::header_key_digests::header_key_digests; +use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; -use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compile_context::delta::HeaderKeys; use crate::bindings::_helpers::compile_context::delta::{capture, capturing}; -use crate::bindings::_helpers::discovery::model_records::model_file_options; - -/// Header keys a delta compile recompiles the model for; any other changed key falls back. -const PATCHED_KEYS: [&str; 6] = [ - "description", - "columns", - "dynamic_columns", - "model_schema", - "audits", - "contract", -]; -/// `materialized` values a delta compile handles; others name materializations it cannot. -const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; -const MATERIALIZED_KEY: &str = "materialized"; -/// Plain `--` comments, which every adapter's SQL reads as comments. -const LINE_COMMENT: &str = "--"; -/// Reference call kinds and the upstream kinds they resolve to. -const REF_CALL: &str = "ref"; -const SOURCE_CALL: &str = "source"; -const SEED_CALL: &str = "seed"; -const MODEL_KIND: &str = "model"; -const SOURCE_KIND: &str = "source"; -const SEED_KIND: &str = "seed"; - -/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. -const KEPT_DIGEST_BYTES: usize = 16; -const HEX: &[u8; 16] = b"0123456789abcdef"; - -/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. -#[derive(Default)] -struct ValueEncoding { - bytes: Vec, -} - -impl ValueEncoding { - fn text(&mut self, tag: u8, text: &str) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(text.len() as u64).to_le_bytes()); - self.bytes.extend_from_slice(text.as_bytes()); - } - - fn list(&mut self, tag: u8, items: &[AuthoredValue]) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(items.len() as u64).to_le_bytes()); - for item in items { - self.value(item); - } - } - - fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(items.len() as u64).to_le_bytes()); - for (key, item) in items { - self.text(b'k', key); - self.value(item); - } - } - - fn value(&mut self, value: &AuthoredValue) { - match value { - AuthoredValue::Null => self.bytes.push(b'n'), - AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), - AuthoredValue::BareWord(word) => self.text(b'w', word), - AuthoredValue::String(string) => self.text(b's', string), - AuthoredValue::List(items) => self.list(b'l', items), - AuthoredValue::Map(items) => self.pairs(b'm', items), - AuthoredValue::Set(items) => self.list(b'e', items), - AuthoredValue::Tuple(items) => self.list(b't', items), - AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), - AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), - AuthoredValue::NamedSqlHook(name, items) => { - self.text(b'h', name); - self.pairs(b'a', items); - } - AuthoredValue::PythonHook(name, items) => { - self.text(b'p', name); - self.pairs(b'a', items); - } - } - } -} - -/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. -fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { - let syntax: LexicalSyntax = LexicalSyntax { - line_comment_prefixes: vec![LINE_COMMENT.to_owned()], - ..LexicalSyntax::default() - }; - let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { - return false; - }; - scan.references.iter().all(|reference| { - let kind: Option<&str> = match reference.kind { - REF_CALL => Some(MODEL_KIND), - SOURCE_CALL => Some(SOURCE_KIND), - SEED_CALL => Some(SEED_KIND), - _ => None, - }; - match (kind, &reference.package) { - (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), - _ => true, - } - }) -} - -/// Whether `upstream` holds the `kind` resource `name`. -fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { - upstream - .iter() - .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) -} - -fn short_hex(digest: &[u8]) -> String { - let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); - for byte in &digest[..KEPT_DIGEST_BYTES] { - hex.push(char::from(HEX[usize::from(byte >> 4)])); - hex.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - hex -} - -fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { - let mut keys: HeaderKeys = values - .iter() - .map(|(key, value)| { - let mut encoding: ValueEncoding = ValueEncoding::default(); - encoding.value(value); - let hex: String = short_hex(&bytes_digest(&encoding.bytes)); - (key.clone(), hex) - }) - .collect(); - keys.sort(); - keys -} /// Record one discovered model's header keys for the image. pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { if !capturing() { return; } - let keys: HeaderKeys = header_keys(values); + let keys: HeaderKeys = header_key_digests(values); let path: String = relative_path.replace('\\', "/"); capture(|capture| { capture.header_keys.insert(path, keys); }); } - -/// What an edit did to one model's header. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum HeaderChange { - /// The header's keys and values are unchanged. - Unchanged, - /// These keys changed, every one of which the delta recompiles. - Patched(BTreeSet), - /// Why the delta cannot recompile this header change. - Unsupported(String), - /// The file does not parse; the delta reports its error as a full compile would. - Unparsed, -} - -/// How the header of the model file at `path` changed against its recorded keys. -pub(crate) fn header_change( - project_dir: &Path, - relative_path: &str, - (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), -) -> HeaderChange { - let Some(stored) = stored else { - return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); - }; - let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { - return HeaderChange::Unparsed; - }; - let file_path: String = project_dir - .join(relative_path) - .to_string_lossy() - .into_owned(); - let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { - return HeaderChange::Unparsed; - }; - if !references_read(&parsed.query_sql, upstream) { - return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); - } - let current: HeaderKeys = header_keys(&parsed.header_values); - let stored_map: BTreeMap<&str, &str> = stored - .iter() - .map(|(key, digest)| (key.as_str(), digest.as_str())) - .collect(); - let current_map: BTreeMap<&str, &str> = current - .iter() - .map(|(key, digest)| (key.as_str(), digest.as_str())) - .collect(); - let changed: BTreeSet = stored_map - .keys() - .chain(current_map.keys()) - .filter(|key| stored_map.get(*key) != current_map.get(*key)) - .map(|key| (*key).to_owned()) - .collect(); - if changed.is_empty() { - return HeaderChange::Unchanged; - } - for key in &changed { - if key == MATERIALIZED_KEY { - let value = parsed.header_values.iter().find(|(name, _)| name == key); - let supported: bool = matches!( - value, - Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) - if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) - ); - if !supported { - return HeaderChange::Unsupported(format!( - "{relative_path} changed its materialization beyond views and tables" - )); - } - } else if !PATCHED_KEYS.contains(&key.as_str()) { - return HeaderChange::Unsupported(format!( - "{relative_path} changed its {key} header key" - )); - } - } - HeaderChange::Patched(changed) -} - -/// Whether a custom rule reads a fact of this model that the changed header keys decide. -pub(crate) fn custom_rule_reads( - fact_digests: &[(Vec, String)], - model: (&str, &str), - changed: &BTreeSet, -) -> Option { - let (path, name) = model; - for (key, _) in fact_digests { - let Some((fact, arguments)) = key.split_first() else { - continue; - }; - if !arguments - .iter() - .any(|argument| argument == path || argument == name) - { - continue; - } - let reads: &[&str] = match fact.as_str() { - "contracts.enforced" | "contract" => &["contract"], - "contracts.grain" | "grain" => &["contract", "columns"], - "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], - "materialized" => &["materialized"], - "tests.for_model" | "audits.for_model" => &["audits", "columns"], - _ => &[], - }; - if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { - return Some(format!("a custom rule reads {name}'s {key}")); - } - } - None -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 43c35fbb89..77a3165166 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -29,7 +29,6 @@ pub(crate) mod raised; pub(crate) mod registration; pub(crate) mod scope_snapshot; pub(crate) mod snapshot; -pub(crate) mod sorted_keys; pub(crate) mod sql_test_blocks; pub(crate) mod sql_test_records; pub(crate) mod yaml_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 9975ca2f83..8b78654cca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -10,15 +10,14 @@ use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use std::path::Path; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::model_file_record::DiscoveredPaths; use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::models::NativeProjectTree; +use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; @@ -106,17 +105,6 @@ pub(crate) struct ModelFileSelection<'py> { pub(crate) on_fault: Option>, } -/// The options model discovery parses headers with. -pub(crate) fn model_file_options() -> ModelFileOptions { - ModelFileOptions { - supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), - removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), - extract_implicit_alias_columns: false, - extract_output_column_locations: false, - path_filter: None, - } -} - /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. pub(crate) fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, @@ -124,15 +112,11 @@ pub(crate) fn discovered_model_files<'py>( selection: ModelFileSelection<'py>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; - let options = ModelFileOptions { - supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), - removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), - extract_implicit_alias_columns, - extract_output_column_locations, - path_filter: crate::bindings::_helpers::compile_context::delta::injection() + let options: ModelFileOptions = model_file_options( + extraction, + crate::bindings::_helpers::compile_context::delta::injection() .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), - }; + ); let items: Vec> = model_items(project_dir, &options)?; let faults = FileFaults { project_dir: project_dir.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 5d2de99d53..df25bb5648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -17,14 +17,12 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::project_files::path_object; use crate::bindings::_helpers::discovery::raised::{ discovery_error, failure_error, read_error, stage_error, }; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::discovery::sql_test_blocks::{ sql_test_block_records, validate_block_names, }; @@ -33,6 +31,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::NativeProjectTree; +use sqlbuild_discovery::constants; +use sqlbuild_discovery::sql_tests::main::sql_test_file_options::sql_test_file_options; const SQL_TESTS_OWNERSHIP_ROOT: &str = "tests/unit"; const SQL_SCENARIOS_OWNERSHIP_ROOT: &str = "tests/scenarios"; @@ -54,12 +54,10 @@ type RecordBuilder<'py, T> = fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; fn options() -> PyResult { - Ok(SqlTestFileOptions { - test_keys: sorted_keys(&constants::SQL_TEST_HEADER_KEYS), - scenario_keys: sorted_keys(&constants::SQL_SCENARIO_HEADER_KEYS), - path_filter: crate::bindings::_helpers::compile_context::delta::injection() + Ok(sql_test_file_options( + crate::bindings::_helpers::compile_context::delta::injection() .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), - }) + )) } fn path_class(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index c7a8584120..cf3ba5e5b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -15,6 +15,9 @@ use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_h use sqlbuild_model_config::header_metadata::models::{DynamicFamily, ParsedColumn}; use sqlbuild_model_config::model_validation::models::ColumnMigration; +use crate::bindings::_helpers::compile_driver::stage_work::{ + NATIVE_HEADER_COLUMNS, PARSED_HEADER_COLUMNS, count_stage, +}; use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{ @@ -107,6 +110,13 @@ pub(crate) fn header_schema<'py>( let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); let model: (&str, &str) = (source.name, source.relative_path); let native: Option<&[NativeColumn]> = native_columns(context, source, columns.as_ref())?; + if columns.is_some() { + let stage = match native { + Some(_) => NATIVE_HEADER_COLUMNS, + None => PARSED_HEADER_COLUMNS, + }; + count_stage(py, stage, 1)?; + } let python_columns: Option> = if native.is_some() { None } else { columns }; let metadata = parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs index f1312f45ac..5c38ef1c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs @@ -58,7 +58,8 @@ pub(crate) const AUDIT_DECLARATION: NamedKind = NamedKind { label: "Generic audit", role: "audits/generic", }; -use crate::bindings::_helpers::scopes::compile_usages::{CURRENT_FOLDER, scope_path_message}; +use crate::bindings::_helpers::scopes::compile_usages::CURRENT_FOLDER; +use sqlbuild_scopes::scope_index::main::declared_scope_path_message::declared_scope_path_message; /// The usage of the indexed declaration `name`, if any, reporting S006 when it is hidden. pub(crate) fn named_usages<'py>( @@ -114,7 +115,7 @@ fn path_error(context: &LoopContext<'_, '_>, declared: &DeclaredScope, label: &s match context .classes .scope_path_error - .call1((scope_path_message(declared, label),)) + .call1((declared_scope_path_message(declared, label),)) { Ok(instance) => PyErr::from_value(instance), Err(failure) => failure, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs new file mode 100644 index 0000000000..aa1bbe6cdc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs @@ -0,0 +1,261 @@ +//! Python config values converted once into `ConfigNode`s, and projected back into Python objects. + +use std::sync::Arc; + +use pyo3::IntoPyObject; +use pyo3::exceptions::{PyOverflowError, PyUnicodeEncodeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, PyTupleMethods, +}; +use sqlbuild_model_config::config_values::models::{ + ConfigNode, HostChildren, HostFacts, HostValue, +}; +use sqlbuild_model_config::config_values::types::HostObject; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::records::construct::record_class; +use crate::bindings::_helpers::records::ids::RecordId; + +/// The typed hook entry classes, which the config build checks hook lists against. +const HOOK_ENTRY_CLASSES: [RecordId; 3] = [ + RecordId::SqlHookEntry, + RecordId::NamedSqlHookEntry, + RecordId::PythonHookEntry, +]; + +/// The Python object an opaque config value stands for. +#[derive(Clone, Debug)] +pub(crate) struct HostHandle(pub(crate) Arc>); + +impl HostObject for HostHandle { + fn is_date_like(&self) -> bool { + Python::attach(|py| PyNode(self.0.bind(py).clone()).is_date_like()) + } + + /// Python's text of the projected value; a `str()` or `repr()` that raises reads as empty, as `PyNode`. + fn display(node: &ConfigNode, repr: bool) -> String { + Python::attach(|py| { + config_object(py, node).map_or_else( + |_| String::new(), + |value| { + let value: PyNode<'_> = PyNode(value); + if repr { + value.python_repr() + } else { + value.python_str() + } + }, + ) + }) + } +} +/// A config value converted from Python. +pub(crate) type PyConfigNode = ConfigNode; +/// A converted config mapping's entries. +pub(crate) type PyConfigEntries = Vec<(PyConfigNode, PyConfigNode)>; + +/// Whether a Python `str` encodes to UTF-8, which it does unless it holds a lone surrogate. +fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(_) => Ok(true), + Err(error) if error.is_instance_of::(text.py()) => Ok(false), + Err(error) => Err(error), + } +} + +/// An opaque value with the cheap facts `PyNode` reads of it, and its converted items. +fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResult { + let node: PyNode<'_> = PyNode(value.clone()); + let strict_text: bool = match value.downcast::() { + Ok(text) => utf8_text(text)?, + Err(_) => false, + }; + Ok(ConfigNode::Host(Box::new(HostValue { + handle: HostHandle(Arc::new(value.clone().unbind())), + facts: HostFacts { + kind: node.kind(), + text: node.text(), + strict_text, + integer: node.integer(), + number: node.number(), + }, + children, + }))) +} + +/// Converted items, or `None` when a container holds itself; `open` holds the enclosing containers. +fn converted_items<'py>( + items: impl Iterator>, + open: &[usize], +) -> PyResult>> { + let mut converted: Vec = Vec::new(); + for item in items { + match convert(&item, open)? { + Some(node) => converted.push(node), + None => return Ok(None), + } + } + Ok(Some(converted)) +} + +fn converted_entries( + mapping: &Bound<'_, PyDict>, + open: &[usize], +) -> PyResult> { + let mut converted: PyConfigEntries = Vec::with_capacity(mapping.len()); + for (key, item) in mapping.iter() { + let (Some(key), Some(item)) = (convert(&key, open)?, convert(&item, open)?) else { + return Ok(None); + }; + converted.push((key, item)); + } + Ok(Some(converted)) +} + +/// A container's converted children, or `None` when it holds itself. +fn container( + value: &Bound<'_, PyAny>, + enclosing: &[usize], +) -> PyResult>> { + let address: usize = value.as_ptr() as usize; + if enclosing.contains(&address) { + return Ok(None); + } + let open: Vec = [enclosing, &[address]].concat(); + Ok(if let Ok(mapping) = value.downcast::() { + converted_entries(mapping, &open)?.map(HostChildren::Entries) + } else if let Ok(items) = value.downcast::() { + converted_items(items.iter(), &open)?.map(HostChildren::Items) + } else if let Ok(items) = value.downcast::() { + converted_items(items.iter(), &open)?.map(HostChildren::Items) + } else { + Some(HostChildren::None) + }) +} + +fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { + if value.is_none() { + return Ok(Some(ConfigNode::Null)); + } + if value.is_exact_instance_of::() { + return Ok(Some(ConfigNode::Bool(value.is_truthy()?))); + } + if value.is_exact_instance_of::() { + match value.extract::() { + Ok(number) => return Ok(Some(ConfigNode::Int(number))), + Err(error) if error.is_instance_of::(value.py()) => {} + Err(error) => return Err(error), + } + } + if value.is_exact_instance_of::() { + return Ok(Some(ConfigNode::Float(value.extract::()?))); + } + if let Ok(text) = value.downcast_exact::() + && utf8_text(text)? + { + return Ok(Some(ConfigNode::Str(text.to_str()?.to_owned()))); + } + let Some(children) = container(value, open)? else { + return Ok(None); + }; + let exact: bool = value.is_exact_instance_of::() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::(); + Ok(Some(match children { + HostChildren::Entries(entries) if exact => ConfigNode::Map(entries), + HostChildren::Items(items) if exact && value.is_exact_instance_of::() => { + ConfigNode::List(items) + } + HostChildren::Items(items) if exact => ConfigNode::Tuple(items), + children => host(value, children)?, + })) +} + +/// An exact `list`, `tuple` or `dict` carried as its own object, its items never converted. +pub(crate) fn opaque_container(value: &Bound<'_, PyAny>) -> PyResult { + host(value, HostChildren::None) +} + +/// Whether `value` is an exact `list`, `tuple` or `dict`. +pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { + value.is_exact_instance_of::() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::() +} + +/// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. +pub(crate) fn config_node(value: &Bound<'_, PyAny>) -> PyResult { + match convert(value, &[])? { + Some(node) => Ok(node), + None => host(value, HostChildren::None), + } +} + +/// A Python mapping's entries as config entries. +pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult { + mapping + .iter() + .map(|(key, value)| Ok((config_node(&key)?, config_node(&value)?))) + .collect() +} + +/// The Python object of one config value: new builtins, and opaque values as themselves. +pub(crate) fn config_object<'py>( + py: Python<'py>, + node: &PyConfigNode, +) -> PyResult> { + Ok(match node { + ConfigNode::Null => py.None().into_bound(py), + ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), + ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), + ConfigNode::Str(text) => PyString::new(py, text).into_any(), + ConfigNode::List(items) => PyList::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Tuple(items) => PyTuple::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), + ConfigNode::Host(host) => host.handle.0.bind(py).clone(), + }) +} + +/// A new Python `dict` of config entries. +pub(crate) fn config_dict<'py>( + py: Python<'py>, + entries: &[(PyConfigNode, PyConfigNode)], +) -> PyResult> { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + mapping.set_item(config_object(py, key)?, config_object(py, value)?)?; + } + Ok(mapping) +} + +/// Whether `node` kind is a Python `str` instance, as `isinstance(value, str)` reads it. +pub(crate) fn is_text_node(node: &PyConfigNode) -> bool { + node.kind() == NodeKind::Str +} + +pub(crate) fn hook_entry(value: &Bound<'_, PyAny>) -> PyResult { + for class in HOOK_ENTRY_CLASSES { + if value.is_instance(&record_class(value.py(), class)?)? { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index 2bc2ff7200..cc1da29c7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -1,29 +1,23 @@ //! Typed values read from raw authored configuration mappings, rejecting the wrong types. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyBool, PyInt, PyList, PyString, PyTuple}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; +use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; +use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; +use sqlbuild_model_config::config_values::models::CursorBound; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_node}; +use crate::bindings::_helpers::model_config::typed_config::type_error; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::ErrorId; -const DATETIME_MODULE: &str = "datetime"; - -/// `values.get(key)`, or `None` when the value is absent or `None`. -fn authored<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { - let value: Bound<'py, PyAny> = mapping_get(values, key)?; - Ok(Some(value).filter(|value| !value.is_none())) -} - -/// `ConfigValueTypeError(key=key, expected=expected, actual_type=type(value))`. -fn wrong_type(key: &str, expected: &str, value: &Bound<'_, PyAny>) -> PyErr { - match config_value_type_error(key, expected, &value.get_type().into_any()) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} +const ISOFORMAT_METHOD: &str = "isoformat"; /// The `ConfigValueTypeError` naming `key`, the `expected` kind and the value's `actual_type`. pub(crate) fn config_value_type_error<'py>( @@ -51,6 +45,15 @@ pub(crate) fn config_value_type_error<'py>( ) } +/// `values.get(key)` as a config node. +fn authored_node<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult<(PyConfigNode, Bound<'py, PyAny>)> { + let value = mapping_get(values, key)?; + Ok((config_node(&value)?, value)) +} + /// An authored string, preserving a present empty string. #[pyfunction(name = "get_config_str")] #[pyo3(signature = (*, values, key))] @@ -58,13 +61,10 @@ pub(crate) fn config_str<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if !value.is_instance_of::() { - return Err(wrong_type(key, "a string", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// An authored boolean, or `None` when the key is absent or `None`. @@ -74,13 +74,10 @@ pub(crate) fn config_bool<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if !value.is_instance_of::() { - return Err(wrong_type(key, "a boolean", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_flag(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// An authored sequence of strings as a tuple, or `None` when absent or `None`. @@ -90,16 +87,13 @@ pub(crate) fn config_string_tuple<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { + let (node, value) = authored_node(values, key)?; + let checked = + checked_texts(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + if checked.is_none() { return Ok(None); - }; - if !(value.is_instance_of::() || value.is_instance_of::()) { - return Err(wrong_type(key, "a list or tuple of strings", &value)); } let items: Vec> = value.try_iter()?.collect::>()?; - if !items.iter().all(|item| item.is_instance_of::()) { - return Err(wrong_type(key, "a list or tuple of strings", &value)); - } PyTuple::new(values.py(), items).map(Some) } @@ -110,13 +104,10 @@ pub(crate) fn config_int<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if value.is_instance_of::() || !value.is_instance_of::() { - return Err(wrong_type(key, "an integer", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_integer(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// A cursor bound as text from a string, integer, date or datetime. @@ -126,26 +117,14 @@ pub(crate) fn config_cursor_bound<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - let py = values.py(); - let date: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "date")?; - let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - let dated: bool = value.is_instance(&date)? || value.is_instance(&datetime)?; - if value.is_instance_of::() - || !(dated || value.is_instance_of::() || value.is_instance_of::()) - { - return Err(wrong_type( - key, - "a string, integer, date, or datetime", - &value, - )); - } - if dated { - return value.call_method0("isoformat").map(Some); - } - value.str().map(|text| Some(text.into_any())) + let (node, value) = authored_node(values, key)?; + let checked = + checked_cursor_bound(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(match checked { + None => None, + Some(CursorBound::Dated(_)) => Some(value.call_method0(ISOFORMAT_METHOD)?), + Some(CursorBound::Stringified(_)) => Some(value.str()?.into_any()), + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs index 3dcabf33db..a81aff8a36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -1,13 +1,20 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. +use std::collections::HashMap; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyStringMethods}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::config_values::main::config_entry::config_entry; +use sqlbuild_model_config::config_values::main::layered_config::layered_config; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_model_config::types::AuthoredNode; -use crate::bindings::_helpers::model_config::model_config_builder::{ - layered_values, raised_config_error, +use crate::bindings::_helpers::model_config::config_nodes::{ + PyConfigEntries, PyConfigNode, config_entries, config_node, config_object, is_text_node, }; +use crate::bindings::_helpers::model_config::model_config_builder::raised_config_error; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; use crate::bindings::_helpers::records::fields::field as native_field; @@ -15,11 +22,11 @@ const DESCRIPTION_KEY: &str = "description"; const MODEL_SCHEMA_KEY: &str = "model_schema"; /// Whether `value` is a string with non-whitespace text. -fn non_empty_text(value: Option>) -> PyResult { - let Some(value) = value.filter(|value| value.is_instance_of::()) else { - return Ok(false); - }; - value.call_method0("strip")?.is_truthy() +fn non_empty_text(value: Option<&PyConfigNode>) -> bool { + value + .filter(|value| is_text_node(value)) + .and_then(|value| value.with_text(|text| !python_strip(text).is_empty())) + .unwrap_or(false) } /// The file path of each model whose layered config or model schema gives it a description. @@ -29,8 +36,10 @@ pub(crate) fn externally_described_models<'py>( ) -> PyResult>> { let py = inputs.py(); let project_config: Bound<'py, PyAny> = native_field(&inputs, "project_config")?; - let defaults: Bound<'py, PyDict> = - project_defaults_mapping(&native_field(&project_config, "defaults")?)?; + let defaults: PyConfigEntries = config_entries(&project_defaults_mapping(&native_field( + &project_config, + "defaults", + )?)?)?; let path_defaults: Bound<'py, PyDict> = native_field(&project_config, "path_defaults")?.downcast_into()?; let path_keys: Vec = path_defaults @@ -38,34 +47,47 @@ pub(crate) fn externally_described_models<'py>( .iter() .map(|key| key.extract::()) .collect::>()?; + let text = |item: &PyConfigNode| -> PyResult { + Ok(config_object(py, item)? + .str()? + .to_string_lossy() + .into_owned()) + }; + let mut layers: HashMap = HashMap::new(); let mut described: Vec> = Vec::new(); for model_file in native_field(&inputs, "model_files")?.try_iter()? { let model_file: Bound<'py, PyAny> = model_file?; let relative_path: String = native_field(&model_file, "relative_path")? .str()? .extract()?; - let mut values: Bound<'py, PyDict> = defaults.copy()?; + let mut values: PyConfigEntries = defaults.clone(); match select_path_default(&relative_path, &path_keys) { PathDefaultChoice::Selected(Some(key)) => { - let layer: Bound<'py, PyDict> = - path_defaults.as_any().get_item(&key)?.downcast_into()?; - values = layered_values(py, &values, &layer)?; + if !layers.contains_key(&key) { + let layer: Bound<'py, PyDict> = + path_defaults.as_any().get_item(&key)?.downcast_into()?; + layers.insert(key.clone(), config_entries(&layer)?); + } + values = layered_config(&values, &layers[&key], &text)?.entries; } PathDefaultChoice::Selected(None) => {} PathDefaultChoice::Conflict(error) => return Err(raised_config_error(py, error)), } let header: Bound<'py, PyDict> = native_field(&model_file, "header_values")?.downcast_into()?; - values = layered_values(py, &values, &header)?; - let schema_description: Option> = - match values.get_item(MODEL_SCHEMA_KEY)? { - Some(name) if name.is_instance_of::() => model_schemas - .get_item(name)? - .map(|schema| schema.getattr(DESCRIPTION_KEY)) - .transpose()?, - _ => None, - }; - if non_empty_text(values.get_item(DESCRIPTION_KEY)?)? || non_empty_text(schema_description)? + values = layered_config(&values, &config_entries(&header)?, &text)?.entries; + let schema_description: Option = match config_entry(&values, MODEL_SCHEMA_KEY) + { + Some(name) if is_text_node(name) => model_schemas + .get_item(config_object(py, name)?)? + .map(|schema| schema.getattr(DESCRIPTION_KEY)) + .transpose()? + .map(|description| config_node(&description)) + .transpose()?, + _ => None, + }; + if non_empty_text(config_entry(&values, DESCRIPTION_KEY)) + || non_empty_text(schema_description.as_ref()) { described.push(native_field(&model_file, "file_path")?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index e428f56314..4ebfd08885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod authored_nodes; pub(crate) mod column_migrations; pub(crate) mod config_classes; pub(crate) mod config_errors; +pub(crate) mod config_nodes; pub(crate) mod config_templates; pub(crate) mod config_values; pub(crate) mod contract_classes; @@ -15,4 +16,5 @@ pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; +pub(crate) mod typed_config; pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 6846442d16..b634471b40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -1,103 +1,46 @@ -//! Effective model config layered and templated natively, as `build_model_config` builds it. +//! The native model config build handed Python inputs once, and its results projected for Python. -use pyo3::exceptions::PyKeyError; +use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, - PyTupleMethods, -}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; -use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::model_validation::main::retention_override::retention_override; -use sqlbuild_model_config::model_validation::main::table_type_override::table_type_override; +use sqlbuild_model_config::model_config_build::main::build_model_config::build_model_config; +use sqlbuild_model_config::model_config_build::main::inherited_config::inherited_config; +use sqlbuild_model_config::model_config_build::main::model_context_values::model_context_values; +use sqlbuild_model_config::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, HeaderEntryFacts, ModelHeader, +}; +use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; -use sqlbuild_model_config::templates::models::TemplateOptions; -use sqlbuild_model_config::types::AuthoredNode; +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_classes::{ConfigClasses, built_config}; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::config_templates::{ - PythonHost, Stop, TemplateSources, expanded, template_error, +use crate::bindings::_helpers::model_config::config_nodes::{ + HostHandle, PyConfigEntries, PyConfigNode, config_dict, config_entries, config_node, + config_object, hook_entry, is_exact_container, opaque_container, }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; -const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; -/// Hook keys in the sorted order `validate_model_hook_config` checks them. -const SORTED_HOOK_KEYS: [&str; 2] = ["post_hooks", "pre_hooks"]; -const MODEL_CONFIG_LABEL: &str = "model config"; -const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; -const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; -const TAGS_KEY: &str = "tags"; -const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; -const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; -const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; -const DATABASE_KEY: &str = "database"; -const SCHEMA_KEY: &str = "schema"; -const ALIAS_KEY: &str = "alias"; -const MATERIALIZED_KEY: &str = "materialized"; -const FULL_REFRESH_KEY: &str = "full_refresh"; -const RETENTION_KEY: &str = "time_travel_retention"; -const TABLE_TYPE_KEY: &str = "table_type"; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -const PRESERVE_TARGET_VALUE: &str = "preserve"; -const RUN_ID_CONTEXT: &str = "run.id"; -const RUN_TARGET_CONTEXT: &str = "run.target"; -const MODEL_NAME_CONTEXT: &str = "model.name"; -const MODEL_DATABASE_CONTEXT: &str = "model.database"; -const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; -const MODEL_ALIAS_CONTEXT: &str = "model.alias"; -const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; -const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; -const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; -const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; - -/// Why the native build stops: the config error, or a Python error. -enum Halt { - Config(ConfigError), - Error(PyErr), -} - -impl From for Halt { - fn from(error: PyErr) -> Self { - Self::Error(error) - } -} - -impl From for Halt { - fn from(error: ConfigError) -> Self { - Self::Config(error) - } -} - -/// The `CompileInputError` `model '' `, raised by the model config build. -fn model_error(model_name: &str, text: &str) -> Halt { - Halt::Config(ConfigError::compile(format!("model '{model_name}' {text}"))) -} +/// The config keys a model's context values read. +const CONTEXT_KEYS: [&str; 3] = ["database", "schema", "alias"]; +const BUILD_STOPPED: &str = "a config value read as text without an error"; -type Built = Result; - -/// The project-wide inputs every model's config build reads, captured once per compile. +/// The project-wide inputs every model's config build reads, converted once per compile. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct NativeModelConfigBuilder { - defaults: Py, - path_defaults: Py, + inputs: ConfigBuildInputs, path_keys: Vec, - hook_entry_types: Py, - variables: Py, environment: Py, - target_name: Option>, - run_id: Py, - target_namespace: Option<(Py, Py)>, } /// One model's built config before Python wraps it in `CompileModelConfig`. @@ -124,6 +67,123 @@ pub(crate) type TargetInputs<'py> = ( Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, ); +/// The process environment and project-var rendering, read through Python. +struct PythonConfigHost<'py> { + environment: Bound<'py, PyAny>, +} + +impl ConfigBuildHost for PythonConfigHost<'_> { + type Handle = HostHandle; + type Error = PyErr; + + fn environment(&self, name: &str) -> PyResult> { + if !self.environment.contains(name)? { + return Ok(None); + } + Ok(Some(config_node(&self.environment.get_item(name)?)?)) + } + + fn render(&self, value: &PyConfigNode, label: &str) -> PyResult> { + let py: Python<'_> = self.environment.py(); + let object: Bound<'_, PyAny> = config_object(py, value)?; + if let Ok(text) = object.downcast_exact::() { + return Ok(Ok(text.to_str()?.to_owned())); + } + match project_var_text(&object, label) { + Ok(text) => Ok(Ok(text)), + Err(error) if error.is_instance_of::(py) => { + Ok(Err(error.value(py).to_string())) + } + Err(error) => Err(error), + } + } + + fn text_error(&self, value: &PyConfigNode) -> PyErr { + let py: Python<'_> = self.environment.py(); + let read = || -> PyResult<()> { + let object: Bound<'_, PyAny> = config_object(py, value)?; + let text: Bound<'_, PyString> = match object.downcast::() { + Ok(text) => text.clone(), + Err(_) => object.str()?, + }; + let _ = text.to_str()?; + Ok(()) + }; + match read() { + Ok(()) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error, + } + } + + fn python_str(&self, value: &PyConfigNode, exact: bool) -> PyResult { + let text: Bound<'_, PyString> = config_object(self.environment.py(), value)?.str()?; + if exact { + return Ok(text.to_str()?.to_owned()); + } + Ok(text.to_string_lossy().into_owned()) + } + + fn typed_hook_entry(&self, value: &PyConfigNode) -> PyResult { + hook_entry(&config_object(self.environment.py(), value)?) + } +} + +/// Header keys whose values the build reads item by item, which are always converted. +const INSPECTED_KEYS: [&str; 7] = [ + "tags", + "row_diff_exclude_columns", + "row_diff_tolerances", + "pre_hooks", + "post_hooks", + "time_travel_retention", + "table_type", +]; + +/// Whether `key`'s container may be carried whole: no template or macro in it, items never read. +fn carried_whole(key: &str, facts: Option<&[(String, HeaderEntryFacts)]>) -> bool { + !INSPECTED_KEYS.contains(&key) + && facts + .and_then(|facts| facts.iter().find(|(name, _)| name == key)) + .is_some_and(|(_, facts)| !facts.template && facts.macro_path == MacroPath::Absent) +} + +/// A header's entries as config nodes; containers the build never looks into stay opaque. +fn header_entries( + header: &Bound<'_, PyDict>, + facts: Option<&[(String, HeaderEntryFacts)]>, +) -> PyResult { + let mut entries: PyConfigEntries = Vec::with_capacity(header.len()); + for (key, value) in header.iter() { + let whole: bool = is_exact_container(&value) + && key + .downcast_exact::() + .is_ok_and(|key| key.to_str().is_ok_and(|key| carried_whole(key, facts))); + let node: PyConfigNode = if whole { + opaque_container(&value)? + } else { + config_node(&value)? + }; + entries.push((config_node(&key)?, node)); + } + Ok(entries) +} + +/// One header entry's parsed facts as the build reads them. +fn entry_facts((key, entry): &(String, EntryFacts)) -> (String, HeaderEntryFacts) { + let facts: HeaderEntryFacts = HeaderEntryFacts { + template: entry.template, + macro_path: entry.macro_path.clone(), + }; + (key.clone(), facts) +} + +fn optional_node(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + config_node(&value).map(Some) +} + #[pymethods] impl NativeModelConfigBuilder { #[new] @@ -133,25 +193,31 @@ impl NativeModelConfigBuilder { target: TargetInputs<'_>, ) -> PyResult { let (run, target_namespace) = target; - let (defaults, path_defaults, hook_entry_types) = layers; - let defaults: Bound<'_, PyDict> = project_defaults_mapping(&defaults)?; - let path_keys = path_defaults - .keys() - .iter() - .map(|key| key.extract::()) - .collect::>>()?; + let (defaults, path_defaults, _hook_entry_types) = layers; + let mut path_keys: Vec = Vec::with_capacity(path_defaults.len()); + let mut path_values: Vec<(String, PyConfigEntries)> = + Vec::with_capacity(path_defaults.len()); + for (key, values) in path_defaults.iter() { + let key: String = key.extract()?; + path_values.push((key.clone(), config_entries(values.downcast::()?)?)); + path_keys.push(key); + } let (target_name, run_id) = run; Ok(Self { - defaults: defaults.unbind(), - path_defaults: path_defaults.unbind(), + inputs: ConfigBuildInputs { + defaults: config_entries(&project_defaults_mapping(&defaults)?)?, + path_defaults: path_values, + variables: config_entries(&sources.0)?, + run_id: config_node(&run_id)?, + target_name: optional_node(target_name)?.unwrap_or(ConfigNode::Null), + target_namespace: target_namespace + .map(|(database, schema)| -> PyResult<_> { + Ok((config_node(&database)?, config_node(&schema)?)) + }) + .transpose()?, + }, path_keys, - hook_entry_types: hook_entry_types.unbind(), - variables: sources.0.unbind(), environment: sources.1.unbind(), - target_name: (!target_name.is_none()).then(|| target_name.unbind()), - run_id: run_id.unbind(), - target_namespace: target_namespace - .map(|(database, schema)| (database.unbind(), schema.unbind())), }) } @@ -176,7 +242,85 @@ impl NativeModelConfigBuilder { } } +/// The Python error a halt raises, or the config error it is rejected with. +fn halted<'py>( + py: Python<'py>, + halt: ConfigBuildHalt, +) -> PyResult> { + Ok(match halt { + ConfigBuildHalt::Config(error) => Ok(error), + ConfigBuildHalt::Host(error) => Err(error), + ConfigBuildHalt::MissingPathDefault(key) => Err(PyKeyError::new_err(key)), + ConfigBuildHalt::KeyNotText(key) => { + Err(match config_object(py, &key)?.downcast::() { + Ok(_) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error.into(), + }) + } + }) +} + +/// The built values as a new `dict`; entries still the header's own keep the header's objects. +fn own_values_dict<'py>( + header: &Bound<'py, PyDict>, + built: &BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in &built.values { + let own: bool = own_key(key, &built.own_header_keys); + let key: Bound<'py, PyAny> = config_object(py, key)?; + let value: Bound<'py, PyAny> = match own.then(|| header.get_item(&key)).transpose()? { + Some(Some(value)) => value, + _ => config_object(py, value)?, + }; + mapping.set_item(key, value)?; + } + Ok(mapping) +} + +/// Whether `key` is a string key named among the header's own keys. +fn own_key(key: &PyConfigNode, own: &[String]) -> bool { + let text: Option<&str> = match key { + ConfigNode::Str(text) => Some(text), + ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.as_deref(), + _ => None, + }; + text.is_some_and(|text| own.iter().any(|name| name == text)) +} + +/// One model's build projected for Python. +fn projected_build<'py>( + header: &Bound<'py, PyDict>, + built: BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + Ok(BuiltConfig { + values: own_values_dict(header, &built)?, + header_keys: built.header_keys, + logical_schema: projected(py, built.logical_schema)?, + layer_schema_configured: built.layer_schema_configured, + logical_database: projected(py, built.logical_database)?, + retention: built.retention, + table_type: built.table_type, + reads: built.reads, + }) +} + +fn projected<'py>( + py: Python<'py>, + value: Option, +) -> PyResult>> { + value.map(|value| config_object(py, &value)).transpose() +} + impl NativeModelConfigBuilder { + fn host<'py>(&self, py: Python<'py>) -> PythonConfigHost<'py> { + PythonConfigHost { + environment: self.environment.bind(py).clone(), + } + } + /// The selected path-default key, or the conflict error Python raises. pub(crate) fn selected_path_default( &self, @@ -197,10 +341,25 @@ impl NativeModelConfigBuilder { matched_path_default: Option<&str>, model_name: &str, ) -> PyResult> { - match self.build_config(py, (header, facts), matched_path_default, model_name) { - Ok(built) => Ok(ConfigBuild::Built(built)), - Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), - Err(Halt::Error(error)) => Err(error), + let facts: Option> = + facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); + let entries: PyConfigEntries = header_entries(header, facts.as_deref())?; + let model_header = ModelHeader { + entries: &entries, + facts: facts.as_deref(), + }; + let host: PythonConfigHost<'py> = self.host(py); + match build_model_config( + (&host, &self.inputs), + model_header, + matched_path_default, + model_name, + ) { + Ok(built) => Ok(ConfigBuild::Built(projected_build(header, built)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), + Err(error) => Err(error), + }, } } @@ -210,33 +369,16 @@ impl NativeModelConfigBuilder { py: Python<'py>, matched_path_default: Option<&str>, ) -> PyResult, Py>> { - let layered = (|| -> Built> { - let mut layered = self.project_defaults(py)?; - if let Some(key) = matched_path_default { - layered = merged_layer(py, &layered, &self.path_default_values(py, key)?)?; - } - merged_layer(py, &layered, &PyDict::new(py)) - })(); - match layered { - Ok(values) => Ok(Ok(values)), - Err(Halt::Config(error)) => Ok(Err(native_config_error(py, error)?)), - Err(Halt::Error(error)) => Err(error), + let host: PythonConfigHost<'py> = self.host(py); + match inherited_config((&host, &self.inputs), matched_path_default) { + Ok(values) => Ok(Ok(config_dict(py, &values)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(Err(native_config_error(py, error)?)), + Err(error) => Err(error), + }, } } - /// `build_run_context_values`: the run's id and target. - fn run_context_values<'py>(&self, py: Python<'py>) -> PyResult> { - let context = PyDict::new(py); - context.set_item(RUN_ID_CONTEXT, self.run_id.bind(py))?; - context.set_item( - RUN_TARGET_CONTEXT, - self.target_name - .as_ref() - .map_or_else(|| py.None().into_bound(py), |item| item.bind(py).clone()), - )?; - Ok(context) - } - /// `build_model_context_values` for a model's config values. pub(crate) fn model_context_values<'py>( &self, @@ -244,349 +386,29 @@ impl NativeModelConfigBuilder { model_name: &str, include_target_values: bool, ) -> PyResult> { - let py: Python<'py> = values.py(); - let none = py.None().into_bound(py); - let database = string_entry(values, DATABASE_KEY)?; - let schema = string_entry(values, SCHEMA_KEY)?; - let alias = match string_entry(values, ALIAS_KEY)? { - Some(alias) => alias, - None => PyString::new(py, model_name).into_any(), - }; - let context = self.run_context_values(py)?; - context.set_item(MODEL_NAME_CONTEXT, model_name)?; - context.set_item(MODEL_DATABASE_CONTEXT, database.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_ALIAS_CONTEXT, &alias)?; - if !include_target_values { - return Ok(context); + let mut read: PyConfigEntries = Vec::with_capacity(CONTEXT_KEYS.len()); + for key in CONTEXT_KEYS { + if let Some(value) = values.get_item(key)? { + read.push((ConfigNode::Str(key.to_owned()), config_node(&value)?)); + } } - let qualified = match (&database, &schema) { - (Some(database), Some(schema)) => Some(format!( - "{}.{}.{}", - text_of(database)?, - text_of(schema)?, - text_of(&alias)? - )), - (None, Some(schema)) => Some(format!("{}.{}", text_of(schema)?, text_of(&alias)?)), - _ => None, - }; - context.set_item( - DESTINATION_DATABASE_CONTEXT, - database.as_ref().unwrap_or(&none), - )?; - context.set_item(DESTINATION_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(DESTINATION_TABLE_CONTEXT, &alias)?; - context.set_item(DESTINATION_QUALIFIED_CONTEXT, qualified)?; - Ok(context) + let context: PyConfigEntries = + model_context_values(&self.inputs, &read, model_name, include_target_values); + config_dict(values.py(), &context) } /// The active target's `(database, schema)`, or `(None, None)` without a target. pub(crate) fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { - match &self.target_namespace { - Some((database, schema)) => PyTuple::new(py, [database.bind(py), schema.bind(py)]), - None => PyTuple::new(py, [py.None(), py.None()]), - } - } - - fn build_config<'py>( - &self, - py: Python<'py>, - input: HeaderInput<'_, 'py>, - matched_path_default: Option<&str>, - model_name: &str, - ) -> Built> { - let header: &Bound<'py, PyDict> = input.0; - check_header_tags(header, model_name)?; - let path_values = matched_path_default - .map(|key| self.path_default_values(py, key)) - .transpose()?; - let mut layered = self.project_defaults(py)?; - if let Some(path_values) = &path_values { - layered = merged_layer(py, &layered, path_values)?; - } - let layered = merged_layer(py, &layered, header)?; - self.check_hooks(py, &layered, model_name)?; - let mut hooks: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); - for key in HOOK_KEYS { - if let Some(value) = layered.get_item(key)? { - hooks.push((key, value)); - layered.del_item(key)?; - } - } - let has_templates = templated(input, &layered)?; - let mut expansion = Expansion { - builder: self, - py, - model_name, - reads: Vec::new(), - }; - let model_values = if has_templates { - let early = expansion.expand( - &layered, - &self.variables.bind(py).clone(), - expansion.run_context()?, - (true, MODEL_CONFIG_LABEL), - )?; - let empty = PyDict::new(py); - let first = expansion.expand( - &early, - &empty, - expansion.model_context(&early, false)?, - (true, MODEL_CONFIG_LABEL), - )?; - expansion.expand( - &first, - &empty, - expansion.model_context(&first, false)?, - (true, MODEL_CONFIG_LABEL), - )? - } else { - layered - }; - let logical_schema = string_entry(&model_values, SCHEMA_KEY)?; - let logical_database = string_entry(&model_values, DATABASE_KEY)?; - let layer_schema_configured = header.contains(SCHEMA_KEY)? - || path_values - .as_ref() - .map(|values| values.contains(SCHEMA_KEY)) - .transpose()? - .unwrap_or(false); - let namespaced = - expansion.namespaced_values(&model_values, &logical_database, &logical_schema)?; - let needs_target_templates = has_templates - || [DATABASE_KEY, SCHEMA_KEY] - .iter() - .map(|key| entry_has_template(&namespaced, key)) - .collect::>>()? - .contains(&true); - let resolved = if needs_target_templates { - let empty = PyDict::new(py); - expansion.expand( - &namespaced, - &empty, - expansion.model_context(&namespaced, true)?, - (false, MODEL_CONFIG_LABEL), - )? - } else { - namespaced - }; - for (key, value) in hooks { - resolved.set_item(key, value)?; - } - let values = storage_free_values(py, &resolved, header)?; - let retention = retention_override( - header.get_item(RETENTION_KEY)?.map(PyNode).as_ref(), - model_name, - )?; - let table_type = table_type_override( - header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref(), - model_name, - )?; - check_no_config_macros(input, &values)?; - Ok(BuiltConfig { - values, - header_keys: sorted_keys(header)?, - logical_schema, - layer_schema_configured, - logical_database, - retention, - table_type, - reads: expansion.reads, - }) - } - - fn path_default_values<'py>(&self, py: Python<'py>, key: &str) -> Built> { - let values = self - .path_defaults - .bind(py) - .get_item(key)? - .ok_or_else(|| PyKeyError::new_err(key.to_owned()))?; - Ok(values.downcast_into::().map_err(PyErr::from)?) - } - - /// Return a fresh `project_defaults_to_mapping` result, with its own `tags` list. - fn project_defaults<'py>(&self, py: Python<'py>) -> Built> { - let defaults = self.defaults.bind(py).copy()?; - if let Some(tags) = defaults.get_item(TAGS_KEY)? { - let tags = tags.downcast_into::().map_err(PyErr::from)?; - defaults.set_item(TAGS_KEY, PyList::new(py, tags.iter())?)?; - } - Ok(defaults) - } - - /// Check hook keys as `validate_model_hook_config` does. - fn check_hooks( - &self, - py: Python<'_>, - values: &Bound<'_, PyDict>, - model_name: &str, - ) -> Built<()> { - let entry_types = self.hook_entry_types.bind(py); - for key in SORTED_HOOK_KEYS { - let Some(value) = values.get_item(key)? else { - continue; - }; - let Some(entries) = sequence_items(&value) else { - return Err(model_error( - model_name, - &format!("{key} must be a list of typed hook entries"), - )); - }; - for entry in entries { - if !entry.is_instance(entry_types)? { - return Err(model_error( - model_name, - &format!( - "{key} entries must use typed inline_sql(...), sql(...), or \ - python(...) hook syntax" - ), - )); - } - } - } - Ok(()) - } -} - -/// The template expansions of one model's build and the reads they recorded. -struct Expansion<'a, 'py> { - builder: &'a NativeModelConfigBuilder, - py: Python<'py>, - model_name: &'a str, - reads: Vec<(&'static str, String)>, -} - -impl<'py> Expansion<'_, 'py> { - /// Expand `values`; `scope` is `preserve_unknown_context` and the context label. - fn expand( - &mut self, - values: &Bound<'py, PyDict>, - variables: &Bound<'py, PyDict>, - context: Bound<'py, PyDict>, - scope: (bool, &str), - ) -> Built> { - let result = self.expand_value(values.as_any(), variables, context, scope)?; - Ok(result.downcast_into::().map_err(PyErr::from)?) - } - - fn expand_value( - &mut self, - value: &Bound<'py, PyAny>, - variables: &Bound<'py, PyDict>, - context: Bound<'py, PyDict>, - scope: (bool, &str), - ) -> Built> { - let (preserve_unknown_context, label) = scope; - let environment = self.builder.environment.bind(self.py).clone(); - let host = PythonHost::new( - self.py, - TemplateSources(variables.clone(), environment, context), - label, - ); - let options = TemplateOptions { - allow_context: true, - preserve_context_tokens: false, - preserve_unknown_context, - }; - let result = expanded(&host, value, options); - self.reads.extend(host.into_reads()); - result.map_err(|stop| match stop { - Stop::Failure(error) => Halt::Config(template_error(&error, label)), - Stop::Python(error) => Halt::Error(error), - }) - } - - fn run_context(&self) -> Built> { - Ok(self.builder.run_context_values(self.py)?) - } - - /// Return `build_model_context_values` for these values. - fn model_context( - &self, - values: &Bound<'py, PyDict>, - include_target_values: bool, - ) -> Built> { - Ok(self - .builder - .model_context_values(values, self.model_name, include_target_values)?) - } - - /// Check preserved namespaces and apply `apply_environment_database_schema_overrides`. - fn namespaced_values( - &mut self, - values: &Bound<'py, PyDict>, - logical_database: &Option>, - logical_schema: &Option>, - ) -> Built> { - let overridden = values.copy()?; - let Some((database, schema)) = &self.builder.target_namespace else { - return Ok(overridden); - }; - let py = self.py; - let namespace = [ - ( - DATABASE_KEY, - database.bind(py).clone(), - logical_database.is_some(), + match &self.inputs.target_namespace { + Some((database, schema)) => PyTuple::new( + py, + [config_object(py, database)?, config_object(py, schema)?], ), - ( - SCHEMA_KEY, - schema.bind(py).clone(), - logical_schema.is_some(), - ), - ]; - let preserved = - |value: &Bound<'py, PyAny>| PyNode(value.clone()).is_text(PRESERVE_TARGET_VALUE); - let missing: Vec<&str> = namespace - .iter() - .filter(|(_, value, logical)| preserved(value) && !logical) - .map(|(key, _, _)| *key) - .collect(); - if !missing.is_empty() { - let dimensions = missing.join(" and "); - return Err(Halt::Config( - ConfigError::compile(format!( - "Model '{}' has no logical {dimensions}, but the selected target sets \ - {dimensions} to 'preserve'", - self.model_name - )) - .with_help(format!( - "Set {dimensions} on the resource or its defaults, or configure a literal \ - target {dimensions}." - )), - )); - } - let context = self.model_context(values, false)?; - for (key, value, _) in namespace { - if value.is_none() || preserved(&value) { - continue; - } - let label = if key == DATABASE_KEY { - ENVIRONMENT_DATABASE_LABEL - } else { - ENVIRONMENT_SCHEMA_LABEL - }; - let variables = self.builder.variables.bind(py).clone(); - let expanded_value = - self.expand_value(&value, &variables, context.copy()?, (false, label))?; - overridden.set_item(key, expanded_value)?; + None => PyTuple::new(py, [py.None(), py.None()]), } - Ok(overridden) } } -/// `merged_layer`, with a config error raised as the Python stage raises it. -pub(crate) fn layered_values<'py>( - py: Python<'py>, - base: &Bound<'py, PyDict>, - overlay: &Bound<'py, PyDict>, -) -> PyResult> { - merged_layer(py, base, overlay).map_err(|halt| match halt { - Halt::Error(error) => error, - Halt::Config(error) => raised_config_error(py, error), - }) -} - /// The Python exception a native config error raises as. pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { let raised = || -> PyResult { @@ -599,266 +421,7 @@ pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { raised().unwrap_or_else(|error| error) } -/// Merge `overlay` over a copy of `base` as `_merged_with_tag_union` does. -fn merged_layer<'py>( - py: Python<'py>, - base: &Bound<'py, PyDict>, - overlay: &Bound<'py, PyDict>, -) -> Built> { - let result = base.copy()?; - result.update(overlay.as_mapping())?; - if let (Some(base_tags), Some(overlay_tags)) = - (set_entry(base, TAGS_KEY)?, set_entry(overlay, TAGS_KEY)?) - { - let tags = merged_strings(&base_tags, &overlay_tags)?; - result.set_item(TAGS_KEY, PyList::new(py, tags)?)?; - } - if let (Some(base_columns), Some(overlay_columns)) = ( - set_entry(base, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, - set_entry(overlay, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, - ) { - let columns = merged_strings(&base_columns, &overlay_columns)?; - result.set_item(ROW_DIFF_EXCLUDE_COLUMNS_KEY, PyTuple::new(py, columns)?)?; - } - if let (Some(base_tolerances), Some(overlay_tolerances)) = ( - set_entry(base, ROW_DIFF_TOLERANCES_KEY)?, - set_entry(overlay, ROW_DIFF_TOLERANCES_KEY)?, - ) { - result.set_item( - ROW_DIFF_TOLERANCES_KEY, - merged_tolerances(&base_tolerances, &overlay_tolerances)?, - )?; - } - Ok(result) -} - -/// Return the strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. -fn merged_strings(base: &Bound<'_, PyAny>, overlay: &Bound<'_, PyAny>) -> Built> { - let mut merged = string_items(base)?; - for item in string_items(overlay)? { - if !merged.contains(&item) { - merged.push(item); - } - } - Ok(merged) -} - -/// Return `_as_string_list(value)`: `str()` of each list or tuple item. -fn string_items(value: &Bound<'_, PyAny>) -> Built> { - let Some(items) = sequence_items(value) else { - return Ok(Vec::new()); - }; - items - .iter() - .map(|item| Ok(item.str()?.to_string_lossy().into_owned())) - .collect() -} - -/// Deep-merge row-diff tolerances by section as `_merge_row_diff_tolerances_mapping` does. -fn merged_tolerances<'py>( - base: &Bound<'py, PyAny>, - overlay: &Bound<'py, PyAny>, -) -> Built> { - let (Ok(base), Ok(overlay)) = (base.downcast::(), overlay.downcast::()) else { - return Ok(overlay.clone()); - }; - let merged = base.copy()?; - for section in AUDIT_OVERRIDE_SECTIONS { - let Some(overlay_section) = set_entry(overlay, section)? else { - continue; - }; - let base_section = set_entry(base, section)?; - if let Some(base_section) = &base_section - && let (Ok(base_section), Ok(overlay_mapping)) = ( - base_section.downcast::(), - overlay_section.downcast::(), - ) - { - let section_values = base_section.copy()?; - section_values.update(overlay_mapping.as_mapping())?; - merged.set_item(section, section_values)?; - } else { - merged.set_item(section, &overlay_section)?; - } - } - for (key, value) in overlay.iter() { - if key.is_instance_of::() - && !AUDIT_OVERRIDE_SECTIONS - .iter() - .any(|section| PyNode(key.clone()).is_text(section)) - { - merged.set_item(key, value)?; - } - } - Ok(merged.into_any()) -} - -/// Return `values.get(key)` unless it is absent or `None`. -fn set_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built>> { - Ok(values.get_item(key)?.filter(|value| !value.is_none())) -} - -/// Return the value when it is a string, as `isinstance(raw, str)` keeps it. -fn string_entry<'py>( - values: &Bound<'py, PyDict>, - key: &str, -) -> PyResult>> { - Ok(values - .get_item(key)? - .filter(pyo3::types::PyAnyMethods::is_instance_of::)) -} - -fn text_of(value: &Bound<'_, PyAny>) -> PyResult { - Ok(value.downcast::()?.to_string_lossy().into_owned()) -} - -fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option>> { - if let Ok(items) = value.downcast::() { - Some(items.iter().collect()) - } else if let Ok(items) = value.downcast::() { - Some(items.iter().collect()) - } else { - None - } -} - -fn entry_has_template(values: &Bound<'_, PyDict>, key: &str) -> Built { - Ok(values - .get_item(key)? - .is_some_and(|value| contains_template(&PyNode(value)))) -} - -/// Check header tags as `_validate_model_header_tags` does. -fn check_header_tags(header: &Bound<'_, PyDict>, model_name: &str) -> Built<()> { - let Some(tags) = set_entry(header, TAGS_KEY)? else { - return Ok(()); - }; - let Ok(tags) = tags.downcast::() else { - return Err(model_error(model_name, "tags must be a list")); - }; - if tags.iter().all(|tag| tag.is_instance_of::()) { - Ok(()) - } else { - Err(model_error(model_name, "tags entries must be strings")) - } -} - -/// Drop storage keys, then `full_refresh` where `build_model_config` drops it. -fn storage_free_values<'py>( - py: Python<'py>, - resolved: &Bound<'py, PyDict>, - header: &Bound<'py, PyDict>, -) -> Built> { - let values = PyDict::new(py); - for (key, value) in resolved.iter() { - let storage = [RETENTION_KEY, TABLE_TYPE_KEY] - .iter() - .any(|name| PyNode(key.clone()).is_text(name)); - if !storage { - values.set_item(key, value)?; - } - } - let incremental = values - .get_item(MATERIALIZED_KEY)? - .is_some_and(|value| PyNode(value).is_text(INCREMENTAL_MATERIALIZATION)); - if !header.contains(FULL_REFRESH_KEY)? && !incremental && values.contains(FULL_REFRESH_KEY)? { - values.del_item(FULL_REFRESH_KEY)?; - } - Ok(values) -} - -/// A model's authored header and, when it was read natively, the header's facts. -type HeaderInput<'a, 'py> = (&'a Bound<'py, PyDict>, Option<&'a HeaderFacts>); - -/// The facts of a config entry whose value is the header's own. -fn header_entry<'a, 'py>( - (header, facts): HeaderInput<'a, 'py>, - entry: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - match facts { - Some(facts) => own_entry(facts, header, entry), - None => Ok(None), - } -} - -/// `contains_template_data(values)`, reading the header's own entries from their facts. -fn templated<'py>(input: HeaderInput<'_, 'py>, values: &Bound<'py, PyDict>) -> Built { - for (key, value) in values.iter() { - let found: bool = match header_entry(input, (&key, &value))? { - Some(facts) => facts.template, - None => contains_template(&PyNode(value)), - }; - if found { - return Ok(true); - } - } - Ok(false) -} - -/// Raise for the first config field outside hooks holding a macro call, as Python's walk does. -fn check_no_config_macros<'py>( - input: HeaderInput<'_, 'py>, - values: &Bound<'py, PyDict>, -) -> Built<()> { - for (key, value) in values.iter() { - if HOOK_KEYS - .iter() - .any(|name| PyNode(key.clone()).is_text(name)) - { - continue; - } - let found: MacroPath = match header_entry(input, (&key, &value))? { - Some(facts) => facts.macro_path.clone(), - None => entry_macro_path(&PyNode(key), &PyNode(value)), - }; - if let MacroPath::Found(path) = found { - return Err(Halt::Config(ConfigError::compile(format!( - "model config field '{}' does not allow macros", - path.join(".") - )))); - } - } - Ok(()) -} - -fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { - let mut keys = header - .keys() - .iter() - .map(|key| { - Ok(key - .downcast::() - .map_err(PyErr::from)? - .to_string_lossy() - .into_owned()) - }) - .collect::>>()?; - keys.sort(); - Ok(keys) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) } - -/// An entry's facts only when its value is the header's own, so replaced values read from Python. -pub(crate) fn own_entry<'a, 'py>( - facts: &'a HeaderFacts, - header: &Bound<'py, PyDict>, - (key, value): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let Ok(name) = key.downcast::() else { - return Ok(None); - }; - let Ok(name) = name.to_str() else { - return Ok(None); - }; - let Some(entry) = facts.entry(name) else { - return Ok(None); - }; - Ok(header - .get_item(name)? - .filter(|own| own.is(value)) - .map(|_| entry)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs new file mode 100644 index 0000000000..593a3ca009 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs @@ -0,0 +1,163 @@ +//! Model config values read as typed config nodes, each key converted on first read and kept. + +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyclass; +use sqlbuild_model_config::config_values::errors::ConfigTypeError; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::models::ConfigNode; + +use crate::bindings::_helpers::compile_driver::stage_work::{TYPED_CONFIG_READS, count_stage}; +use crate::bindings::_helpers::model_config::config_nodes::{ + HostHandle, PyConfigNode, config_node, config_object, +}; +use crate::bindings::_helpers::model_config::config_values::config_value_type_error; +use crate::bindings::_helpers::records::fields::{ + cached_value, field as native_field, mapping_get, store_cached, +}; +use crate::bindings::_helpers::records::models::NativeRecord; + +const CONFIG_FIELD: &str = "config"; +const VALUES_FIELD: &str = "values"; +/// The config record's cache slot holding its typed values; no Python attribute reads it. +const TYPED_VALUES_SLOT: &str = "__typed_values__"; +const POISONED: &str = "typed config values cache is poisoned"; + +/// The typed nodes of a config record's values read so far, by key. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TypedConfigValues { + read: Mutex)>>, +} + +/// A config record's values and the typed nodes read of them, each key converted on first read. +pub(crate) type ConfigValues<'py> = (Bound<'py, PyAny>, Bound<'py, TypedConfigValues>); + +/// `values.get(key)` as a typed node, `None` when absent; each key is converted once. +fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { + let (mapping, typed) = values; + let py: Python<'_> = mapping.py(); + let read: &Mutex)>> = &typed.get().read; + let poisoned = || PyRuntimeError::new_err(POISONED); + let found: Option> = read + .lock() + .map_err(|_| poisoned())? + .iter() + .find(|(name, _)| name == key) + .map(|(_, node)| Arc::clone(node)); + if let Some(node) = found { + return Ok(node); + } + let node: Arc = Arc::new(config_node(&mapping_get(mapping, key)?)?); + count_stage(py, TYPED_CONFIG_READS, 1)?; + read.lock() + .map_err(|_| poisoned())? + .push((key.to_owned(), Arc::clone(&node))); + Ok(node) +} + +/// A config record's values, their typed nodes kept on the record so each key converts once. +pub(crate) fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = config.py(); + let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; + let fresh = || { + Bound::new( + py, + TypedConfigValues { + read: Mutex::new(Vec::new()), + }, + ) + }; + let Ok(record) = config.cast::() else { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + return Ok((values, typed)); + }; + let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { + Some(typed) => typed.into_bound(py).cast_into()?, + None => { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + store_cached( + record, + TYPED_VALUES_SLOT, + &typed.clone().into_any().unbind(), + )?; + typed + } + }; + Ok((values, typed)) +} + +/// A model's config values as typed nodes, read key by key. +pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + typed_values(&native_field(model, CONFIG_FIELD)?) +} + +/// The `ConfigValueTypeError` a typed read raises, naming the value's Python type. +pub(crate) fn type_error(py: Python<'_>, error: &ConfigTypeError<'_, HostHandle>) -> PyErr { + let raised = || -> PyResult { + let actual_type = config_object(py, error.value)?.get_type().into_any(); + Ok(PyErr::from_value(config_value_type_error( + &error.key, + error.expected, + &actual_type, + )?)) + }; + raised().unwrap_or_else(|error| error) +} + +/// The exact text of a string node; a string Python cannot encode raises as reading it does. +pub(crate) fn node_text(py: Python<'_>, node: &PyConfigNode) -> PyResult { + match node { + ConfigNode::Str(text) => Ok(text.clone()), + ConfigNode::Host(host) if host.facts.strict_text => { + Ok(host.facts.text.clone().unwrap_or_default()) + } + _ => config_object(py, node)?.extract(), + } +} + +/// `key`'s string value, or `None` when absent or `None`. +pub(crate) fn config_text( + py: Python<'_>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult> { + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| node_text(py, value)) + .transpose() +} + +/// `key`'s string value as its Python object, or `None` when absent or `None`. +pub(crate) fn config_text_object<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult>> { + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() +} + +/// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. +pub(crate) fn config_text_is( + py: Python<'_>, + values: &ConfigValues<'_>, + key: &str, + expected: &str, +) -> PyResult { + let node: Arc = node(values, key)?; + let text: Option<&PyConfigNode> = + checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; + Ok(match text { + Some(ConfigNode::Str(text)) => text == expected, + Some(ConfigNode::Host(host)) if host.facts.strict_text => { + host.facts.text.as_deref() == Some(expected) + } + _ => false, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 0fa98dc16f..e9ca93cb6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -4,7 +4,7 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -437,12 +437,7 @@ pub(crate) fn model_change<'py>( /// The model's resolved `replay_on_change` policy. fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = model.py(); - let raw: Option = authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - REPLAY_CONFIG_KEY, - )? - .map(|value| value.extract()) - .transpose()?; + let raw: Option = config_text(py, &model_values(model)?, REPLAY_CONFIG_KEY)?; resolve_replay_on_change(raw.as_deref()).map_err(|error| replay_policy_error(py, error)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index 8c91a000be..caf8713de1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,7 +1,7 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; @@ -41,11 +41,9 @@ fn display_action( } fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - Ok(authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .unwrap_or_else(|| model.py().None().into_bound(model.py()))) + let py = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) } fn display_entry<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 58f2542027..3aaab66766 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,7 +1,7 @@ //! Boundary conversions for the column migration planner. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::records::construct::new_record_with; @@ -47,11 +47,9 @@ pub(super) fn native<'py>( } pub(super) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - Ok(authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .unwrap_or_else(|| model.py().None().into_bound(model.py()))) + let py = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) } pub(super) fn definition_hash<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index cf6f809bba..4376b8d325 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -3,7 +3,7 @@ use sqlbuild_host::entries::models::HostId; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords}; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{enum_member, new_record, new_record_with}; @@ -159,12 +159,7 @@ pub(crate) fn by_name<'py>( /// A model config value as text, `None` when absent or not text. pub(crate) fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { - authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .map(|value| value.extract()) - .transpose() + config_text(model.py(), &model_values(model)?, key) } /// The compiled object key of the model `name`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 8f58f43a4b..e209fbbc4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -4,9 +4,6 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; -use crate::bindings::_helpers::discovery::constants::{ - SCENARIO_EXPECTED_COLUMNS_ALL, SCENARIO_EXPECTED_COLUMNS_KEY, -}; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; @@ -37,6 +34,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; use sqlbuild_analysis::compiler::main::scenario_expected_columns::scenario_expected_columns; use sqlbuild_analysis::compiler::main::scenario_reader_sql::scenario_reader_sql; +use sqlbuild_discovery::constants::{SCENARIO_EXPECTED_COLUMNS_ALL, SCENARIO_EXPECTED_COLUMNS_KEY}; #[pyfunction] #[pyo3(signature = (**kwargs))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 9976607ea7..e25f1f94c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_is, model_values}; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -49,11 +49,12 @@ fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult> /// Whether a model's `incremental_mode` is microbatch. fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { - let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; - Ok(match config_str(&values, "incremental_mode")? { - Some(mode) => mode.eq(MICROBATCH_MODE)?, - None => false, - }) + config_text_is( + model.py(), + &model_values(model)?, + "incremental_mode", + MICROBATCH_MODE, + ) } /// A model's watermark input names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index bd93222659..ea1f64a8a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -8,22 +8,20 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::compile_usages::main::distinct_positions::distinct_positions; -use sqlbuild_scopes::compile_usages::main::merged_dependencies::merged_dependencies; +use sqlbuild_scopes::compile_usages::main::merged_usages::merged_usages; use sqlbuild_scopes::compile_usages::main::schema_scope_usages::schema_scope_usages; use sqlbuild_scopes::compile_usages::models::{ - DependencySource, InaccessibleDiagnostic, NamedRecord, SchemaDeclaration, SchemaUsages, + DependencySource, InaccessibleDiagnostic, MergedUsages, NamedRecord, SchemaDeclaration, + SchemaUsages, UsageKey, }; -use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::declared_scope_path_message::declared_scope_path_message; use sqlbuild_scopes::scope_index::models::{ - DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, ScopePathFailure, + DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, }; use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; -use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::construct::new_record; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_items; +use crate::bindings::_helpers::records::construct::{error_class, new_record}; +use crate::bindings::_helpers::records::fields::{field as native_field, mapping_items}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; /// The folder a scope path is relative to when its owner names none. @@ -34,15 +32,6 @@ const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); -/// Consumer, declaration, usage kind, what it is used through, and the enum member. -#[derive(Clone, Hash, PartialEq, Eq)] -pub(crate) struct UsageRow { - pub(crate) consumer: QualifiedRow, - pub(crate) declaration: QualifiedRow, - pub(crate) kind: String, - pub(crate) through: Option, - pub(crate) enum_member: Option, -} /// Path, scope, private owner, owning path and ownership root path of a named record. struct RecordRow { path: String, @@ -249,7 +238,10 @@ fn scope_schema_usages( Err(rejected) => ( Vec::new(), Vec::new(), - Some(scope_path_message(&rejected.declared, &rejected.path)), + Some(declared_scope_path_message( + &rejected.declared, + &rejected.path, + )), ), }, ) @@ -269,56 +261,22 @@ fn source_rows(sources: Option>) -> Option> /// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. pub(crate) fn merged_usage_rows( declarations: &[(QualifiedRow, Option>)], - prior: &[UsageRow], - collected: &[UsageRow], + prior: &[UsageKey], + collected: &[UsageKey], ) -> MergeRows { - let distinct: Vec = distinct_positions(collected); - let mut consumed: Vec<(Option, QualifiedRow)> = - Vec::with_capacity(distinct.len()); - for position in &distinct { - let UsageRow { - consumer, - declaration, - .. - } = &collected[*position]; - let by_declaration: Option = DeclarationKind::parse(&consumer.0) - .is_some() - .then(|| consumer.clone()); - consumed.push((by_declaration, declaration.clone())); - } - let merged: Vec>> = merged_dependencies(declarations, &consumed) - .into_iter() - .map(source_rows) - .collect(); - let mut combined: Vec<&UsageRow> = prior.iter().collect(); - combined.extend(distinct.iter().map(|position| &collected[*position])); - let combined_positions: Vec = distinct_positions(&combined); - (distinct, merged, combined_positions) + let MergedUsages { + distinct, + merged, + combined, + } = merged_usages(declarations, prior, collected); + ( + distinct, + merged.into_iter().map(source_rows).collect(), + combined, + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; Ok(()) } - -/// The `InvalidScopePathError` message of the first path Python's visibility check rejects. -pub(crate) fn scope_path_message(declared: &DeclaredScope, label: &str) -> String { - let owner: &str = declared - .owning_path - .as_deref() - .filter(|path| !path.is_empty()) - .unwrap_or(CURRENT_FOLDER); - let (path, failure) = match normalize_scope_path(label) { - Err(failure) => (label, failure), - Ok(_) => match normalize_scope_path(owner) { - Err(failure) => (owner, failure), - Ok(_) => (label, ScopePathFailure::NotProjectRelative), - }, - }; - match failure { - ScopePathFailure::NotProjectRelative => { - format!("Scope path must be project-relative: {path}") - } - ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs index 6e8ce688a7..2cdb1cab5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs @@ -7,6 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, intern}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::parse_identity::parse_identity; +use sqlbuild_scopes::scope_index::main::posix_parent_text::posix_parent_text; +use sqlbuild_scopes::scope_index::main::posix_path_text::posix_path_text; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, IdentityFailure, QualifiedIdentity, ResourceIdentity, ScopePathFailure, }; @@ -82,7 +84,10 @@ impl NativeDeclarationContexts { native_field(record, "ownership_root")?.getattr(intern!(py, "path"))?; let key: Bound<'py, PyTuple> = PyTuple::new( py, - [PyString::new(py, &posix_parent(&path)).into_any(), root], + [ + PyString::new(py, &posix_parent_text(&path)).into_any(), + root, + ], )?; if let Some(cached) = cache.get_item(&key)? { let consumer: Bound<'py, PyAny> = native_field(record, "identity")?; @@ -187,7 +192,7 @@ impl NativeDeclarationContexts { let owner: String = owning_path .filter(|path| !path.is_empty()) .unwrap_or_else(|| CURRENT_FOLDER.to_owned()); - return Ok(posix_path(&format!( + return Ok(posix_path_text(&format!( "{owner}{POSIX_SEPARATOR}{DECLARATION_STEM}" ))); } @@ -341,28 +346,6 @@ fn target_path(resolver: &Bound<'_, PyAny>, file_path: &str) -> PyResult }) } -/// `Path(path).parent.as_posix()` of a normalized relative path. -fn posix_parent(path: &str) -> String { - match path.rfind(POSIX_SEPARATOR) { - Some(index) => posix_path(&path[..index]), - None => CURRENT_FOLDER.to_owned(), - } -} - -/// `Path(path).as_posix()`: empty and `.` components dropped. -fn posix_path(path: &str) -> String { - let absolute: bool = path.starts_with(POSIX_SEPARATOR); - let parts: Vec<&str> = path - .split(POSIX_SEPARATOR) - .filter(|part| !part.is_empty() && *part != CURRENT_FOLDER) - .collect(); - match (absolute, parts.is_empty()) { - (true, _) => format!("{POSIX_SEPARATOR}{}", parts.join("/")), - (false, true) => CURRENT_FOLDER.to_owned(), - (false, false) => parts.join("/"), - } -} - fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { match class.call1((message,)) { Ok(instance) => PyErr::from_value(instance), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 84db425ada..eed4695a73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -12,6 +12,7 @@ use sqlbuild_scopes::relationship_names::models::RelationshipScan; use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationInput, DeclarationKind, GrantRecord, GrantThrough, @@ -22,13 +23,14 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::mapping_values; -use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; +use crate::bindings::_helpers::records::fields::{ + field as native_field, mapping_values, path_stem, +}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::relationship_scan::{ RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, }; -use crate::bindings::_helpers::scopes::scope_lookup::{path_message, scope_path_error}; +use crate::bindings::_helpers::scopes::scope_lookup::scope_path_error; use crate::bindings::_helpers::scopes::scope_objects::{ ScopeClasses, field, optional_item, root_object, value_of, }; @@ -662,7 +664,7 @@ impl<'py> Walk<'py> { fn normalized(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { let path: String = text(path)?; normalize_scope_path(&path).map_err(|failure| { - scope_path_error(py, path_message(&RejectedPath { failure, path })) + scope_path_error(py, rejected_path_message(&RejectedPath { failure, path })) .unwrap_or_else(|error| error) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 3ee4cd952a..46ac2d125a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -5,16 +5,18 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{pyclass, pyfunction, wrap_pyfunction}; -use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; use sqlbuild_scopes::scope_index::main::report_target::report_target; use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; use sqlbuild_scopes::scope_index::models::{ - GrantKind, IdentityFailure, KeyedGroups, RejectedPath, ReportLookup, ReportRecords, - ReportTarget, ResolvedVisibility, ResolvedVisible, ResourceRecord, ScopePathFailure, - TargetFailure, TargetMatch, VisibilityGroups, + GrantKind, KeyedGroups, RejectedPath, ReportLookup, ReportRecords, ReportTarget, + ResolvedVisibility, ResolvedVisible, ResourceRecord, TargetFailure, TargetMatch, + VisibilityGroups, }; +use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::error_class; @@ -22,7 +24,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, qualified_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, read_index, report_records, resource_identity_object, tuple_attribute, + ScopeClasses, read_index, resource_identity_object, tuple_attribute, }; use crate::bindings::types::CompilerDetach; @@ -76,34 +78,6 @@ pub(crate) fn grant_kind(value: &str) -> PyResult { } } -/// Python's `InvalidScopePathError` message for a rejected path. -pub(crate) fn path_message(rejected: &RejectedPath) -> String { - match rejected.failure { - ScopePathFailure::NotProjectRelative => { - format!("Scope path must be project-relative: {}", rejected.path) - } - ScopePathFailure::EscapesProject => { - format!("Scope path escapes the project: {}", rejected.path) - } - } -} - -/// Python's `InvalidQualifiedIdentityError` message for a rejected identity. -pub(crate) fn identity_message(failure: &IdentityFailure, value: &str) -> String { - let shown = |text: &str| -> String { python_str_repr(text) }; - match failure { - IdentityFailure::NoName => format!("Qualified identity has no name: {}", shown(value)), - IdentityFailure::UnknownKind(kind) => { - format!("Unknown qualified identity kind: {}", shown(kind)) - } - IdentityFailure::InvalidPrivate => format!("Invalid private identity: {}", shown(value)), - IdentityFailure::Unqualified => format!( - "Expected a kind-qualified identity such as 'model:name': {}", - shown(value) - ), - } -} - /// Input positions as canonical positions. fn canonical(order: &[usize], positions: Vec) -> Vec { positions @@ -139,14 +113,14 @@ impl NativeScopeLookup { Err(TargetFailure::Identity(failure)) => TargetRows { declarations: false, positions: Vec::new(), - identity_error: Some(identity_message(&failure, text)), + identity_error: Some(identity_failure_message(&failure, text)), path_error: None, }, Err(TargetFailure::Path(rejected)) => TargetRows { declarations: false, positions: Vec::new(), identity_error: None, - path_error: Some(path_message(&rejected)), + path_error: Some(rejected_path_message(&rejected)), }, }) } @@ -163,7 +137,11 @@ impl NativeScopeLookup { .map_err(compiler_error)?; Ok(match resolved { Ok(resolved) => resolved_rows(resolved, &self.canonical.grants), - Err(rejected) => (Vec::new(), Vec::new(), Some(path_message(&rejected))), + Err(rejected) => ( + Vec::new(), + Vec::new(), + Some(rejected_path_message(&rejected)), + ), }) } } @@ -279,13 +257,13 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { - let records: ReportRecords = report_records(&read_index(index)?); + let records: ReportRecords = cached_report_records(&read_index(index)?); let built = py .compiler_detach(|| Ok(report_lookup(&records))) .map_err(compiler_error)?; let lookup: ReportLookup = match built { Ok(lookup) => lookup, - Err(rejected) => return Err(scope_path_error(py, path_message(&rejected))?), + Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), }; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; @@ -365,7 +343,7 @@ fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult Ok(normalized), Err(failure) => Err(scope_path_error( py, - path_message(&RejectedPath { failure, path }), + rejected_path_message(&RejectedPath { failure, path }), )?), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index 8f9205a9c8..eddbcb156f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -7,14 +7,9 @@ use sqlbuild_scopes::scope_cache::models::{ CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, CachedVisibility, ConstantFacts, EnumFacts, MacroFacts, }; -use sqlbuild_scopes::scope_index::main::format_identity::format_identity; use sqlbuild_scopes::scope_index::models::{ - DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportDiagnosticKey, ReportGrant, - ReportRecords, ReportUsage, ResourceIdentity, ResourceKind, ResourceRecord, -}; -use sqlbuild_scopes::scope_report::models::{ - Completeness, ConstantValues, DeclarationValues, DiagnosticFacts, EnumValues, MacroValues, - ProjectionGrant, ProjectionIndex, ProjectionStanding, ProjectionUsage, ReportValues, + DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportGrant, ResourceIdentity, + ResourceKind, ResourceRecord, }; use crate::bindings::_helpers::records::construct::{new_record, record_class}; @@ -307,155 +302,6 @@ pub(crate) fn read_placement_index(index: &Bound<'_, PyAny>) -> PyResult ReportRecords { - ReportRecords { - roots: index.roots.clone(), - resources: index.resources.clone(), - declarations: index - .declarations - .iter() - .map(|item| item.record.clone()) - .collect(), - usages: index - .usages - .iter() - .map(|item| ReportUsage { - consumer: item.consumer.clone(), - declaration: item.declaration.clone(), - kind: item.kind.clone(), - through: item.through.clone(), - }) - .collect(), - grants: index.grants.clone(), - diagnostics: index - .diagnostics - .iter() - .map(|item| ReportDiagnosticKey { - path: item.path.clone(), - line: item.line, - column: item.column, - code: item.code.clone(), - }) - .collect(), - } -} - -/// One declaration's report values. -pub(crate) fn declaration_values(item: &CachedDeclaration) -> DeclarationValues { - DeclarationValues { - macro_values: item.macro_facts.as_ref().map(|facts| MacroValues { - parameters: facts.parameters.clone(), - dependencies: facts.dependencies.clone(), - source_digest: facts.source_digest.clone(), - }), - enum_values: item.enum_facts.as_ref().map(|facts| EnumValues { - members: facts.members.clone(), - scalar_type: facts.scalar_type.clone(), - }), - constant_values: item.constant_facts.as_ref().map(|facts| ConstantValues { - logical_type: facts.logical_type.clone(), - collection_kind: facts.collection_kind.clone(), - item_count: facts.item_count, - nullable: facts.nullable, - render_as: facts.render_as.clone(), - }), - } -} - -/// The six completeness sections as the report engine reads them. -pub(crate) fn completeness(done: [bool; 6]) -> Completeness { - Completeness { - discovery: done[0], - static_visibility: done[1], - runtime_usage: done[2], - relationships: done[3], - placement: done[4], - promotion_impact: done[5], - } -} - -/// A diagnostic's facts, with its identities as qualified text. -pub(crate) fn diagnostic_facts(item: CachedDiagnostic) -> DiagnosticFacts { - DiagnosticFacts { - code: item.code, - message: item.message, - severity: item.severity, - path: item.path, - line: item.line, - column: item.column, - declaration: item - .declaration - .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity))), - resource: item - .resource - .map(|identity| format_identity(&QualifiedIdentity::Resource(identity))), - } -} - -/// The metadata projection's view of a full index. -pub(crate) fn projection_index(index: CachedIndex) -> ProjectionIndex { - ProjectionIndex { - roots: index.roots, - resources: index.resources, - values: ReportValues { - declarations: index.declarations.iter().map(declaration_values).collect(), - completeness: completeness(index.completeness), - }, - declarations: index - .declarations - .into_iter() - .map(|item| item.record) - .collect(), - usages: index - .usages - .into_iter() - .map(|item| ProjectionUsage { - consumer: item.consumer, - declaration: item.declaration, - kind: item.kind, - through: item.through, - enum_member: item.enum_member, - }) - .collect(), - grants: index - .grants - .into_iter() - .map(|item| ProjectionGrant { - resource: item.resource, - declaration: item.declaration, - through: item.through, - kind: item.kind, - }) - .collect(), - visibility: index - .visibility - .into_iter() - .map(|item| ProjectionStanding { - resource: item.resource, - declaration: item.declaration, - reason: item.reason, - through: item.through, - }) - .collect(), - inaccessible: index - .inaccessible - .into_iter() - .map(|item| ProjectionStanding { - resource: item.resource, - declaration: item.declaration, - reason: item.reason, - through: None, - }) - .collect(), - diagnostics: index - .diagnostics - .into_iter() - .map(diagnostic_facts) - .collect(), - } -} - /// A `ResourceIdentity`. pub(crate) fn resource_identity_object<'py>( classes: &ScopeClasses<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 6f50a0d376..c6b64dd4d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -8,6 +8,7 @@ use pyo3::types::{ PyTypeMethods, }; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_scopes::scope_report::main::cached_diagnostic_facts::cached_diagnostic_facts; use sqlbuild_scopes::scope_report::models::{ BrowseResult, DeclarationReport, DeclarationTarget, Explanation, ListResult, MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, ScopeChainEntry, @@ -17,8 +18,7 @@ use sqlbuild_scopes::scope_report::models::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, diagnostic_facts, field, optional_item, read_declaration_identity, - read_diagnostic, read_qualified, + ScopeClasses, field, optional_item, read_declaration_identity, read_diagnostic, read_qualified, }; type PyObject = Py; @@ -553,9 +553,9 @@ fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> native_field(item, "diagnostics")? .try_iter()? .map(|diagnostic| { - Ok(ReportDiagnostic::Raised(diagnostic_facts(read_diagnostic( - &diagnostic?, - )?))) + Ok(ReportDiagnostic::Raised(cached_diagnostic_facts( + read_diagnostic(&diagnostic?)?, + ))) }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index a532e253a1..82496de146 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -7,8 +7,13 @@ use pyo3::prelude::{ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; +use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; +use sqlbuild_scopes::scope_report::main::cached_completeness::cached_completeness; +use sqlbuild_scopes::scope_report::main::cached_declaration_values::cached_declaration_values; +use sqlbuild_scopes::scope_report::main::cached_projection_index::cached_projection_index; use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; @@ -24,12 +29,9 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; -use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, identity_message, path_message, -}; +use crate::bindings::_helpers::scopes::scope_lookup::NativeScopeLookup; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, completeness, declaration_values, projection_index, read_completeness, - read_declaration, read_index, + ScopeClasses, read_completeness, read_declaration, read_index, }; use crate::bindings::_helpers::scopes::scope_report_objects::{ browse_object, diagnostics_object, explanation_object, json_to_plain, json_to_python, @@ -85,7 +87,7 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { let native: Py = native_field(lookup, "native")?.extract()?; let rows: Vec = native_field(&index, "declarations")? .try_iter()? - .map(|record| Ok(declaration_values(&read_declaration(&record?)?))) + .map(|record| Ok(cached_declaration_values(&read_declaration(&record?)?))) .collect::>()?; let canonical: &[usize] = &native.get().lookup.declarations; if rows.len() != canonical.len() { @@ -101,7 +103,7 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { lookup: native, values: ReportValues { declarations, - completeness: completeness(read_completeness(&index)?), + completeness: cached_completeness(read_completeness(&index)?), }, }) } @@ -115,13 +117,15 @@ fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult, failure: &ReportFailure) -> PyResult { match failure { - ReportFailure::Path(rejected) => { - scope_error(py, ErrorId::InvalidScopePath, path_message(rejected)) - } + ReportFailure::Path(rejected) => scope_error( + py, + ErrorId::InvalidScopePath, + rejected_path_message(rejected), + ), ReportFailure::Identity(failure, text) => scope_error( py, ErrorId::InvalidQualifiedIdentity, - identity_message(failure, text), + identity_failure_message(failure, text), ), ReportFailure::Serialization(message) => Ok(PyValueError::new_err(message.clone())), ReportFailure::IndexOutOfRange => Ok(PyIndexError::new_err("tuple index out of range")), @@ -342,7 +346,7 @@ pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - let index: ProjectionIndex = projection_index(read_index(index)?); + let index: ProjectionIndex = cached_projection_index(read_index(index)?); Ok(scope_metadata_projection(&index)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index bd5caf53ff..973f36e8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::compile_usages::models::UsageKey; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; use sqlbuild_scopes::placement::main::relocate_for_resource_move::relocate_for_resource_move; @@ -14,13 +15,11 @@ use sqlbuild_scopes::placement::models::{ use sqlbuild_scopes::scope_index::models::ResourceIdentity; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::records::construct::enum_member; -use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::replaced_keywords; +use crate::bindings::_helpers::records::construct::{enum_member, error_class}; +use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::compile_usages::{ - MergeRows, SourceRow, UsageRow, merged_usage_rows, model_schema_scope_usages, + MergeRows, SourceRow, merged_usage_rows, model_schema_scope_usages, }; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ @@ -56,8 +55,8 @@ fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult) -> PyResult { - Ok(UsageRow { +fn usage_row(usage: &Bound<'_, PyAny>) -> PyResult { + Ok(UsageKey { consumer: qualified_row(&native_field(usage, "consumer")?)?, declaration: qualified_row(&native_field(usage, "declaration")?)?, kind: value_of(usage, "kind")?, @@ -125,11 +124,11 @@ fn merged_usage_index<'py>( .iter() .map(|record| dependency_row(&record)) .collect::>()?; - let prior_rows: Vec = prior + let prior_rows: Vec = prior .iter() .map(|usage| usage_row(&usage)) .collect::>()?; - let collected_rows: Vec = collected + let collected_rows: Vec = collected .iter() .map(|usage| usage_row(&usage)) .collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 46b48fc41a..c4265dcd29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, }; @@ -14,7 +15,7 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, ResolvedRows, TargetRows, path_message, scope_path_error, + NativeScopeLookup, ResolvedRows, TargetRows, scope_path_error, }; use crate::bindings::_helpers::scopes::scope_objects::{ ScopeClasses, qualified_row, read_resource, tuple_attribute, @@ -217,7 +218,7 @@ fn scope_resolve_path_visibility( failure, path: text, }; - return Err(scope_path_error(py, path_message(&rejected))?); + return Err(scope_path_error(py, rejected_path_message(&rejected))?); } }; let resource: ResourceRecord = ResourceRecord { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index 3807ce7aab..e7976ddc8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; -use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::records::construct::{enum_from_value, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -238,8 +238,8 @@ fn ordering(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py = model.py(); - let values = native_field(&native_field(model, "config")?, "values")?; - let declared = config_str(&values, "cursor_type")?.filter(|value| { + let values = model_values(model)?; + let declared = config_text_object(py, &values, "cursor_type")?.filter(|value| { value .extract::() .is_ok_and(|value| !value.is_empty()) @@ -248,7 +248,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { declared.unwrap_or_else(|| PyString::new(py, TIMESTAMP_CURSOR_TYPE).into_any()), ))?; let type_name: String = native_field(&cursor_type, "value")?.extract()?; - let grain = match config_str(&values, "cursor_grain")? { + let grain = match config_text_object(py, &values, "cursor_grain")? { Some(grain) if type_name == TIMESTAMP_CURSOR_TYPE => { let grain = enum_from_value(py, RecordId::CursorGrain, grain)?; let session = CursorSession::new(py)?; diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs similarity index 86% rename from crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs rename to crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs index b1a563fe9d..8b04822d29 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs @@ -5,7 +5,7 @@ use std::hash::Hash; /// The positions of the first occurrence of each distinct key. #[must_use] -pub fn distinct_positions(keys: &[K]) -> Vec { +pub(crate) fn distinct_positions(keys: &[K]) -> Vec { let mut seen: HashSet<&K> = HashSet::with_capacity(keys.len()); let mut positions: Vec = Vec::new(); for (position, key) in keys.iter().enumerate() { diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs similarity index 96% rename from crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs rename to crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs index 7aa9399940..c60d744552 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs @@ -7,7 +7,7 @@ use crate::compile_usages::models::DependencySource; /// Per declaration, `None` without macro metadata, else own then consumed dependencies, deduped. #[must_use] -pub fn merged_dependencies( +pub(crate) fn merged_dependencies( declarations: &[(K, Option>)], usages: &[(Option, K)], ) -> Vec>> { diff --git a/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs index 04abf7838f..7880f80eea 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod distinct_positions; +pub(crate) mod merged_dependencies; pub(crate) mod schema_checks; diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs new file mode 100644 index 0000000000..0881e02946 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs @@ -0,0 +1,39 @@ +//! New runtime usages merged into an index's declarations and usages. + +use crate::compile_usages::_helpers::distinct_positions::distinct_positions; +use crate::compile_usages::_helpers::merged_dependencies::merged_dependencies; +use crate::compile_usages::models::{MergedUsages, UsageKey}; +use crate::compile_usages::types::QualifiedKey; +use crate::scope_index::models::DeclarationKind; + +/// Distinct `collected` usages, each declaration's merged dependencies, and the distinct prior-then-new usages. +#[must_use] +pub fn merged_usages( + declarations: &[(QualifiedKey, Option>)], + prior: &[UsageKey], + collected: &[UsageKey], +) -> MergedUsages { + let distinct: Vec = distinct_positions(collected); + let mut consumed: Vec<(Option, QualifiedKey)> = + Vec::with_capacity(distinct.len()); + for position in &distinct { + let UsageKey { + consumer, + declaration, + .. + } = &collected[*position]; + let by_declaration: Option = DeclarationKind::parse(&consumer.0) + .is_some() + .then(|| consumer.clone()); + consumed.push((by_declaration, declaration.clone())); + } + let merged = merged_dependencies(declarations, &consumed); + let mut combined: Vec<&UsageKey> = prior.iter().collect(); + combined.extend(distinct.iter().map(|position| &collected[*position])); + let combined: Vec = distinct_positions(&combined); + MergedUsages { + distinct, + merged, + combined, + } +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs index b8f3347ba3..c91cdee02f 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs @@ -1,3 +1,2 @@ -pub mod distinct_positions; -pub mod merged_dependencies; +pub mod merged_usages; pub mod schema_scope_usages; diff --git a/crates/sqlbuild-scopes/src/compile_usages/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/mod.rs index 1e64f3d116..1644773e62 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-scopes/src/compile_usages/models.rs b/crates/sqlbuild-scopes/src/compile_usages/models.rs index 1ed254abf0..ba794bec24 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/models.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/models.rs @@ -1,5 +1,6 @@ //! The schema declarations and indexed records compile usage facts are read from. +use crate::compile_usages::types::QualifiedKey; use crate::scope_index::models::DeclaredScope; /// The first indexed record of a named schema or enum, by its caller handle. @@ -51,3 +52,24 @@ pub enum DependencySource { /// The declaration of the collected usage at this position. Usage(usize), } + +/// One usage: consumer, declaration, usage kind, what it is used through, and the enum member. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct UsageKey { + pub consumer: QualifiedKey, + pub declaration: QualifiedKey, + pub kind: String, + pub through: Option, + pub enum_member: Option, +} + +/// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MergedUsages { + /// Positions of the distinct collected usages. + pub distinct: Vec, + /// Per declaration, its merged dependency sources; `None` without macro metadata. + pub merged: Vec>>, + /// Positions of the distinct usages among the prior usages followed by the new ones. + pub combined: Vec, +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs index 2ad12b9697..9401aefd87 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs @@ -1,5 +1,5 @@ -use crate::compile_usages::main::distinct_positions::distinct_positions; -use crate::compile_usages::main::merged_dependencies::merged_dependencies; +use crate::compile_usages::_helpers::distinct_positions::distinct_positions; +use crate::compile_usages::_helpers::merged_dependencies::merged_dependencies; use crate::compile_usages::main::schema_scope_usages::schema_scope_usages; use crate::compile_usages::models::{DependencySource, SchemaUsages}; use crate::compile_usages::tests::helpers::{ diff --git a/crates/sqlbuild-scopes/src/compile_usages/types.rs b/crates/sqlbuild-scopes/src/compile_usages/types.rs new file mode 100644 index 0000000000..b8fa417cd2 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/types.rs @@ -0,0 +1,4 @@ +//! The identity keys compile usages are merged by. + +/// A declaration's kind, name and private owner `(kind, name)`; a resource's kind and name with no owner. +pub type QualifiedKey = (String, String, Option<(String, String)>); diff --git a/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs new file mode 100644 index 0000000000..e5d0462974 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs @@ -0,0 +1,28 @@ +//! The message of a declaration whose scope path or owning path is rejected. + +use crate::scope_index::constants::CURRENT_PATH_COMPONENT; +use crate::scope_index::main::normalize_scope_path::normalize_scope_path; +use crate::scope_index::models::{DeclaredScope, ScopePathFailure}; + +/// The `InvalidScopePathError` message of the first path Python's visibility check rejects. +#[must_use] +pub fn declared_scope_path_message(declared: &DeclaredScope, label: &str) -> String { + let owner: &str = declared + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_PATH_COMPONENT); + let (path, failure) = match normalize_scope_path(label) { + Err(failure) => (label, failure), + Ok(_) => match normalize_scope_path(owner) { + Err(failure) => (owner, failure), + Ok(_) => (label, ScopePathFailure::NotProjectRelative), + }, + }; + match failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {path}") + } + ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs new file mode 100644 index 0000000000..b1dba9bc13 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs @@ -0,0 +1,22 @@ +//! The message of a rejected qualified identity. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::scope_index::models::IdentityFailure; + +/// Python's `InvalidQualifiedIdentityError` message for a rejected identity. +#[must_use] +pub fn identity_failure_message(failure: &IdentityFailure, value: &str) -> String { + let shown = |text: &str| -> String { python_str_repr(text) }; + match failure { + IdentityFailure::NoName => format!("Qualified identity has no name: {}", shown(value)), + IdentityFailure::UnknownKind(kind) => { + format!("Unknown qualified identity kind: {}", shown(kind)) + } + IdentityFailure::InvalidPrivate => format!("Invalid private identity: {}", shown(value)), + IdentityFailure::Unqualified => format!( + "Expected a kind-qualified identity such as 'model:name': {}", + shown(value) + ), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs index 083babc7d0..8ce06eaa13 100644 --- a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -1,13 +1,18 @@ pub mod build_scope_index; pub mod classify_report_resource; pub mod classify_resource; +pub mod declared_scope_path_message; pub mod explain_report_declaration; pub mod format_identity; +pub mod identity_failure_message; pub mod identity_repr; pub mod macro_source_digest; pub mod normalize_scope_path; pub mod parse_identity; pub mod path_visible; +pub mod posix_parent_text; +pub mod posix_path_text; +pub mod rejected_path_message; pub mod relationship_grants; pub mod report_lookup; pub mod report_target; diff --git a/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs b/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs new file mode 100644 index 0000000000..4f6e6c99da --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs @@ -0,0 +1,13 @@ +//! The parent of a `/`-separated path as Python's `Path.as_posix` prints it. + +use crate::scope_index::constants::{CURRENT_PATH_COMPONENT, PATH_SEPARATOR}; +use crate::scope_index::main::posix_path_text::posix_path_text; + +/// `Path(path).parent.as_posix()` of a normalized relative path. +#[must_use] +pub fn posix_parent_text(path: &str) -> String { + match path.rfind(PATH_SEPARATOR) { + Some(index) => posix_path_text(&path[..index]), + None => CURRENT_PATH_COMPONENT.to_owned(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs b/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs new file mode 100644 index 0000000000..1c50a740f4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs @@ -0,0 +1,18 @@ +//! A `/`-separated path as Python's `Path.as_posix` prints it. + +use crate::scope_index::constants::{CURRENT_PATH_COMPONENT, PATH_SEPARATOR}; + +/// `Path(path).as_posix()`: empty and `.` components dropped. +#[must_use] +pub fn posix_path_text(path: &str) -> String { + let absolute: bool = path.starts_with(PATH_SEPARATOR); + let parts: Vec<&str> = path + .split(PATH_SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT_PATH_COMPONENT) + .collect(); + match (absolute, parts.is_empty()) { + (true, _) => format!("{PATH_SEPARATOR}{}", parts.join("/")), + (false, true) => CURRENT_PATH_COMPONENT.to_owned(), + (false, false) => parts.join("/"), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs new file mode 100644 index 0000000000..e8626234ec --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs @@ -0,0 +1,16 @@ +//! The message of a rejected scope path. + +use crate::scope_index::models::{RejectedPath, ScopePathFailure}; + +/// Python's `InvalidScopePathError` message for a rejected path. +#[must_use] +pub fn rejected_path_message(rejected: &RejectedPath) -> String { + match rejected.failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {}", rejected.path) + } + ScopePathFailure::EscapesProject => { + format!("Scope path escapes the project: {}", rejected.path) + } + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs index 4bfad639e1..77f150b393 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -10,6 +10,8 @@ mod macro_source_digest; mod parse_identity; #[path = "tests/test_path_visible.rs"] mod path_visible; +#[path = "tests/test_posix_path_text.rs"] +mod posix_path_text; #[path = "tests/test_relationship_grants.rs"] mod relationship_grants; #[path = "tests/test_report_lookup.rs"] diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs new file mode 100644 index 0000000000..dad0d3258f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs @@ -0,0 +1,41 @@ +use crate::scope_index::main::posix_parent_text::posix_parent_text; +use crate::scope_index::main::posix_path_text::posix_path_text; +use crate::scope_index::tests::test_types::PosixPathTestCase; + +#[test] +fn given_slash_separated_paths_when_printed_then_they_match_python_as_posix() { + let test_cases = [ + PosixPathTestCase { + description: "empty and current components are dropped", + path: "models//./orders.sql", + expected_text: "models/orders.sql", + expected_parent: "models", + }, + PosixPathTestCase { + description: "an empty path prints as the current folder", + path: "", + expected_text: ".", + expected_parent: ".", + }, + PosixPathTestCase { + description: "an absolute path keeps its root", + path: "/models/orders.sql", + expected_text: "/models/orders.sql", + expected_parent: "/models", + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + posix_path_text(test_case.path), + posix_parent_text(test_case.path) + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_parent.to_owned() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs index 25c945a4cc..f98a132691 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -115,3 +115,11 @@ pub(super) struct MacroDigestTestCase { pub(super) source: &'static str, pub(super) expected_digest: &'static str, } + +/// A `/`-separated path and how Python prints it and its parent. +pub(super) struct PosixPathTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) expected_text: &'static str, + pub(super) expected_parent: &'static str, +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs new file mode 100644 index 0000000000..675c7dd26c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs @@ -0,0 +1,16 @@ +//! The completeness sections of a cached index. + +use crate::scope_report::models::Completeness; + +/// The six completeness sections as the report engine reads them. +#[must_use] +pub fn cached_completeness(done: [bool; 6]) -> Completeness { + Completeness { + discovery: done[0], + static_visibility: done[1], + runtime_usage: done[2], + relationships: done[3], + placement: done[4], + promotion_impact: done[5], + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs new file mode 100644 index 0000000000..0145b81061 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs @@ -0,0 +1,27 @@ +//! One cached declaration's report values. + +use crate::scope_cache::models::CachedDeclaration; +use crate::scope_report::models::{ConstantValues, DeclarationValues, EnumValues, MacroValues}; + +/// One declaration's report values. +#[must_use] +pub fn cached_declaration_values(item: &CachedDeclaration) -> DeclarationValues { + DeclarationValues { + macro_values: item.macro_facts.as_ref().map(|facts| MacroValues { + parameters: facts.parameters.clone(), + dependencies: facts.dependencies.clone(), + source_digest: facts.source_digest.clone(), + }), + enum_values: item.enum_facts.as_ref().map(|facts| EnumValues { + members: facts.members.clone(), + scalar_type: facts.scalar_type.clone(), + }), + constant_values: item.constant_facts.as_ref().map(|facts| ConstantValues { + logical_type: facts.logical_type.clone(), + collection_kind: facts.collection_kind.clone(), + item_count: facts.item_count, + nullable: facts.nullable, + render_as: facts.render_as.clone(), + }), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs new file mode 100644 index 0000000000..823b475e41 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs @@ -0,0 +1,25 @@ +//! One cached diagnostic's report facts. + +use crate::scope_cache::models::CachedDiagnostic; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::QualifiedIdentity; +use crate::scope_report::models::DiagnosticFacts; + +/// A diagnostic's facts, with its identities as qualified text. +#[must_use] +pub fn cached_diagnostic_facts(item: CachedDiagnostic) -> DiagnosticFacts { + DiagnosticFacts { + code: item.code, + message: item.message, + severity: item.severity, + path: item.path, + line: item.line, + column: item.column, + declaration: item + .declaration + .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity))), + resource: item + .resource + .map(|identity| format_identity(&QualifiedIdentity::Resource(identity))), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs new file mode 100644 index 0000000000..929ae3d66e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs @@ -0,0 +1,77 @@ +//! The metadata projection's view of a cached full index. + +use crate::scope_cache::models::CachedIndex; +use crate::scope_report::main::cached_completeness::cached_completeness; +use crate::scope_report::main::cached_declaration_values::cached_declaration_values; +use crate::scope_report::main::cached_diagnostic_facts::cached_diagnostic_facts; +use crate::scope_report::models::{ + ProjectionGrant, ProjectionIndex, ProjectionStanding, ProjectionUsage, ReportValues, +}; + +/// The metadata projection's view of a full index. +#[must_use] +pub fn cached_projection_index(index: CachedIndex) -> ProjectionIndex { + ProjectionIndex { + roots: index.roots, + resources: index.resources, + values: ReportValues { + declarations: index + .declarations + .iter() + .map(cached_declaration_values) + .collect(), + completeness: cached_completeness(index.completeness), + }, + declarations: index + .declarations + .into_iter() + .map(|item| item.record) + .collect(), + usages: index + .usages + .into_iter() + .map(|item| ProjectionUsage { + consumer: item.consumer, + declaration: item.declaration, + kind: item.kind, + through: item.through, + enum_member: item.enum_member, + }) + .collect(), + grants: index + .grants + .into_iter() + .map(|item| ProjectionGrant { + resource: item.resource, + declaration: item.declaration, + through: item.through, + kind: item.kind, + }) + .collect(), + visibility: index + .visibility + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: item.through, + }) + .collect(), + inaccessible: index + .inaccessible + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: None, + }) + .collect(), + diagnostics: index + .diagnostics + .into_iter() + .map(cached_diagnostic_facts) + .collect(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs new file mode 100644 index 0000000000..c5885e1871 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs @@ -0,0 +1,39 @@ +//! The lookup records of a cached full index. + +use crate::scope_cache::models::CachedIndex; +use crate::scope_index::models::{ReportDiagnosticKey, ReportRecords, ReportUsage}; + +/// The lookup's records of a full index. +#[must_use] +pub fn cached_report_records(index: &CachedIndex) -> ReportRecords { + ReportRecords { + roots: index.roots.clone(), + resources: index.resources.clone(), + declarations: index + .declarations + .iter() + .map(|item| item.record.clone()) + .collect(), + usages: index + .usages + .iter() + .map(|item| ReportUsage { + consumer: item.consumer.clone(), + declaration: item.declaration.clone(), + kind: item.kind.clone(), + through: item.through.clone(), + }) + .collect(), + grants: index.grants.clone(), + diagnostics: index + .diagnostics + .iter() + .map(|item| ReportDiagnosticKey { + path: item.path.clone(), + line: item.line, + column: item.column, + code: item.code.clone(), + }) + .collect(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/mod.rs b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs index ed557b8a5e..3d0b07f168 100644 --- a/crates/sqlbuild-scopes/src/scope_report/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs @@ -1,5 +1,10 @@ pub mod browse_scope_folders; pub mod build_scope_report; +pub mod cached_completeness; +pub mod cached_declaration_values; +pub mod cached_diagnostic_facts; +pub mod cached_projection_index; +pub mod cached_report_records; pub mod explain_scope_declaration; pub mod list_scope_declarations; pub mod preview_scope_move; diff --git a/fensu.toml b/fensu.toml index 69c953cc3b..c401ef48db 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1942,14 +1942,15 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", @@ -2380,7 +2381,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 2cbd009334..9d10264b4c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1199,3 +1199,12 @@ class CustomHostOutcomeTestCase: rule_body: str expected_exit_code: int expected_messages: tuple[str, ...] + + +@dataclass(frozen=True) +class TypedConfigReadsTestCase: + """Config keys planning never reads, added to every model of a planned project.""" + + description: str + unread_keys: str + expected_same_reads: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index df194d7a58..74bc48c9e9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -985,3 +985,26 @@ def diagnostics_by_model(diagnostics: list[dict[str, object]]) -> dict[str, str] name: str = cast(str, item["resource_name"]) by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") return by_model + + +def typed_config_reads(*, project_dir: Path, extra_header: str) -> int: + """How many config values planning converted into typed nodes for three cursor models.""" + + project_dir.mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "main"\n', + encoding="utf-8", + ) + models_dir: Path = project_dir / "models" + models_dir.mkdir() + for index in range(3): + (models_dir / f"orders_{index}.sql").write_text( + f'MODEL (description "Orders.", materialized table, tags ["daily"]{extra_header});\n\n' + f"SELECT {index} AS order_id, 1 AS loaded_at\n", + encoding="utf-8", + ) + _ = native_module.compile_stage_work_info(reset=True) + exit_code: int = main(["--project-dir", str(project_dir), "plan", "--json", "--no-cache"]) + assert exit_code == 0 + return native_module.compile_stage_work_info(reset=True).get("typed_config_reads", 0) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py b/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py new file mode 100644 index 0000000000..b657e430c8 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py @@ -0,0 +1,38 @@ +"""Planning converts only the config values its readers ask for.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + TypedConfigReadsTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import typed_config_reads + + +@pytest.mark.parametrize( + "test_case", + [ + TypedConfigReadsTestCase( + description="row-diff settings no planner reader reads", + unread_keys=( + ', row_diff_exclude_columns ["loaded_at"], row_diff_sample_seed 7, ' + "row_diff_tolerances (default 1)" + ), + expected_same_reads=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_config_keys_planning_never_reads_when_planning_then_they_are_not_converted( + test_case: TypedConfigReadsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + plain: int = typed_config_reads(project_dir=tmp_path / "plain", extra_header="") + extended: int = typed_config_reads( + project_dir=tmp_path / "extended", extra_header=test_case.unread_keys + ) + _ = capsys.readouterr() + + assert (plain > 0, extended == plain) == (True, test_case.expected_same_reads) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index b510731f3e..b574754880 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -33,3 +33,60 @@ class ColumnMetadataExpansionTestCase: header_column: str expected_type: str | None expected_error: str | None + + +@dataclass(frozen=True) +class ConfigValueBuildTestCase: + """A Python-given MODEL header and the config values the build gives the model.""" + + description: str + header: dict[object, object] + expected_values: tuple[tuple[str, str, str], ...] + expected_header_keys: tuple[str, ...] + + +@dataclass(frozen=True) +class ConfigValueRejectionTestCase: + """A Python-given MODEL header the config build rejects, and the error it raises.""" + + description: str + header: dict[object, object] + expected_error: tuple[str, str] + + +@dataclass(frozen=True) +class OpaqueConfigValueTestCase: + """A MODEL header holding an opaque value under `key`, which reaches the config unchanged.""" + + description: str + header: dict[object, object] + key: object + expected_same_object: bool + + +@dataclass(frozen=True) +class HeaderColumnPathTestCase: + """MODEL headers with columns, and how many models reuse or reparse their native columns.""" + + description: str + headers: tuple[str, ...] + expected_native: int + expected_parsed: int + + +@dataclass(frozen=True) +class TemplateTextTestCase: + """A MODEL header and CLI vars whose templated alias reads values as Python's text.""" + + description: str + header: dict[object, object] + cli_vars: dict[str, object] + expected_outcome: tuple[str, str] + + +@dataclass(frozen=True) +class OpaqueReadsTestCase: + """Whether an opaque header value reaches the config, and which of its texts compiling reads.""" + + description: str + expected_reads: tuple[bool, tuple[str, ...]] diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 34f74aa3be..697c4439f6 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -6,8 +6,9 @@ import re from collections.abc import Callable, Iterator from dataclasses import dataclass, fields, replace +from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, TypeGuard, cast +from typing import TYPE_CHECKING, ClassVar, TypeGuard, cast import pytest @@ -29,7 +30,7 @@ if TYPE_CHECKING: from _typeshed import DataclassInstance - from sqlbuild._native import CompileInputsView + from sqlbuild._native import CompileInputsView, CompileModelConfig _TEXTS: tuple[object, ...] = ( "INTEGER", @@ -533,3 +534,201 @@ def column_metadata_outcome(*, project_dir: Path, header_column: str) -> tuple[o tuple[object, ...], _entry_metadata(inputs.model_inputs[0].schema_entry)[0] ) return getattr(columns[0], "type", None), None + + +CONFIG_VALUE_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = "orders.duckdb"\n\n' + '[vars]\nregion = "north"\nratio = "0.5"\n\n' + '[defaults]\nmaterialized = "table"\ncontract = "enforced"\ntags = ["core"]\n\n' + '[path_defaults."marts"]\nrow_diff_exclude_columns = ["loaded_at"]\ntags = ["mart"]\n\n' + '[targets.dev]\nconnection = "local"\nschema = "${region}_${CTX:model.name}"\n' + ), + "models/marts/orders.sql": 'MODEL (description "Orders.");\n\nSELECT 1 AS order_id\n', +} + + +class Severity(StrEnum): + """A string enum whose members config values and keys hold as themselves.""" + + WARN = "warn" + + +class TagList(list[object]): + """A list subclass a Python caller may pass as a config value.""" + + +class SettingMap(dict[str, object]): + """A dict subclass a Python caller may pass as a config value.""" + + +class Marker: + """An opaque object the build carries through without reading it.""" + + def __repr__(self) -> str: + return "Marker()" + + +MARKER: Marker = Marker() + + +def config_value_inputs( + *, project_dir: Path, header: dict[object, object], cli_vars: dict[str, object] | None = None +) -> CompileInputsView: + """Build compile inputs for the config value project with its one model's header replaced.""" + + for relative_path, contents in CONFIG_VALUE_PROJECT.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + model_file: DiscoveredSqlModelFile = discovered.model_files[0] + given: DiscoveredSqlModelFile = DiscoveredSqlModelFile( + file_path=model_file.file_path, + relative_path=model_file.relative_path, + contents=model_file.contents, + header_values=cast("dict[str, object]", header), + header_column_locations={}, + output_column_locations={}, + query_sql=model_file.query_sql, + ) + adapter: DuckDbAdapter = DuckDbAdapter() + return build_compile_inputs( + discovered_inputs=replace(discovered, model_files=(given,)), + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + cli_vars=cli_vars, + run_id="20261007T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + + +def config_value_outcome( + *, project_dir: Path, header: dict[object, object] +) -> tuple[tuple[tuple[str, str, str], ...], tuple[str, ...]]: + """The model's config values as `(key repr, value type, value repr)` and its header keys.""" + + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header).model_inputs[0].config + ) + return ( + tuple( + (repr(key), type(value).__name__, repr(value)) for key, value in config.values.items() + ), + config.model_header_keys, + ) + + +def marker_kept(*, project_dir: Path, header: dict[object, object], key: object) -> bool: + """Whether the opaque value under `key` reaches the model's config values as the same object.""" + + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header).model_inputs[0].config + ) + return config.values[cast("str", key)] is MARKER + + +def header_column_paths(*, project_dir: Path, headers: tuple[str, ...]) -> tuple[int, int]: + """How many models reuse their natively parsed header columns, and how many reparse them.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": CONFIG_VALUE_PROJECT["sqlbuild_project.toml"], + **{ + f"models/marts/orders_{index}.sql": f"MODEL ({header});\n\nSELECT 1 AS order_id\n" + for index, header in enumerate(headers) + }, + } + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + adapter: DuckDbAdapter = DuckDbAdapter() + _ = _native.compile_stage_work_info(reset=True) + _ = build_compile_inputs( + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="20261007T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + work: dict[str, int] = _native.compile_stage_work_info(reset=True) + return (work.get("native_header_columns", 0), work.get("parsed_header_columns", 0)) + + +class Exploding: + """An opaque value whose `str()` and `repr()` raise and count each call.""" + + calls: ClassVar[list[str]] = [] + + def __str__(self) -> str: + Exploding.calls.append("str") + raise ValueError("no str") + + def __repr__(self) -> str: + Exploding.calls.append("repr") + raise ValueError("no repr") + + +class ExplodingText(str): + """A string whose `str()` raises, which config unions print.""" + + __slots__ = () + + def __str__(self) -> str: + raise ValueError("no str") + + +def templated_alias( + *, project_dir: Path, header: dict[object, object], cli_vars: dict[str, object] +) -> tuple[str, str]: + """The model's built alias as `("alias", text)`, or the raised error as `(type, message)`.""" + + try: + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header, cli_vars=cli_vars) + .model_inputs[0] + .config + ) + except Exception as error: + return (type(error).__name__, str(error)) + return ("alias", str(config.values.get("alias"))) + + +def opaque_reads(*, project_dir: Path) -> tuple[bool, tuple[str, ...]]: + """Whether an opaque header value reaches the config, and which of its texts compiling read.""" + + Exploding.calls.clear() + marker: Exploding = Exploding() + config: CompileModelConfig = ( + config_value_inputs( + project_dir=project_dir, + header={"description": "x", "alias": "${region}", Severity.WARN: marker}, + ) + .model_inputs[0] + .config + ) + return (config.values[cast("str", Severity.WARN)] is marker, tuple(Exploding.calls)) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py b/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py new file mode 100644 index 0000000000..cb1d0a8772 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py @@ -0,0 +1,79 @@ +"""Config text read as Python reads it: containers by `str()`, errors raised as Python raises them.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + OpaqueReadsTestCase, + TemplateTextTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( + MARKER, + Exploding, + ExplodingText, + opaque_reads, + templated_alias, +) + +_SAME_ALIAS: str = "${if(eq(given, expected), 'same', 'different')}" + + +@pytest.mark.parametrize( + "test_case", + [ + TemplateTextTestCase( + description="a list holding a format character compares as Python prints it", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": ["\u200b\u00ad"], "expected": "['\\u200b\\xad']"}, + expected_outcome=("alias", "same"), + ), + TemplateTextTestCase( + description="a list holding an opaque object compares by the object's repr", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": [MARKER], "expected": "[Marker()]"}, + expected_outcome=("alias", "same"), + ), + TemplateTextTestCase( + description="a list whose item cannot be printed raises the item's error", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": [Exploding()], "expected": "x"}, + expected_outcome=("ValueError", "no repr"), + ), + TemplateTextTestCase( + description="a tag union over an item that cannot be printed raises its error", + header={"tags": ["daily", ExplodingText("weekly")]}, + cli_vars={}, + expected_outcome=("ValueError", "no str"), + ), + ], + ids=lambda case: case.description, +) +def test_given_non_scalar_values_when_building_config_then_text_and_errors_match_python( + test_case: TemplateTextTestCase, tmp_path: Path +) -> None: + outcome: tuple[str, str] = templated_alias( + project_dir=tmp_path / "orders", header=test_case.header, cli_vars=test_case.cli_vars + ) + + assert outcome == test_case.expected_outcome + + +@pytest.mark.parametrize( + "test_case", + [ + OpaqueReadsTestCase( + description="an opaque value under a str-enum key beside a templated alias", + expected_reads=(True, ("str",)), + ), + ], + ids=lambda case: case.description, +) +def test_given_opaque_value_when_building_config_then_only_the_config_facts_read_its_text( + test_case: OpaqueReadsTestCase, tmp_path: Path +) -> None: + reads: tuple[bool, tuple[str, ...]] = opaque_reads(project_dir=tmp_path / "orders") + + assert reads == test_case.expected_reads diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py b/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py new file mode 100644 index 0000000000..5d98dc6ff1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py @@ -0,0 +1,213 @@ +"""Config values the native build gives models: builtin, subclassed and opaque Python values.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + ConfigValueBuildTestCase, + ConfigValueRejectionTestCase, + OpaqueConfigValueTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( + MARKER, + SettingMap, + Severity, + TagList, + config_value_inputs, + config_value_outcome, + marker_kept, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigValueBuildTestCase( + description="builtin values, unions and templates", + header={ + "description": "Orders.", + "tags": ["${region}-${ratio}", "core"], + "row_diff_exclude_columns": ["updated_at"], + "table_settings": {"owner": "${CTX:model.name}", "weight": 10**30, "enabled": True}, + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart', 'north-0.5']"), + ("'row_diff_exclude_columns'", "tuple", "('loaded_at', 'updated_at')"), + ( + "'table_settings'", + "dict", + "{'owner': 'orders', 'weight': 1000000000000000000000000000000, 'enabled': True}", + ), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=( + "description", + "row_diff_exclude_columns", + "table_settings", + "tags", + ), + ), + ConfigValueBuildTestCase( + description="subclasses, str enums, dates and opaque values", + header={ + "description": "Orders.", + "tags": TagList(["${region}", Severity.WARN]), + "table_settings": SettingMap(owner="${CTX:model.name}", level=Severity.WARN), + "row_diff_tolerances": {"by_type": {"DOUBLE": 0.5}, "default": date(2024, 1, 2)}, + Severity.WARN: MARKER, + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart', 'north', 'warn']"), + ("'row_diff_exclude_columns'", "list", "['loaded_at']"), + ( + "'table_settings'", + "dict", + "{'owner': 'orders', 'level': }", + ), + ( + "'row_diff_tolerances'", + "dict", + "{'by_type': {'DOUBLE': 0.5}, 'default': datetime.date(2024, 1, 2)}", + ), + ("", "Marker", "Marker()"), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=( + "description", + "row_diff_tolerances", + "table_settings", + "tags", + "warn", + ), + ), + ConfigValueBuildTestCase( + description="target context and whole-value templates", + header={ + "database": "db", + "alias": "${CTX:destination.qualified}", + "row_diff_sample_seed": "${ratio}", + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart']"), + ("'row_diff_exclude_columns'", "list", "['loaded_at']"), + ("'database'", "str", "'db'"), + ("'alias'", "str", "'db.north_orders.${CTX:destination.qualified}'"), + ("'row_diff_sample_seed'", "str", "'0.5'"), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=("alias", "database", "row_diff_sample_seed"), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_given_header_when_building_config_then_values_keep_types_and_order( + test_case: ConfigValueBuildTestCase, tmp_path: Path +) -> None: + outcome: tuple[tuple[tuple[str, str, str], ...], tuple[str, ...]] = config_value_outcome( + project_dir=tmp_path / "orders", header=test_case.header + ) + + assert outcome == (test_case.expected_values, test_case.expected_header_keys) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigValueRejectionTestCase( + description="a lone surrogate in a templated config", + header={"description": "x\udc80", "alias": "${region}"}, + expected_error=( + "UnicodeEncodeError", + "'utf-8' codec can't encode character '\\udc80' in position 1: surrogates not allowed", + ), + ), + ConfigValueRejectionTestCase( + description="a hook list of plain strings", + header={"pre_hooks": ["select 1"]}, + expected_error=( + "CompileInputError", + "model 'orders' pre_hooks entries must use typed inline_sql(...), sql(...), or python(...) hook syntax", + ), + ), + ConfigValueRejectionTestCase( + description="a missing hook list", + header={"post_hooks": None}, + expected_error=( + "CompileInputError", + "model 'orders' post_hooks must be a list of typed hook entries", + ), + ), + ConfigValueRejectionTestCase( + description="a non-string tag", + header={"tags": ["a", 1]}, + expected_error=("CompileInputError", "model 'orders' tags entries must be strings"), + ), + ConfigValueRejectionTestCase( + description="a non-string header key", + header={1: "x"}, + expected_error=("TypeError", "'int' object cannot be converted to 'PyString'"), + ), + ConfigValueRejectionTestCase( + description="a nested macro call", + header={"table_settings": {"a": ["@x ("]}}, + expected_error=( + "CompileInputError", + "model config field 'table_settings.a' does not allow macros", + ), + ), + ConfigValueRejectionTestCase( + description="an unknown variable", + header={"alias": "${missing}"}, + expected_error=( + "CompileInputError", + "model config references unknown variable 'missing'", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_header_when_building_config_then_the_build_raises_as_before( + test_case: ConfigValueRejectionTestCase, tmp_path: Path +) -> None: + with pytest.raises(Exception) as raised: + config_value_inputs(project_dir=tmp_path / "orders", header=test_case.header) + + assert (type(raised.value).__name__, str(raised.value)) == test_case.expected_error + + +@pytest.mark.parametrize( + "test_case", + [ + OpaqueConfigValueTestCase( + description="an object under a str-enum key", + header={"description": "x", Severity.WARN: MARKER}, + key=Severity.WARN, + expected_same_object=True, + ), + OpaqueConfigValueTestCase( + description="an object beside templated values", + header={"alias": "${region}", "owner": MARKER}, + key="owner", + expected_same_object=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_opaque_config_value_when_building_config_then_the_same_object_reaches_the_model( + test_case: OpaqueConfigValueTestCase, tmp_path: Path +) -> None: + kept: bool = marker_kept( + project_dir=tmp_path / "orders", header=test_case.header, key=test_case.key + ) + + assert kept == test_case.expected_same_object diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py new file mode 100644 index 0000000000..a17748b6a7 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py @@ -0,0 +1,44 @@ +"""Natively parsed MODEL header columns are reused whenever the built config keeps them.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + HeaderColumnPathTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import header_column_paths + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderColumnPathTestCase( + description="columns beside materialized, tags or a description", + headers=( + "materialized table, columns (order_id (type INTEGER))", + 'tags ["daily"], columns (order_id (type INTEGER))', + 'description "Orders.", columns (order_id (type INTEGER))', + ), + expected_native=3, + expected_parsed=0, + ), + HeaderColumnPathTestCase( + description="columns in a model whose other config values hold templates", + headers=('alias "orders_${region}", columns (order_id (type INTEGER))',), + expected_native=1, + expected_parsed=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_header_columns_when_building_model_inputs_then_native_columns_are_reused( + test_case: HeaderColumnPathTestCase, tmp_path: Path +) -> None: + paths: tuple[int, int] = header_column_paths( + project_dir=tmp_path / "orders", headers=test_case.headers + ) + + assert paths == (test_case.expected_native, test_case.expected_parsed)