From fd1ca821e3e0aac9dc9bafa73fdead8c42bd4f4a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:34:40 +0100 Subject: [PATCH 1/9] refactor(native): move discovery's header keys and header changes into sqlbuild-discovery The header key sets each statement accepts and the options built from them now live in `sqlbuild-discovery` (`constants`, `model_file_options`, `declaration_file_options`, `sql_test_file_options`), and the bindings call them. The delta compile's header comparison moves too: `header_changes` digests each header key, decides how an edit changed a model header and whether a custom rule reads a changed key. The bindings keep only the recording of digests into the compile image. The comparison gains unit tests. --- Cargo.lock | 1 + crates/sqlbuild-discovery/Cargo.toml | 1 + crates/sqlbuild-discovery/src/_helpers/mod.rs | 1 + .../src/_helpers}/sorted_keys.rs | 2 +- crates/sqlbuild-discovery/src/constants.rs | 121 ++++++++ .../main/declaration_file_options.rs | 15 + .../src/declaration_files/main/mod.rs | 1 + .../src/header_changes/_helpers/mod.rs | 2 + .../src/header_changes/_helpers/references.rs | 39 +++ .../header_changes/_helpers/value_encoding.rs | 79 ++++++ .../src/header_changes/constants.rs | 27 ++ .../header_changes/main/custom_rule_reads.rs | 36 +++ .../src/header_changes/main/header_change.rs | 81 ++++++ .../header_changes/main/header_key_digests.rs | 17 ++ .../src/header_changes/main/mod.rs | 3 + .../src/header_changes/mod.rs | 10 + .../src/header_changes/models.rs | 16 ++ .../src/header_changes/tests.rs | 8 + .../src/header_changes/tests/helpers.rs | 42 +++ .../tests/test_custom_rule_reads.rs | 48 ++++ .../tests/test_header_change.rs | 85 ++++++ .../src/header_changes/tests/test_types.rs | 16 ++ .../src/header_changes/types.rs | 4 + crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/model_files/main/mod.rs | 1 + .../model_files/main/model_file_options.rs | 24 ++ .../src/sql_tests/main/mod.rs | 1 + .../sql_tests/main/sql_test_file_options.rs | 18 ++ .../_helpers/compile_context/delta.rs | 5 +- .../bindings/_helpers/compile_delta/driver.rs | 6 +- .../bindings/_helpers/compile_delta/image.rs | 2 +- .../bindings/_helpers/discovery/constants.rs | 116 +------- .../_helpers/discovery/declaration_records.rs | 16 +- .../_helpers/discovery/header_keys.rs | 259 +----------------- .../src/bindings/_helpers/discovery/mod.rs | 1 - .../_helpers/discovery/model_records.rs | 26 +- .../_helpers/discovery/sql_test_records.rs | 12 +- .../_helpers/planner_scenario/execution.rs | 4 +- 38 files changed, 724 insertions(+), 423 deletions(-) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-discovery/src/_helpers}/sorted_keys.rs (77%) create mode 100644 crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/constants.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/main/header_change.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/mod.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/models.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/tests.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs create mode 100644 crates/sqlbuild-discovery/src/header_changes/types.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs diff --git a/Cargo.lock b/Cargo.lock index d2737da3f8..187548a457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,6 +1279,7 @@ version = "0.164.0" dependencies = [ "junction", "rayon", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-model-config", diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 3b1132a79f..0cd082a7fc 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-model-config.workspace = true diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs index 7c99418bd0..5b2e5261fc 100644 --- a/crates/sqlbuild-discovery/src/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -4,5 +4,6 @@ pub(crate) mod locations; pub(crate) mod pool; pub(crate) mod reading; pub(crate) mod scoped_paths; +pub(crate) mod sorted_keys; pub(crate) mod statement_headers; pub(crate) mod suggestions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs b/crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs rename to crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs index f377167577..4f04b9db33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs +++ b/crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs @@ -1,4 +1,4 @@ -//! Header key sets as the sorted lists native discovery options take. +//! Header key sets as the sorted lists the discovery options take. /// A header key set as owned keys, sorted by code point as Python sorts them. pub(crate) fn sorted_keys(keys: &[&str]) -> Vec { diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index f1bb0e5c90..e415a42148 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -47,3 +47,124 @@ pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; /// The singular audit directory below an audit role. pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; +/// Header keys a `MODEL` statement accepts, sorted by code point. +pub const SQL_MODEL_HEADER_KEYS: [&str; 73] = [ + "alias", + "append_cursor_inclusive", + "audit_factories", + "audits", + "batch_concurrency", + "batch_size", + "check_columns", + "columns", + "config", + "constants", + "contract", + "cursor", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_grain", + "cursor_inputs", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_type", + "cursor_watermark_mode", + "database", + "description", + "dynamic_columns", + "enabled", + "engine", + "enums", + "full_refresh", + "historical_input", + "incremental_mode", + "incremental_strategy", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "max_duration", + "max_microbatches", + "merge_exclude_columns", + "microbatch_limit", + "microbatch_strategy", + "migrate_force", + "migrate_from", + "model_schema", + "observed_at", + "old_name_view", + "on_schema_change", + "order_by", + "partition_by", + "placeholders", + "post_hooks", + "pre_hooks", + "primary_key", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "sql_analysis", + "sql_validation", + "table_settings", + "table_type", + "tags", + "time_travel_retention", + "unaccounted_partition_policy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +/// `MODEL` header keys that were removed and fail with a dedicated message. +pub const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"]; +/// Header keys a `FUNCTION` statement accepts. +pub const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [ + "arguments", + "database", + "description", + "returns", + "schema", + "tags", +]; +/// Header keys an `AUDIT` statement accepts. +pub const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [ + "always_run", + "evaluation", + "minimum_samples", + "name", + "run_scope", + "sample_count", + "sample_unit", + "severity", + "sql_analysis", + "thresholds", + "value", +]; +/// Header keys a `HOOK` statement accepts. +pub const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"]; +/// Header keys a `TEST` statement accepts. +pub const SQL_TEST_HEADER_KEYS: [&str; 8] = [ + "cases", + "cursor_end", + "cursor_start", + "mode", + "name", + "parameters", + "sql_analysis", + "timeout", +]; +/// Header keys a `SCENARIO` statement accepts. +pub const SQL_SCENARIO_HEADER_KEYS: [&str; 4] = + ["description", "expected_columns", "tags", "timeout"]; +/// `SCENARIO (expected_columns ...)` values: compare listed columns, or require every column. +pub const SCENARIO_EXPECTED_COLUMNS_KEY: &str = "expected_columns"; +pub const SCENARIO_EXPECTED_COLUMNS_VALUES: [&str; 2] = ["listed", "all"]; +pub const SCENARIO_EXPECTED_COLUMNS_ALL: &str = "all"; diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs b/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs new file mode 100644 index 0000000000..c677c9cc6a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs @@ -0,0 +1,15 @@ +//! The options declaration discovery parses headers with. + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{SQL_AUDIT_HEADER_KEYS, SQL_FUNCTION_HEADER_KEYS, SQL_HOOK_HEADER_KEYS}; +use crate::declaration_files::models::DeclarationFileOptions; + +/// The header keys `FUNCTION`, `AUDIT` and `HOOK` statements accept. +#[must_use] +pub fn declaration_file_options() -> DeclarationFileOptions { + DeclarationFileOptions { + function_keys: sorted_keys(&SQL_FUNCTION_HEADER_KEYS), + audit_keys: sorted_keys(&SQL_AUDIT_HEADER_KEYS), + hook_keys: sorted_keys(&SQL_HOOK_HEADER_KEYS), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs index 81483be916..443692bd2e 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod bare_number_failure; +pub mod declaration_file_options; pub mod discover_declaration_collection; pub mod parse_declaration_text; pub mod python_function_files; diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs new file mode 100644 index 0000000000..76aeb91265 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod references; +pub(crate) mod value_encoding; diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs new file mode 100644 index 0000000000..0d218d21c8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs @@ -0,0 +1,39 @@ +//! Whether an edited query still references only what the stored model read. + +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::header_changes::constants::{ + LINE_COMMENT, MODEL_KIND, REF_CALL, SEED_CALL, SEED_KIND, SOURCE_CALL, SOURCE_KIND, +}; + +/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. +pub(crate) fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT.to_owned()], + ..LexicalSyntax::default() + }; + let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { + return false; + }; + scan.references.iter().all(|reference| { + let kind: Option<&str> = match reference.kind { + REF_CALL => Some(MODEL_KIND), + SOURCE_CALL => Some(SOURCE_KIND), + SEED_CALL => Some(SEED_KIND), + _ => None, + }; + match (kind, &reference.package) { + (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), + _ => true, + } + }) +} + +/// Whether `upstream` holds the `kind` resource `name`. +fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { + upstream + .iter() + .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs b/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs new file mode 100644 index 0000000000..6d28d570ad --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs @@ -0,0 +1,79 @@ +//! Header values as short digests: an unambiguous byte encoding, hashed and truncated. + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::constants::{HEX, KEPT_DIGEST_BYTES}; + +/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. +#[derive(Default)] +pub(crate) struct ValueEncoding { + bytes: Vec, +} + +impl ValueEncoding { + fn text(&mut self, tag: u8, text: &str) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(text.len() as u64).to_le_bytes()); + self.bytes.extend_from_slice(text.as_bytes()); + } + + fn list(&mut self, tag: u8, items: &[AuthoredValue]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for item in items { + self.value(item); + } + } + + fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for (key, item) in items { + self.text(b'k', key); + self.value(item); + } + } + + pub(crate) fn value(&mut self, value: &AuthoredValue) { + match value { + AuthoredValue::Null => self.bytes.push(b'n'), + AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), + AuthoredValue::BareWord(word) => self.text(b'w', word), + AuthoredValue::String(string) => self.text(b's', string), + AuthoredValue::List(items) => self.list(b'l', items), + AuthoredValue::Map(items) => self.pairs(b'm', items), + AuthoredValue::Set(items) => self.list(b'e', items), + AuthoredValue::Tuple(items) => self.list(b't', items), + AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), + AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), + AuthoredValue::NamedSqlHook(name, items) => { + self.text(b'h', name); + self.pairs(b'a', items); + } + AuthoredValue::PythonHook(name, items) => { + self.text(b'p', name); + self.pairs(b'a', items); + } + } + } +} + +pub(crate) fn short_hex(digest: &[u8]) -> String { + let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); + for byte in &digest[..KEPT_DIGEST_BYTES] { + hex.push(char::from(HEX[usize::from(byte >> 4)])); + hex.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + hex +} + +/// The short digest of one header value. +pub(crate) fn value_digest(value: &AuthoredValue) -> String { + let mut encoding: ValueEncoding = ValueEncoding::default(); + encoding.value(value); + short_hex(&bytes_digest(&encoding.bytes)) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/constants.rs b/crates/sqlbuild-discovery/src/header_changes/constants.rs new file mode 100644 index 0000000000..fcea7e085b --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/constants.rs @@ -0,0 +1,27 @@ +//! The header keys and values a delta compile can recompile a model for. + +/// Header keys a delta compile recompiles the model for; any other changed key falls back. +pub(crate) const PATCHED_KEYS: [&str; 6] = [ + "description", + "columns", + "dynamic_columns", + "model_schema", + "audits", + "contract", +]; +/// `materialized` values a delta compile handles; others name materializations it cannot. +pub(crate) const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +/// Plain `--` comments, which every adapter's SQL reads as comments. +pub(crate) const LINE_COMMENT: &str = "--"; +/// Reference call kinds and the upstream kinds they resolve to. +pub(crate) const REF_CALL: &str = "ref"; +pub(crate) const SOURCE_CALL: &str = "source"; +pub(crate) const SEED_CALL: &str = "seed"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const SEED_KIND: &str = "seed"; + +/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. +pub(crate) const KEPT_DIGEST_BYTES: usize = 16; +pub(crate) const HEX: &[u8; 16] = b"0123456789abcdef"; diff --git a/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs b/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs new file mode 100644 index 0000000000..e833b04d0c --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs @@ -0,0 +1,36 @@ +//! Whether a custom rule reads what changed in a model header. + +use std::collections::BTreeSet; + +/// Whether a custom rule reads a fact of this model that the changed header keys decide. +#[must_use] +pub fn custom_rule_reads( + fact_digests: &[(Vec, String)], + model: (&str, &str), + changed: &BTreeSet, +) -> Option { + let (path, name) = model; + for (key, _) in fact_digests { + let Some((fact, arguments)) = key.split_first() else { + continue; + }; + if !arguments + .iter() + .any(|argument| argument == path || argument == name) + { + continue; + } + let reads: &[&str] = match fact.as_str() { + "contracts.enforced" | "contract" => &["contract"], + "contracts.grain" | "grain" => &["contract", "columns"], + "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], + "materialized" => &["materialized"], + "tests.for_model" | "audits.for_model" => &["audits", "columns"], + _ => &[], + }; + if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { + return Some(format!("a custom rule reads {name}'s {key}")); + } + } + None +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs b/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs new file mode 100644 index 0000000000..10dc646bca --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/header_change.rs @@ -0,0 +1,81 @@ +//! How an edit changed one model file's header. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::_helpers::references::references_read; +use crate::header_changes::constants::{MATERIALIZED_KEY, PATCHED_KEYS, PATCHED_MATERIALIZATIONS}; +use crate::header_changes::main::header_key_digests::header_key_digests; +use crate::header_changes::models::HeaderChange; +use crate::header_changes::types::HeaderKeys; +use crate::model_files::main::model_file_options::model_file_options; +use crate::model_files::main::parse_model_text::parse_model_text; + +/// How the header of the model file at `path` changed against its recorded keys. +#[must_use] +pub fn header_change( + project_dir: &Path, + relative_path: &str, + (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), +) -> HeaderChange { + let Some(stored) = stored else { + return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); + }; + let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { + return HeaderChange::Unparsed; + }; + let file_path: String = project_dir + .join(relative_path) + .to_string_lossy() + .into_owned(); + let Ok(parsed) = parse_model_text( + &file_path, + contents, + &model_file_options((false, false), None), + ) else { + return HeaderChange::Unparsed; + }; + if !references_read(&parsed.query_sql, upstream) { + return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); + } + let current: HeaderKeys = header_key_digests(&parsed.header_values); + let stored_map: BTreeMap<&str, &str> = stored + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let current_map: BTreeMap<&str, &str> = current + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let changed: BTreeSet = stored_map + .keys() + .chain(current_map.keys()) + .filter(|key| stored_map.get(*key) != current_map.get(*key)) + .map(|key| (*key).to_owned()) + .collect(); + if changed.is_empty() { + return HeaderChange::Unchanged; + } + for key in &changed { + if key == MATERIALIZED_KEY { + let value = parsed.header_values.iter().find(|(name, _)| name == key); + let supported: bool = matches!( + value, + Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) + if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) + ); + if !supported { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its materialization beyond views and tables" + )); + } + } else if !PATCHED_KEYS.contains(&key.as_str()) { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its {key} header key" + )); + } + } + HeaderChange::Patched(changed) +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs b/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs new file mode 100644 index 0000000000..02ff9c77ab --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/header_key_digests.rs @@ -0,0 +1,17 @@ +//! A model header's keys as short value digests. + +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::header_changes::_helpers::value_encoding::value_digest; +use crate::header_changes::types::HeaderKeys; + +/// Each header key with its value's short digest, sorted by key. +#[must_use] +pub fn header_key_digests(values: &[(String, AuthoredValue)]) -> HeaderKeys { + let mut keys: HeaderKeys = values + .iter() + .map(|(key, value)| (key.clone(), value_digest(value))) + .collect(); + keys.sort(); + keys +} diff --git a/crates/sqlbuild-discovery/src/header_changes/main/mod.rs b/crates/sqlbuild-discovery/src/header_changes/main/mod.rs new file mode 100644 index 0000000000..9d20ee7ce1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/main/mod.rs @@ -0,0 +1,3 @@ +pub mod custom_rule_reads; +pub mod header_change; +pub mod header_key_digests; diff --git a/crates/sqlbuild-discovery/src/header_changes/mod.rs b/crates/sqlbuild-discovery/src/header_changes/mod.rs new file mode 100644 index 0000000000..15398bd447 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/mod.rs @@ -0,0 +1,10 @@ +//! Model header changes: each header's keys as digests, and which keys an edit changed. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/header_changes/models.rs b/crates/sqlbuild-discovery/src/header_changes/models.rs new file mode 100644 index 0000000000..a8c12f79dd --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/models.rs @@ -0,0 +1,16 @@ +//! What an edit did to a model header. + +use std::collections::BTreeSet; + +/// What an edit did to one model's header. +#[derive(Debug, PartialEq, Eq)] +pub enum HeaderChange { + /// The header's keys and values are unchanged. + Unchanged, + /// These keys changed, every one of which the delta recompiles. + Patched(BTreeSet), + /// Why the delta cannot recompile this header change. + Unsupported(String), + /// The file does not parse; the delta reports its error as a full compile would. + Unparsed, +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests.rs b/crates/sqlbuild-discovery/src/header_changes/tests.rs new file mode 100644 index 0000000000..412e8eb6d9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_custom_rule_reads.rs"] +mod custom_rule_reads; +#[path = "tests/test_header_change.rs"] +mod header_change; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs b/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs new file mode 100644 index 0000000000..a3b65c38e5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/helpers.rs @@ -0,0 +1,42 @@ +use std::collections::BTreeSet; + +use crate::header_changes::main::header_change::header_change; +use crate::header_changes::main::header_key_digests::header_key_digests; +use crate::header_changes::models::HeaderChange; +use crate::header_changes::tests::test_types::HeaderChangeTestCase; +use crate::model_files::main::model_file_options::model_file_options; +use crate::model_files::main::parse_model_text::parse_model_text; + +const RELATIVE_PATH: &str = "models/orders.sql"; +const UPSTREAM: [(&str, &str); 1] = [("model", "customers")]; + +/// The change of a recorded model after its file is rewritten, or deleted when none is given. +pub(super) fn edited_change(test_case: &HeaderChangeTestCase) -> HeaderChange { + let project = tempfile::tempdir().expect("project directory"); + let path = project.path().join(RELATIVE_PATH); + std::fs::create_dir_all(path.parent().expect("models directory")).expect("models directory"); + let stored = parse_model_text( + &path.to_string_lossy(), + test_case.stored_contents.to_owned(), + &model_file_options((false, false), None), + ) + .expect("stored model parses"); + let keys = header_key_digests(&stored.header_values); + let upstream: Vec<(String, String)> = UPSTREAM + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(); + let _ = test_case + .edited_contents + .map(|contents| std::fs::write(&path, contents).expect("edited model")); + header_change( + project.path(), + RELATIVE_PATH, + (test_case.recorded.then_some(&keys), &upstream), + ) +} + +/// The set of changed keys a header change names. +pub(super) fn keys(names: &[&str]) -> BTreeSet { + names.iter().map(|name| (*name).to_owned()).collect() +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs new file mode 100644 index 0000000000..c64d0166e6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_custom_rule_reads.rs @@ -0,0 +1,48 @@ +use crate::header_changes::main::custom_rule_reads::custom_rule_reads; +use crate::header_changes::tests::helpers::keys; +use crate::header_changes::tests::test_types::CustomRuleReadsTestCase; + +#[test] +fn given_custom_rule_facts_when_a_header_changes_then_reads_of_changed_keys_are_named() { + let test_cases = [ + CustomRuleReadsTestCase { + description: "a fact of the model deciding a changed key is named", + fact_key: vec!["columns.names", "orders"], + changed: vec!["columns"], + expected_reason: Some("a custom rule reads orders's columns"), + }, + CustomRuleReadsTestCase { + description: "a fact of another model is ignored", + fact_key: vec!["columns.names", "customers"], + changed: vec!["columns"], + expected_reason: None, + }, + CustomRuleReadsTestCase { + description: "a fact no changed key decides is ignored", + fact_key: vec!["materialized", "models/orders.sql"], + changed: vec!["description"], + expected_reason: None, + }, + ]; + for test_case in test_cases { + let facts = vec![( + test_case + .fact_key + .iter() + .map(|part| (*part).to_owned()) + .collect::>(), + "digest".to_owned(), + )]; + assert_eq!( + custom_rule_reads( + &facts, + ("models/orders.sql", "orders"), + &keys(&test_case.changed) + ) + .as_deref(), + test_case.expected_reason, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs new file mode 100644 index 0000000000..a919ad92fd --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_header_change.rs @@ -0,0 +1,85 @@ +use crate::header_changes::models::HeaderChange; +use crate::header_changes::tests::helpers::{edited_change, keys}; +use crate::header_changes::tests::test_types::HeaderChangeTestCase; + +const STORED: &str = + "MODEL (materialized view, description 'orders');\nSELECT id FROM __ref(\"customers\")"; + +#[test] +fn given_an_edited_model_header_when_compared_then_the_change_is_classified() { + let test_cases = [ + HeaderChangeTestCase { + description: "an unrecorded model is unsupported", + stored_contents: STORED, + edited_contents: Some(STORED), + recorded: false, + expected_change: HeaderChange::Unsupported( + "models/orders.sql has no recorded header".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "an unreadable file is unparsed", + stored_contents: STORED, + edited_contents: None, + recorded: true, + expected_change: HeaderChange::Unparsed, + }, + HeaderChangeTestCase { + description: "the same header is unchanged", + stored_contents: STORED, + edited_contents: Some(STORED), + recorded: true, + expected_change: HeaderChange::Unchanged, + }, + HeaderChangeTestCase { + description: "a changed description and a table materialization are patched", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized table, description 'all orders');\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Patched(keys(&["description", "materialized"])), + }, + HeaderChangeTestCase { + description: "an incremental materialization is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized incremental, description 'orders');\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed its materialization beyond views and tables".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "another changed key is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized view, description 'orders', unique_key id);\nSELECT id FROM __ref(\"customers\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed its unique_key header key".to_owned(), + ), + }, + HeaderChangeTestCase { + description: "a new reference is unsupported", + stored_contents: STORED, + edited_contents: Some( + "MODEL (materialized view, description 'orders');\nSELECT id FROM __ref(\"products\")", + ), + recorded: true, + expected_change: HeaderChange::Unsupported( + "models/orders.sql changed what it references".to_owned(), + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + edited_change(&test_case), + test_case.expected_change, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs b/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs new file mode 100644 index 0000000000..7efd8e2582 --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::header_changes::models::HeaderChange; + +pub(super) struct HeaderChangeTestCase { + pub(super) description: &'static str, + pub(super) stored_contents: &'static str, + pub(super) edited_contents: Option<&'static str>, + pub(super) recorded: bool, + pub(super) expected_change: HeaderChange, +} + +pub(super) struct CustomRuleReadsTestCase { + pub(super) description: &'static str, + pub(super) fact_key: Vec<&'static str>, + pub(super) changed: Vec<&'static str>, + pub(super) expected_reason: Option<&'static str>, +} diff --git a/crates/sqlbuild-discovery/src/header_changes/types.rs b/crates/sqlbuild-discovery/src/header_changes/types.rs new file mode 100644 index 0000000000..817f5bc85d --- /dev/null +++ b/crates/sqlbuild-discovery/src/header_changes/types.rs @@ -0,0 +1,4 @@ +//! The recorded header of one model. + +/// Each header key with its value's short digest, sorted by key. +pub type HeaderKeys = Vec<(String, String)>; diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 9226c314a6..0f9a9b8350 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -6,6 +6,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod declaration_files; pub mod declarations; +pub mod header_changes; pub mod key_checks; pub mod model_files; pub mod models; diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index eb6f515e35..4d936ccc0a 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,4 +1,5 @@ pub mod discover_model_files; pub mod header_facts; +pub mod model_file_options; pub mod model_file_stem; pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs b/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs new file mode 100644 index 0000000000..a0e201f27f --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/model_file_options.rs @@ -0,0 +1,24 @@ +//! The options model discovery parses headers with. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS}; +use crate::model_files::models::ModelFileOptions; + +/// Model header options; `extraction` is `(implicit alias columns, output locations)`. +#[must_use] +pub fn model_file_options( + extraction: (bool, bool), + path_filter: Option>>, +) -> ModelFileOptions { + let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; + ModelFileOptions { + supported_keys: sorted_keys(&SQL_MODEL_HEADER_KEYS), + removed_keys: sorted_keys(&REMOVED_SQL_MODEL_HEADER_KEYS), + extract_implicit_alias_columns, + extract_output_column_locations, + path_filter, + } +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs index 032628c3f1..87039a540f 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs @@ -2,3 +2,4 @@ pub mod discover_scenario_files; pub mod discover_sql_test_files; pub mod parse_scenario_text; pub mod parse_sql_test_text; +pub mod sql_test_file_options; diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs b/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs new file mode 100644 index 0000000000..d6cae443d8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/sql_test_file_options.rs @@ -0,0 +1,18 @@ +//! The options SQL test and scenario discovery parses headers with. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::_helpers::sorted_keys::sorted_keys; +use crate::constants::{SQL_SCENARIO_HEADER_KEYS, SQL_TEST_HEADER_KEYS}; +use crate::sql_tests::models::SqlTestFileOptions; + +/// The header keys `TEST` and `SCENARIO` statements accept, discovering only `path_filter`. +#[must_use] +pub fn sql_test_file_options(path_filter: Option>>) -> SqlTestFileOptions { + SqlTestFileOptions { + test_keys: sorted_keys(&SQL_TEST_HEADER_KEYS), + scenario_keys: sorted_keys(&SQL_SCENARIO_HEADER_KEYS), + path_filter, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs index 3b9826048f..03a24163f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs @@ -4,18 +4,15 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use pyo3::prelude::PyResult; - use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_rules::engine::main::retained_request::retained_request; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; /// Whether edits that report or raise errors compile as deltas; until then they fall back. pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; -/// One model's header keys with the digest of each authored value, sorted by key. -pub(crate) type HeaderKeys = Vec<(String, String)>; - /// What one model contributes to the partial-check reasons, apart from its dependencies. #[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub(crate) struct ModelCoverage { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 127756794c..df7b38b822 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -33,10 +33,10 @@ use crate::bindings::_helpers::compile_driver::command::{ PhasesOutcome, resolved_request, unrecorded_compile, }; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; -use crate::bindings::_helpers::discovery::header_keys::{ - HeaderChange, custom_rule_reads, header_change, -}; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; +use sqlbuild_discovery::header_changes::main::header_change::header_change; +use sqlbuild_discovery::header_changes::models::HeaderChange; /// Set to `1` to always compile the whole project. pub(crate) const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index aebdc05f91..24322ec018 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -7,13 +7,13 @@ use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; -use crate::bindings::_helpers::compile_context::delta::HeaderKeys; use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compile_context::delta::{BaseRules, PendingRules}; use crate::bindings::_helpers::compile_context::delta::{ CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, }; use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; +use sqlbuild_discovery::header_changes::types::HeaderKeys; /// Changes whenever the image's meaning changes, so an older image is never read. pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs index 6f14cf3cd6..ef9a10ebaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -1,119 +1,5 @@ -//! Header keys native discovery accepts, sorted by code point as Python sorts them. +//! The fields discovery reports for SQL hook entries. -pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 73] = [ - "alias", - "append_cursor_inclusive", - "audit_factories", - "audits", - "batch_concurrency", - "batch_size", - "check_columns", - "columns", - "config", - "constants", - "contract", - "cursor", - "cursor_end", - "cursor_future_action", - "cursor_future_max_distance", - "cursor_grain", - "cursor_inputs", - "cursor_start", - "cursor_start_max_action", - "cursor_start_max_ahead", - "cursor_type", - "cursor_watermark_mode", - "database", - "description", - "dynamic_columns", - "enabled", - "engine", - "enums", - "full_refresh", - "historical_input", - "incremental_mode", - "incremental_strategy", - "initial_valid_from", - "invalidate_hard_deletes", - "lookback", - "materialized", - "max_duration", - "max_microbatches", - "merge_exclude_columns", - "microbatch_limit", - "microbatch_strategy", - "migrate_force", - "migrate_from", - "model_schema", - "observed_at", - "old_name_view", - "on_schema_change", - "order_by", - "partition_by", - "placeholders", - "post_hooks", - "pre_hooks", - "primary_key", - "replay_on_change", - "row_diff_exclude_columns", - "row_diff_sample_rows", - "row_diff_sample_seed", - "row_diff_tolerances", - "schema", - "snapshot_full_refresh", - "snapshot_schema_change", - "snapshot_strategy", - "sql_analysis", - "sql_validation", - "table_settings", - "table_type", - "tags", - "time_travel_retention", - "unaccounted_partition_policy", - "unique_key", - "updated_at", - "valid_from_column", - "valid_to_column", -]; -pub(crate) const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"]; -pub(crate) const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [ - "arguments", - "database", - "description", - "returns", - "schema", - "tags", -]; -pub(crate) const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [ - "always_run", - "evaluation", - "minimum_samples", - "name", - "run_scope", - "sample_count", - "sample_unit", - "severity", - "sql_analysis", - "thresholds", - "value", -]; -pub(crate) const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"]; -pub(crate) const SQL_TEST_HEADER_KEYS: [&str; 8] = [ - "cases", - "cursor_end", - "cursor_start", - "mode", - "name", - "parameters", - "sql_analysis", - "timeout", -]; -pub(crate) const SQL_SCENARIO_HEADER_KEYS: [&str; 4] = - ["description", "expected_columns", "tags", "timeout"]; -/// `SCENARIO (expected_columns ...)` values: compare listed columns, or require every column. -pub(crate) const SCENARIO_EXPECTED_COLUMNS_KEY: &str = "expected_columns"; -pub(crate) const SCENARIO_EXPECTED_COLUMNS_VALUES: [&str; 2] = ["listed", "all"]; -pub(crate) const SCENARIO_EXPECTED_COLUMNS_ALL: &str = "all"; /// The fields a SQL hook entry reports, in output order. pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ "statement", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index c1415677ac..bc3b154b28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -7,8 +7,8 @@ use sqlbuild_discovery::declaration_files::main::discover_declaration_collection use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ AuditBlock, AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, - DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, - FunctionFile, HookFile, MacroFile, SchemaFile, ScopedFile, + DeclarationCollection, DiscoverySession, EnumFile, FileScope, FunctionFile, HookFile, + MacroFile, SchemaFile, ScopedFile, }; use sqlbuild_discovery::declaration_files::types::Collection; use sqlbuild_discovery::models::{FileOutcome, LineColumnSpan}; @@ -18,7 +18,6 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords}; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; use crate::bindings::_helpers::discovery::project_files::path_object; @@ -26,7 +25,6 @@ use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, st use crate::bindings::_helpers::discovery::snapshot::{ in_snapshot, project_root, project_tree, snapshot_memo, }; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::model_config::header_columns::schema_columns; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::{enum_from_value, new_record, new_record_with}; @@ -34,6 +32,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; +use sqlbuild_discovery::declaration_files::main::declaration_file_options::declaration_file_options; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; @@ -377,14 +376,7 @@ pub(crate) fn discovery_session<'py>( let created: Bound<'py, NativeDiscoverySession> = Bound::new( py, NativeDiscoverySession { - inner: DiscoverySession::new( - project_root(project_dir)?, - DeclarationFileOptions { - function_keys: sorted_keys(&constants::SQL_FUNCTION_HEADER_KEYS), - audit_keys: sorted_keys(&constants::SQL_AUDIT_HEADER_KEYS), - hook_keys: sorted_keys(&constants::SQL_HOOK_HEADER_KEYS), - }, - ), + inner: DiscoverySession::new(project_root(project_dir)?, declaration_file_options()), }, )?; memo.set_item(SESSION_MEMO_KEY, &created)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs index 5abec8d9f4..9fdbdf97be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs @@ -1,270 +1,19 @@ -//! Each model header's keys as digests, so a delta compile can tell which keys an edit changed. +//! Record each discovered model header's keys as digests, so a delta compile can tell which keys an edit changed. -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; - -use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; -use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; +use sqlbuild_discovery::header_changes::main::header_key_digests::header_key_digests; +use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; -use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compile_context::delta::HeaderKeys; use crate::bindings::_helpers::compile_context::delta::{capture, capturing}; -use crate::bindings::_helpers::discovery::model_records::model_file_options; - -/// Header keys a delta compile recompiles the model for; any other changed key falls back. -const PATCHED_KEYS: [&str; 6] = [ - "description", - "columns", - "dynamic_columns", - "model_schema", - "audits", - "contract", -]; -/// `materialized` values a delta compile handles; others name materializations it cannot. -const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; -const MATERIALIZED_KEY: &str = "materialized"; -/// Plain `--` comments, which every adapter's SQL reads as comments. -const LINE_COMMENT: &str = "--"; -/// Reference call kinds and the upstream kinds they resolve to. -const REF_CALL: &str = "ref"; -const SOURCE_CALL: &str = "source"; -const SEED_CALL: &str = "seed"; -const MODEL_KIND: &str = "model"; -const SOURCE_KIND: &str = "source"; -const SEED_KIND: &str = "seed"; - -/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. -const KEPT_DIGEST_BYTES: usize = 16; -const HEX: &[u8; 16] = b"0123456789abcdef"; - -/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. -#[derive(Default)] -struct ValueEncoding { - bytes: Vec, -} - -impl ValueEncoding { - fn text(&mut self, tag: u8, text: &str) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(text.len() as u64).to_le_bytes()); - self.bytes.extend_from_slice(text.as_bytes()); - } - - fn list(&mut self, tag: u8, items: &[AuthoredValue]) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(items.len() as u64).to_le_bytes()); - for item in items { - self.value(item); - } - } - - fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { - self.bytes.push(tag); - self.bytes - .extend_from_slice(&(items.len() as u64).to_le_bytes()); - for (key, item) in items { - self.text(b'k', key); - self.value(item); - } - } - - fn value(&mut self, value: &AuthoredValue) { - match value { - AuthoredValue::Null => self.bytes.push(b'n'), - AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), - AuthoredValue::BareWord(word) => self.text(b'w', word), - AuthoredValue::String(string) => self.text(b's', string), - AuthoredValue::List(items) => self.list(b'l', items), - AuthoredValue::Map(items) => self.pairs(b'm', items), - AuthoredValue::Set(items) => self.list(b'e', items), - AuthoredValue::Tuple(items) => self.list(b't', items), - AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), - AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), - AuthoredValue::NamedSqlHook(name, items) => { - self.text(b'h', name); - self.pairs(b'a', items); - } - AuthoredValue::PythonHook(name, items) => { - self.text(b'p', name); - self.pairs(b'a', items); - } - } - } -} - -/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. -fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { - let syntax: LexicalSyntax = LexicalSyntax { - line_comment_prefixes: vec![LINE_COMMENT.to_owned()], - ..LexicalSyntax::default() - }; - let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { - return false; - }; - scan.references.iter().all(|reference| { - let kind: Option<&str> = match reference.kind { - REF_CALL => Some(MODEL_KIND), - SOURCE_CALL => Some(SOURCE_KIND), - SEED_CALL => Some(SEED_KIND), - _ => None, - }; - match (kind, &reference.package) { - (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), - _ => true, - } - }) -} - -/// Whether `upstream` holds the `kind` resource `name`. -fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { - upstream - .iter() - .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) -} - -fn short_hex(digest: &[u8]) -> String { - let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); - for byte in &digest[..KEPT_DIGEST_BYTES] { - hex.push(char::from(HEX[usize::from(byte >> 4)])); - hex.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - hex -} - -fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { - let mut keys: HeaderKeys = values - .iter() - .map(|(key, value)| { - let mut encoding: ValueEncoding = ValueEncoding::default(); - encoding.value(value); - let hex: String = short_hex(&bytes_digest(&encoding.bytes)); - (key.clone(), hex) - }) - .collect(); - keys.sort(); - keys -} /// Record one discovered model's header keys for the image. pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { if !capturing() { return; } - let keys: HeaderKeys = header_keys(values); + let keys: HeaderKeys = header_key_digests(values); let path: String = relative_path.replace('\\', "/"); capture(|capture| { capture.header_keys.insert(path, keys); }); } - -/// What an edit did to one model's header. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum HeaderChange { - /// The header's keys and values are unchanged. - Unchanged, - /// These keys changed, every one of which the delta recompiles. - Patched(BTreeSet), - /// Why the delta cannot recompile this header change. - Unsupported(String), - /// The file does not parse; the delta reports its error as a full compile would. - Unparsed, -} - -/// How the header of the model file at `path` changed against its recorded keys. -pub(crate) fn header_change( - project_dir: &Path, - relative_path: &str, - (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), -) -> HeaderChange { - let Some(stored) = stored else { - return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); - }; - let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { - return HeaderChange::Unparsed; - }; - let file_path: String = project_dir - .join(relative_path) - .to_string_lossy() - .into_owned(); - let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { - return HeaderChange::Unparsed; - }; - if !references_read(&parsed.query_sql, upstream) { - return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); - } - let current: HeaderKeys = header_keys(&parsed.header_values); - let stored_map: BTreeMap<&str, &str> = stored - .iter() - .map(|(key, digest)| (key.as_str(), digest.as_str())) - .collect(); - let current_map: BTreeMap<&str, &str> = current - .iter() - .map(|(key, digest)| (key.as_str(), digest.as_str())) - .collect(); - let changed: BTreeSet = stored_map - .keys() - .chain(current_map.keys()) - .filter(|key| stored_map.get(*key) != current_map.get(*key)) - .map(|key| (*key).to_owned()) - .collect(); - if changed.is_empty() { - return HeaderChange::Unchanged; - } - for key in &changed { - if key == MATERIALIZED_KEY { - let value = parsed.header_values.iter().find(|(name, _)| name == key); - let supported: bool = matches!( - value, - Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) - if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) - ); - if !supported { - return HeaderChange::Unsupported(format!( - "{relative_path} changed its materialization beyond views and tables" - )); - } - } else if !PATCHED_KEYS.contains(&key.as_str()) { - return HeaderChange::Unsupported(format!( - "{relative_path} changed its {key} header key" - )); - } - } - HeaderChange::Patched(changed) -} - -/// Whether a custom rule reads a fact of this model that the changed header keys decide. -pub(crate) fn custom_rule_reads( - fact_digests: &[(Vec, String)], - model: (&str, &str), - changed: &BTreeSet, -) -> Option { - let (path, name) = model; - for (key, _) in fact_digests { - let Some((fact, arguments)) = key.split_first() else { - continue; - }; - if !arguments - .iter() - .any(|argument| argument == path || argument == name) - { - continue; - } - let reads: &[&str] = match fact.as_str() { - "contracts.enforced" | "contract" => &["contract"], - "contracts.grain" | "grain" => &["contract", "columns"], - "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], - "materialized" => &["materialized"], - "tests.for_model" | "audits.for_model" => &["audits", "columns"], - _ => &[], - }; - if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { - return Some(format!("a custom rule reads {name}'s {key}")); - } - } - None -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 43c35fbb89..77a3165166 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -29,7 +29,6 @@ pub(crate) mod raised; pub(crate) mod registration; pub(crate) mod scope_snapshot; pub(crate) mod snapshot; -pub(crate) mod sorted_keys; pub(crate) mod sql_test_blocks; pub(crate) mod sql_test_records; pub(crate) mod yaml_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 9975ca2f83..8b78654cca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -10,15 +10,14 @@ use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use std::path::Path; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::model_file_record::DiscoveredPaths; use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::models::NativeProjectTree; +use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; @@ -106,17 +105,6 @@ pub(crate) struct ModelFileSelection<'py> { pub(crate) on_fault: Option>, } -/// The options model discovery parses headers with. -pub(crate) fn model_file_options() -> ModelFileOptions { - ModelFileOptions { - supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), - removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), - extract_implicit_alias_columns: false, - extract_output_column_locations: false, - path_filter: None, - } -} - /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. pub(crate) fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, @@ -124,15 +112,11 @@ pub(crate) fn discovered_model_files<'py>( selection: ModelFileSelection<'py>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; - let options = ModelFileOptions { - supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), - removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), - extract_implicit_alias_columns, - extract_output_column_locations, - path_filter: crate::bindings::_helpers::compile_context::delta::injection() + let options: ModelFileOptions = model_file_options( + extraction, + crate::bindings::_helpers::compile_context::delta::injection() .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), - }; + ); let items: Vec> = model_items(project_dir, &options)?; let faults = FileFaults { project_dir: project_dir.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 5d2de99d53..df25bb5648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -17,14 +17,12 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::project_files::path_object; use crate::bindings::_helpers::discovery::raised::{ discovery_error, failure_error, read_error, stage_error, }; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::discovery::sql_test_blocks::{ sql_test_block_records, validate_block_names, }; @@ -33,6 +31,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::NativeProjectTree; +use sqlbuild_discovery::constants; +use sqlbuild_discovery::sql_tests::main::sql_test_file_options::sql_test_file_options; const SQL_TESTS_OWNERSHIP_ROOT: &str = "tests/unit"; const SQL_SCENARIOS_OWNERSHIP_ROOT: &str = "tests/scenarios"; @@ -54,12 +54,10 @@ type RecordBuilder<'py, T> = fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; fn options() -> PyResult { - Ok(SqlTestFileOptions { - test_keys: sorted_keys(&constants::SQL_TEST_HEADER_KEYS), - scenario_keys: sorted_keys(&constants::SQL_SCENARIO_HEADER_KEYS), - path_filter: crate::bindings::_helpers::compile_context::delta::injection() + Ok(sql_test_file_options( + crate::bindings::_helpers::compile_context::delta::injection() .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), - }) + )) } fn path_class(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 8f58f43a4b..e209fbbc4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -4,9 +4,6 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; -use crate::bindings::_helpers::discovery::constants::{ - SCENARIO_EXPECTED_COLUMNS_ALL, SCENARIO_EXPECTED_COLUMNS_KEY, -}; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; @@ -37,6 +34,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; use sqlbuild_analysis::compiler::main::scenario_expected_columns::scenario_expected_columns; use sqlbuild_analysis::compiler::main::scenario_reader_sql::scenario_reader_sql; +use sqlbuild_discovery::constants::{SCENARIO_EXPECTED_COLUMNS_ALL, SCENARIO_EXPECTED_COLUMNS_KEY}; #[pyfunction] #[pyo3(signature = (**kwargs))] From 941409efd5a3a8397a612e1227b5c7c62e06c9c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:44:26 +0100 Subject: [PATCH 2/9] refactor(native): move plain scope logic from the bindings into sqlbuild-scopes The conversions from a cached full index to the report and projection views (`cached_report_records`, `cached_projection_index` and their parts), the scope path and qualified identity messages, the `as_posix` path text helpers, and the merge of new runtime usages (`merged_usages`, with `UsageKey`) now live in `sqlbuild-scopes`. The bindings read Python objects into those types and convert the results back, nothing more. `distinct_positions` and `merged_dependencies` become helpers of the usage merge, their only remaining caller. --- .../_helpers/input_loops/model_loop_named.rs | 5 +- .../_helpers/scopes/compile_usages.rs | 88 +++------- .../_helpers/scopes/declaration_resolution.rs | 31 +--- .../bindings/_helpers/scopes/scope_index.rs | 10 +- .../bindings/_helpers/scopes/scope_lookup.rs | 56 ++----- .../bindings/_helpers/scopes/scope_objects.rs | 158 +----------------- .../_helpers/scopes/scope_report_objects.rs | 10 +- .../bindings/_helpers/scopes/scope_reports.rs | 28 ++-- .../_helpers/scopes/scope_validation.rs | 17 +- .../_helpers/scopes/scope_visibility.rs | 5 +- .../{main => _helpers}/distinct_positions.rs | 2 +- .../{main => _helpers}/merged_dependencies.rs | 2 +- .../src/compile_usages/_helpers/mod.rs | 2 + .../src/compile_usages/main/merged_usages.rs | 39 +++++ .../src/compile_usages/main/mod.rs | 3 +- .../sqlbuild-scopes/src/compile_usages/mod.rs | 1 + .../src/compile_usages/models.rs | 22 +++ .../tests/test_compile_usages.rs | 4 +- .../src/compile_usages/types.rs | 4 + .../main/declared_scope_path_message.rs | 28 ++++ .../main/identity_failure_message.rs | 22 +++ .../src/scope_index/main/mod.rs | 5 + .../src/scope_index/main/posix_parent_text.rs | 13 ++ .../src/scope_index/main/posix_path_text.rs | 18 ++ .../scope_index/main/rejected_path_message.rs | 16 ++ .../sqlbuild-scopes/src/scope_index/tests.rs | 2 + .../scope_index/tests/test_posix_path_text.rs | 41 +++++ .../src/scope_index/tests/test_types.rs | 8 + .../scope_report/main/cached_completeness.rs | 16 ++ .../main/cached_declaration_values.rs | 27 +++ .../main/cached_diagnostic_facts.rs | 25 +++ .../main/cached_projection_index.rs | 77 +++++++++ .../main/cached_report_records.rs | 39 +++++ .../src/scope_report/main/mod.rs | 5 + 34 files changed, 505 insertions(+), 324 deletions(-) rename crates/sqlbuild-scopes/src/compile_usages/{main => _helpers}/distinct_positions.rs (86%) rename crates/sqlbuild-scopes/src/compile_usages/{main => _helpers}/merged_dependencies.rs (96%) create mode 100644 crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs create mode 100644 crates/sqlbuild-scopes/src/compile_usages/types.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs create mode 100644 crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs create mode 100644 crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs create mode 100644 crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs create mode 100644 crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs create mode 100644 crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs index f1312f45ac..5c38ef1c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs @@ -58,7 +58,8 @@ pub(crate) const AUDIT_DECLARATION: NamedKind = NamedKind { label: "Generic audit", role: "audits/generic", }; -use crate::bindings::_helpers::scopes::compile_usages::{CURRENT_FOLDER, scope_path_message}; +use crate::bindings::_helpers::scopes::compile_usages::CURRENT_FOLDER; +use sqlbuild_scopes::scope_index::main::declared_scope_path_message::declared_scope_path_message; /// The usage of the indexed declaration `name`, if any, reporting S006 when it is hidden. pub(crate) fn named_usages<'py>( @@ -114,7 +115,7 @@ fn path_error(context: &LoopContext<'_, '_>, declared: &DeclaredScope, label: &s match context .classes .scope_path_error - .call1((scope_path_message(declared, label),)) + .call1((declared_scope_path_message(declared, label),)) { Ok(instance) => PyErr::from_value(instance), Err(failure) => failure, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index bd93222659..ea1f64a8a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -8,22 +8,20 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::compile_usages::main::distinct_positions::distinct_positions; -use sqlbuild_scopes::compile_usages::main::merged_dependencies::merged_dependencies; +use sqlbuild_scopes::compile_usages::main::merged_usages::merged_usages; use sqlbuild_scopes::compile_usages::main::schema_scope_usages::schema_scope_usages; use sqlbuild_scopes::compile_usages::models::{ - DependencySource, InaccessibleDiagnostic, NamedRecord, SchemaDeclaration, SchemaUsages, + DependencySource, InaccessibleDiagnostic, MergedUsages, NamedRecord, SchemaDeclaration, + SchemaUsages, UsageKey, }; -use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::declared_scope_path_message::declared_scope_path_message; use sqlbuild_scopes::scope_index::models::{ - DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, ScopePathFailure, + DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, }; use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; -use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::construct::new_record; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_items; +use crate::bindings::_helpers::records::construct::{error_class, new_record}; +use crate::bindings::_helpers::records::fields::{field as native_field, mapping_items}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; /// The folder a scope path is relative to when its owner names none. @@ -34,15 +32,6 @@ const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); -/// Consumer, declaration, usage kind, what it is used through, and the enum member. -#[derive(Clone, Hash, PartialEq, Eq)] -pub(crate) struct UsageRow { - pub(crate) consumer: QualifiedRow, - pub(crate) declaration: QualifiedRow, - pub(crate) kind: String, - pub(crate) through: Option, - pub(crate) enum_member: Option, -} /// Path, scope, private owner, owning path and ownership root path of a named record. struct RecordRow { path: String, @@ -249,7 +238,10 @@ fn scope_schema_usages( Err(rejected) => ( Vec::new(), Vec::new(), - Some(scope_path_message(&rejected.declared, &rejected.path)), + Some(declared_scope_path_message( + &rejected.declared, + &rejected.path, + )), ), }, ) @@ -269,56 +261,22 @@ fn source_rows(sources: Option>) -> Option> /// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. pub(crate) fn merged_usage_rows( declarations: &[(QualifiedRow, Option>)], - prior: &[UsageRow], - collected: &[UsageRow], + prior: &[UsageKey], + collected: &[UsageKey], ) -> MergeRows { - let distinct: Vec = distinct_positions(collected); - let mut consumed: Vec<(Option, QualifiedRow)> = - Vec::with_capacity(distinct.len()); - for position in &distinct { - let UsageRow { - consumer, - declaration, - .. - } = &collected[*position]; - let by_declaration: Option = DeclarationKind::parse(&consumer.0) - .is_some() - .then(|| consumer.clone()); - consumed.push((by_declaration, declaration.clone())); - } - let merged: Vec>> = merged_dependencies(declarations, &consumed) - .into_iter() - .map(source_rows) - .collect(); - let mut combined: Vec<&UsageRow> = prior.iter().collect(); - combined.extend(distinct.iter().map(|position| &collected[*position])); - let combined_positions: Vec = distinct_positions(&combined); - (distinct, merged, combined_positions) + let MergedUsages { + distinct, + merged, + combined, + } = merged_usages(declarations, prior, collected); + ( + distinct, + merged.into_iter().map(source_rows).collect(), + combined, + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; Ok(()) } - -/// The `InvalidScopePathError` message of the first path Python's visibility check rejects. -pub(crate) fn scope_path_message(declared: &DeclaredScope, label: &str) -> String { - let owner: &str = declared - .owning_path - .as_deref() - .filter(|path| !path.is_empty()) - .unwrap_or(CURRENT_FOLDER); - let (path, failure) = match normalize_scope_path(label) { - Err(failure) => (label, failure), - Ok(_) => match normalize_scope_path(owner) { - Err(failure) => (owner, failure), - Ok(_) => (label, ScopePathFailure::NotProjectRelative), - }, - }; - match failure { - ScopePathFailure::NotProjectRelative => { - format!("Scope path must be project-relative: {path}") - } - ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs index 6e8ce688a7..2cdb1cab5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs @@ -7,6 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, intern}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::parse_identity::parse_identity; +use sqlbuild_scopes::scope_index::main::posix_parent_text::posix_parent_text; +use sqlbuild_scopes::scope_index::main::posix_path_text::posix_path_text; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, IdentityFailure, QualifiedIdentity, ResourceIdentity, ScopePathFailure, }; @@ -82,7 +84,10 @@ impl NativeDeclarationContexts { native_field(record, "ownership_root")?.getattr(intern!(py, "path"))?; let key: Bound<'py, PyTuple> = PyTuple::new( py, - [PyString::new(py, &posix_parent(&path)).into_any(), root], + [ + PyString::new(py, &posix_parent_text(&path)).into_any(), + root, + ], )?; if let Some(cached) = cache.get_item(&key)? { let consumer: Bound<'py, PyAny> = native_field(record, "identity")?; @@ -187,7 +192,7 @@ impl NativeDeclarationContexts { let owner: String = owning_path .filter(|path| !path.is_empty()) .unwrap_or_else(|| CURRENT_FOLDER.to_owned()); - return Ok(posix_path(&format!( + return Ok(posix_path_text(&format!( "{owner}{POSIX_SEPARATOR}{DECLARATION_STEM}" ))); } @@ -341,28 +346,6 @@ fn target_path(resolver: &Bound<'_, PyAny>, file_path: &str) -> PyResult }) } -/// `Path(path).parent.as_posix()` of a normalized relative path. -fn posix_parent(path: &str) -> String { - match path.rfind(POSIX_SEPARATOR) { - Some(index) => posix_path(&path[..index]), - None => CURRENT_FOLDER.to_owned(), - } -} - -/// `Path(path).as_posix()`: empty and `.` components dropped. -fn posix_path(path: &str) -> String { - let absolute: bool = path.starts_with(POSIX_SEPARATOR); - let parts: Vec<&str> = path - .split(POSIX_SEPARATOR) - .filter(|part| !part.is_empty() && *part != CURRENT_FOLDER) - .collect(); - match (absolute, parts.is_empty()) { - (true, _) => format!("{POSIX_SEPARATOR}{}", parts.join("/")), - (false, true) => CURRENT_FOLDER.to_owned(), - (false, false) => parts.join("/"), - } -} - fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { match class.call1((message,)) { Ok(instance) => PyErr::from_value(instance), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 84db425ada..eed4695a73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -12,6 +12,7 @@ use sqlbuild_scopes::relationship_names::models::RelationshipScan; use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationInput, DeclarationKind, GrantRecord, GrantThrough, @@ -22,13 +23,14 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::mapping_values; -use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; +use crate::bindings::_helpers::records::fields::{ + field as native_field, mapping_values, path_stem, +}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::relationship_scan::{ RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, }; -use crate::bindings::_helpers::scopes::scope_lookup::{path_message, scope_path_error}; +use crate::bindings::_helpers::scopes::scope_lookup::scope_path_error; use crate::bindings::_helpers::scopes::scope_objects::{ ScopeClasses, field, optional_item, root_object, value_of, }; @@ -662,7 +664,7 @@ impl<'py> Walk<'py> { fn normalized(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { let path: String = text(path)?; normalize_scope_path(&path).map_err(|failure| { - scope_path_error(py, path_message(&RejectedPath { failure, path })) + scope_path_error(py, rejected_path_message(&RejectedPath { failure, path })) .unwrap_or_else(|error| error) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 3ee4cd952a..46ac2d125a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -5,16 +5,18 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{pyclass, pyfunction, wrap_pyfunction}; -use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; use sqlbuild_scopes::scope_index::main::report_target::report_target; use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; use sqlbuild_scopes::scope_index::models::{ - GrantKind, IdentityFailure, KeyedGroups, RejectedPath, ReportLookup, ReportRecords, - ReportTarget, ResolvedVisibility, ResolvedVisible, ResourceRecord, ScopePathFailure, - TargetFailure, TargetMatch, VisibilityGroups, + GrantKind, KeyedGroups, RejectedPath, ReportLookup, ReportRecords, ReportTarget, + ResolvedVisibility, ResolvedVisible, ResourceRecord, TargetFailure, TargetMatch, + VisibilityGroups, }; +use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::error_class; @@ -22,7 +24,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, qualified_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, read_index, report_records, resource_identity_object, tuple_attribute, + ScopeClasses, read_index, resource_identity_object, tuple_attribute, }; use crate::bindings::types::CompilerDetach; @@ -76,34 +78,6 @@ pub(crate) fn grant_kind(value: &str) -> PyResult { } } -/// Python's `InvalidScopePathError` message for a rejected path. -pub(crate) fn path_message(rejected: &RejectedPath) -> String { - match rejected.failure { - ScopePathFailure::NotProjectRelative => { - format!("Scope path must be project-relative: {}", rejected.path) - } - ScopePathFailure::EscapesProject => { - format!("Scope path escapes the project: {}", rejected.path) - } - } -} - -/// Python's `InvalidQualifiedIdentityError` message for a rejected identity. -pub(crate) fn identity_message(failure: &IdentityFailure, value: &str) -> String { - let shown = |text: &str| -> String { python_str_repr(text) }; - match failure { - IdentityFailure::NoName => format!("Qualified identity has no name: {}", shown(value)), - IdentityFailure::UnknownKind(kind) => { - format!("Unknown qualified identity kind: {}", shown(kind)) - } - IdentityFailure::InvalidPrivate => format!("Invalid private identity: {}", shown(value)), - IdentityFailure::Unqualified => format!( - "Expected a kind-qualified identity such as 'model:name': {}", - shown(value) - ), - } -} - /// Input positions as canonical positions. fn canonical(order: &[usize], positions: Vec) -> Vec { positions @@ -139,14 +113,14 @@ impl NativeScopeLookup { Err(TargetFailure::Identity(failure)) => TargetRows { declarations: false, positions: Vec::new(), - identity_error: Some(identity_message(&failure, text)), + identity_error: Some(identity_failure_message(&failure, text)), path_error: None, }, Err(TargetFailure::Path(rejected)) => TargetRows { declarations: false, positions: Vec::new(), identity_error: None, - path_error: Some(path_message(&rejected)), + path_error: Some(rejected_path_message(&rejected)), }, }) } @@ -163,7 +137,11 @@ impl NativeScopeLookup { .map_err(compiler_error)?; Ok(match resolved { Ok(resolved) => resolved_rows(resolved, &self.canonical.grants), - Err(rejected) => (Vec::new(), Vec::new(), Some(path_message(&rejected))), + Err(rejected) => ( + Vec::new(), + Vec::new(), + Some(rejected_path_message(&rejected)), + ), }) } } @@ -279,13 +257,13 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { - let records: ReportRecords = report_records(&read_index(index)?); + let records: ReportRecords = cached_report_records(&read_index(index)?); let built = py .compiler_detach(|| Ok(report_lookup(&records))) .map_err(compiler_error)?; let lookup: ReportLookup = match built { Ok(lookup) => lookup, - Err(rejected) => return Err(scope_path_error(py, path_message(&rejected))?), + Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), }; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; @@ -365,7 +343,7 @@ fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult Ok(normalized), Err(failure) => Err(scope_path_error( py, - path_message(&RejectedPath { failure, path }), + rejected_path_message(&RejectedPath { failure, path }), )?), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index 8f9205a9c8..eddbcb156f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -7,14 +7,9 @@ use sqlbuild_scopes::scope_cache::models::{ CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, CachedVisibility, ConstantFacts, EnumFacts, MacroFacts, }; -use sqlbuild_scopes::scope_index::main::format_identity::format_identity; use sqlbuild_scopes::scope_index::models::{ - DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportDiagnosticKey, ReportGrant, - ReportRecords, ReportUsage, ResourceIdentity, ResourceKind, ResourceRecord, -}; -use sqlbuild_scopes::scope_report::models::{ - Completeness, ConstantValues, DeclarationValues, DiagnosticFacts, EnumValues, MacroValues, - ProjectionGrant, ProjectionIndex, ProjectionStanding, ProjectionUsage, ReportValues, + DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportGrant, ResourceIdentity, + ResourceKind, ResourceRecord, }; use crate::bindings::_helpers::records::construct::{new_record, record_class}; @@ -307,155 +302,6 @@ pub(crate) fn read_placement_index(index: &Bound<'_, PyAny>) -> PyResult ReportRecords { - ReportRecords { - roots: index.roots.clone(), - resources: index.resources.clone(), - declarations: index - .declarations - .iter() - .map(|item| item.record.clone()) - .collect(), - usages: index - .usages - .iter() - .map(|item| ReportUsage { - consumer: item.consumer.clone(), - declaration: item.declaration.clone(), - kind: item.kind.clone(), - through: item.through.clone(), - }) - .collect(), - grants: index.grants.clone(), - diagnostics: index - .diagnostics - .iter() - .map(|item| ReportDiagnosticKey { - path: item.path.clone(), - line: item.line, - column: item.column, - code: item.code.clone(), - }) - .collect(), - } -} - -/// One declaration's report values. -pub(crate) fn declaration_values(item: &CachedDeclaration) -> DeclarationValues { - DeclarationValues { - macro_values: item.macro_facts.as_ref().map(|facts| MacroValues { - parameters: facts.parameters.clone(), - dependencies: facts.dependencies.clone(), - source_digest: facts.source_digest.clone(), - }), - enum_values: item.enum_facts.as_ref().map(|facts| EnumValues { - members: facts.members.clone(), - scalar_type: facts.scalar_type.clone(), - }), - constant_values: item.constant_facts.as_ref().map(|facts| ConstantValues { - logical_type: facts.logical_type.clone(), - collection_kind: facts.collection_kind.clone(), - item_count: facts.item_count, - nullable: facts.nullable, - render_as: facts.render_as.clone(), - }), - } -} - -/// The six completeness sections as the report engine reads them. -pub(crate) fn completeness(done: [bool; 6]) -> Completeness { - Completeness { - discovery: done[0], - static_visibility: done[1], - runtime_usage: done[2], - relationships: done[3], - placement: done[4], - promotion_impact: done[5], - } -} - -/// A diagnostic's facts, with its identities as qualified text. -pub(crate) fn diagnostic_facts(item: CachedDiagnostic) -> DiagnosticFacts { - DiagnosticFacts { - code: item.code, - message: item.message, - severity: item.severity, - path: item.path, - line: item.line, - column: item.column, - declaration: item - .declaration - .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity))), - resource: item - .resource - .map(|identity| format_identity(&QualifiedIdentity::Resource(identity))), - } -} - -/// The metadata projection's view of a full index. -pub(crate) fn projection_index(index: CachedIndex) -> ProjectionIndex { - ProjectionIndex { - roots: index.roots, - resources: index.resources, - values: ReportValues { - declarations: index.declarations.iter().map(declaration_values).collect(), - completeness: completeness(index.completeness), - }, - declarations: index - .declarations - .into_iter() - .map(|item| item.record) - .collect(), - usages: index - .usages - .into_iter() - .map(|item| ProjectionUsage { - consumer: item.consumer, - declaration: item.declaration, - kind: item.kind, - through: item.through, - enum_member: item.enum_member, - }) - .collect(), - grants: index - .grants - .into_iter() - .map(|item| ProjectionGrant { - resource: item.resource, - declaration: item.declaration, - through: item.through, - kind: item.kind, - }) - .collect(), - visibility: index - .visibility - .into_iter() - .map(|item| ProjectionStanding { - resource: item.resource, - declaration: item.declaration, - reason: item.reason, - through: item.through, - }) - .collect(), - inaccessible: index - .inaccessible - .into_iter() - .map(|item| ProjectionStanding { - resource: item.resource, - declaration: item.declaration, - reason: item.reason, - through: None, - }) - .collect(), - diagnostics: index - .diagnostics - .into_iter() - .map(diagnostic_facts) - .collect(), - } -} - /// A `ResourceIdentity`. pub(crate) fn resource_identity_object<'py>( classes: &ScopeClasses<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 6f50a0d376..c6b64dd4d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -8,6 +8,7 @@ use pyo3::types::{ PyTypeMethods, }; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_scopes::scope_report::main::cached_diagnostic_facts::cached_diagnostic_facts; use sqlbuild_scopes::scope_report::models::{ BrowseResult, DeclarationReport, DeclarationTarget, Explanation, ListResult, MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, ScopeChainEntry, @@ -17,8 +18,7 @@ use sqlbuild_scopes::scope_report::models::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, diagnostic_facts, field, optional_item, read_declaration_identity, - read_diagnostic, read_qualified, + ScopeClasses, field, optional_item, read_declaration_identity, read_diagnostic, read_qualified, }; type PyObject = Py; @@ -553,9 +553,9 @@ fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> native_field(item, "diagnostics")? .try_iter()? .map(|diagnostic| { - Ok(ReportDiagnostic::Raised(diagnostic_facts(read_diagnostic( - &diagnostic?, - )?))) + Ok(ReportDiagnostic::Raised(cached_diagnostic_facts( + read_diagnostic(&diagnostic?)?, + ))) }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index a532e253a1..82496de146 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -7,8 +7,13 @@ use pyo3::prelude::{ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; +use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; +use sqlbuild_scopes::scope_report::main::cached_completeness::cached_completeness; +use sqlbuild_scopes::scope_report::main::cached_declaration_values::cached_declaration_values; +use sqlbuild_scopes::scope_report::main::cached_projection_index::cached_projection_index; use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; @@ -24,12 +29,9 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; -use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, identity_message, path_message, -}; +use crate::bindings::_helpers::scopes::scope_lookup::NativeScopeLookup; use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, completeness, declaration_values, projection_index, read_completeness, - read_declaration, read_index, + ScopeClasses, read_completeness, read_declaration, read_index, }; use crate::bindings::_helpers::scopes::scope_report_objects::{ browse_object, diagnostics_object, explanation_object, json_to_plain, json_to_python, @@ -85,7 +87,7 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { let native: Py = native_field(lookup, "native")?.extract()?; let rows: Vec = native_field(&index, "declarations")? .try_iter()? - .map(|record| Ok(declaration_values(&read_declaration(&record?)?))) + .map(|record| Ok(cached_declaration_values(&read_declaration(&record?)?))) .collect::>()?; let canonical: &[usize] = &native.get().lookup.declarations; if rows.len() != canonical.len() { @@ -101,7 +103,7 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { lookup: native, values: ReportValues { declarations, - completeness: completeness(read_completeness(&index)?), + completeness: cached_completeness(read_completeness(&index)?), }, }) } @@ -115,13 +117,15 @@ fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult, failure: &ReportFailure) -> PyResult { match failure { - ReportFailure::Path(rejected) => { - scope_error(py, ErrorId::InvalidScopePath, path_message(rejected)) - } + ReportFailure::Path(rejected) => scope_error( + py, + ErrorId::InvalidScopePath, + rejected_path_message(rejected), + ), ReportFailure::Identity(failure, text) => scope_error( py, ErrorId::InvalidQualifiedIdentity, - identity_message(failure, text), + identity_failure_message(failure, text), ), ReportFailure::Serialization(message) => Ok(PyValueError::new_err(message.clone())), ReportFailure::IndexOutOfRange => Ok(PyIndexError::new_err("tuple index out of range")), @@ -342,7 +346,7 @@ pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - let index: ProjectionIndex = projection_index(read_index(index)?); + let index: ProjectionIndex = cached_projection_index(read_index(index)?); Ok(scope_metadata_projection(&index)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index bd5caf53ff..973f36e8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::compile_usages::models::UsageKey; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; use sqlbuild_scopes::placement::main::relocate_for_resource_move::relocate_for_resource_move; @@ -14,13 +15,11 @@ use sqlbuild_scopes::placement::models::{ use sqlbuild_scopes::scope_index::models::ResourceIdentity; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::records::construct::enum_member; -use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::replaced_keywords; +use crate::bindings::_helpers::records::construct::{enum_member, error_class}; +use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::compile_usages::{ - MergeRows, SourceRow, UsageRow, merged_usage_rows, model_schema_scope_usages, + MergeRows, SourceRow, merged_usage_rows, model_schema_scope_usages, }; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ @@ -56,8 +55,8 @@ fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult) -> PyResult { - Ok(UsageRow { +fn usage_row(usage: &Bound<'_, PyAny>) -> PyResult { + Ok(UsageKey { consumer: qualified_row(&native_field(usage, "consumer")?)?, declaration: qualified_row(&native_field(usage, "declaration")?)?, kind: value_of(usage, "kind")?, @@ -125,11 +124,11 @@ fn merged_usage_index<'py>( .iter() .map(|record| dependency_row(&record)) .collect::>()?; - let prior_rows: Vec = prior + let prior_rows: Vec = prior .iter() .map(|usage| usage_row(&usage)) .collect::>()?; - let collected_rows: Vec = collected + let collected_rows: Vec = collected .iter() .map(|usage| usage_row(&usage)) .collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 46b48fc41a..c4265dcd29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, }; @@ -14,7 +15,7 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, ResolvedRows, TargetRows, path_message, scope_path_error, + NativeScopeLookup, ResolvedRows, TargetRows, scope_path_error, }; use crate::bindings::_helpers::scopes::scope_objects::{ ScopeClasses, qualified_row, read_resource, tuple_attribute, @@ -217,7 +218,7 @@ fn scope_resolve_path_visibility( failure, path: text, }; - return Err(scope_path_error(py, path_message(&rejected))?); + return Err(scope_path_error(py, rejected_path_message(&rejected))?); } }; let resource: ResourceRecord = ResourceRecord { diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs similarity index 86% rename from crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs rename to crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs index b1a563fe9d..8b04822d29 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/distinct_positions.rs @@ -5,7 +5,7 @@ use std::hash::Hash; /// The positions of the first occurrence of each distinct key. #[must_use] -pub fn distinct_positions(keys: &[K]) -> Vec { +pub(crate) fn distinct_positions(keys: &[K]) -> Vec { let mut seen: HashSet<&K> = HashSet::with_capacity(keys.len()); let mut positions: Vec = Vec::new(); for (position, key) in keys.iter().enumerate() { diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs similarity index 96% rename from crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs rename to crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs index 7aa9399940..c60d744552 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/merged_dependencies.rs @@ -7,7 +7,7 @@ use crate::compile_usages::models::DependencySource; /// Per declaration, `None` without macro metadata, else own then consumed dependencies, deduped. #[must_use] -pub fn merged_dependencies( +pub(crate) fn merged_dependencies( declarations: &[(K, Option>)], usages: &[(Option, K)], ) -> Vec>> { diff --git a/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs index 04abf7838f..7880f80eea 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod distinct_positions; +pub(crate) mod merged_dependencies; pub(crate) mod schema_checks; diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs new file mode 100644 index 0000000000..0881e02946 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/merged_usages.rs @@ -0,0 +1,39 @@ +//! New runtime usages merged into an index's declarations and usages. + +use crate::compile_usages::_helpers::distinct_positions::distinct_positions; +use crate::compile_usages::_helpers::merged_dependencies::merged_dependencies; +use crate::compile_usages::models::{MergedUsages, UsageKey}; +use crate::compile_usages::types::QualifiedKey; +use crate::scope_index::models::DeclarationKind; + +/// Distinct `collected` usages, each declaration's merged dependencies, and the distinct prior-then-new usages. +#[must_use] +pub fn merged_usages( + declarations: &[(QualifiedKey, Option>)], + prior: &[UsageKey], + collected: &[UsageKey], +) -> MergedUsages { + let distinct: Vec = distinct_positions(collected); + let mut consumed: Vec<(Option, QualifiedKey)> = + Vec::with_capacity(distinct.len()); + for position in &distinct { + let UsageKey { + consumer, + declaration, + .. + } = &collected[*position]; + let by_declaration: Option = DeclarationKind::parse(&consumer.0) + .is_some() + .then(|| consumer.clone()); + consumed.push((by_declaration, declaration.clone())); + } + let merged = merged_dependencies(declarations, &consumed); + let mut combined: Vec<&UsageKey> = prior.iter().collect(); + combined.extend(distinct.iter().map(|position| &collected[*position])); + let combined: Vec = distinct_positions(&combined); + MergedUsages { + distinct, + merged, + combined, + } +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs index b8f3347ba3..c91cdee02f 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs @@ -1,3 +1,2 @@ -pub mod distinct_positions; -pub mod merged_dependencies; +pub mod merged_usages; pub mod schema_scope_usages; diff --git a/crates/sqlbuild-scopes/src/compile_usages/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/mod.rs index 1e64f3d116..1644773e62 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/mod.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-scopes/src/compile_usages/models.rs b/crates/sqlbuild-scopes/src/compile_usages/models.rs index 1ed254abf0..ba794bec24 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/models.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/models.rs @@ -1,5 +1,6 @@ //! The schema declarations and indexed records compile usage facts are read from. +use crate::compile_usages::types::QualifiedKey; use crate::scope_index::models::DeclaredScope; /// The first indexed record of a named schema or enum, by its caller handle. @@ -51,3 +52,24 @@ pub enum DependencySource { /// The declaration of the collected usage at this position. Usage(usize), } + +/// One usage: consumer, declaration, usage kind, what it is used through, and the enum member. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct UsageKey { + pub consumer: QualifiedKey, + pub declaration: QualifiedKey, + pub kind: String, + pub through: Option, + pub enum_member: Option, +} + +/// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MergedUsages { + /// Positions of the distinct collected usages. + pub distinct: Vec, + /// Per declaration, its merged dependency sources; `None` without macro metadata. + pub merged: Vec>>, + /// Positions of the distinct usages among the prior usages followed by the new ones. + pub combined: Vec, +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs index 2ad12b9697..9401aefd87 100644 --- a/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs +++ b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs @@ -1,5 +1,5 @@ -use crate::compile_usages::main::distinct_positions::distinct_positions; -use crate::compile_usages::main::merged_dependencies::merged_dependencies; +use crate::compile_usages::_helpers::distinct_positions::distinct_positions; +use crate::compile_usages::_helpers::merged_dependencies::merged_dependencies; use crate::compile_usages::main::schema_scope_usages::schema_scope_usages; use crate::compile_usages::models::{DependencySource, SchemaUsages}; use crate::compile_usages::tests::helpers::{ diff --git a/crates/sqlbuild-scopes/src/compile_usages/types.rs b/crates/sqlbuild-scopes/src/compile_usages/types.rs new file mode 100644 index 0000000000..b8fa417cd2 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/types.rs @@ -0,0 +1,4 @@ +//! The identity keys compile usages are merged by. + +/// A declaration's kind, name and private owner `(kind, name)`; a resource's kind and name with no owner. +pub type QualifiedKey = (String, String, Option<(String, String)>); diff --git a/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs new file mode 100644 index 0000000000..e5d0462974 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/declared_scope_path_message.rs @@ -0,0 +1,28 @@ +//! The message of a declaration whose scope path or owning path is rejected. + +use crate::scope_index::constants::CURRENT_PATH_COMPONENT; +use crate::scope_index::main::normalize_scope_path::normalize_scope_path; +use crate::scope_index::models::{DeclaredScope, ScopePathFailure}; + +/// The `InvalidScopePathError` message of the first path Python's visibility check rejects. +#[must_use] +pub fn declared_scope_path_message(declared: &DeclaredScope, label: &str) -> String { + let owner: &str = declared + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_PATH_COMPONENT); + let (path, failure) = match normalize_scope_path(label) { + Err(failure) => (label, failure), + Ok(_) => match normalize_scope_path(owner) { + Err(failure) => (owner, failure), + Ok(_) => (label, ScopePathFailure::NotProjectRelative), + }, + }; + match failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {path}") + } + ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs new file mode 100644 index 0000000000..b1dba9bc13 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/identity_failure_message.rs @@ -0,0 +1,22 @@ +//! The message of a rejected qualified identity. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::scope_index::models::IdentityFailure; + +/// Python's `InvalidQualifiedIdentityError` message for a rejected identity. +#[must_use] +pub fn identity_failure_message(failure: &IdentityFailure, value: &str) -> String { + let shown = |text: &str| -> String { python_str_repr(text) }; + match failure { + IdentityFailure::NoName => format!("Qualified identity has no name: {}", shown(value)), + IdentityFailure::UnknownKind(kind) => { + format!("Unknown qualified identity kind: {}", shown(kind)) + } + IdentityFailure::InvalidPrivate => format!("Invalid private identity: {}", shown(value)), + IdentityFailure::Unqualified => format!( + "Expected a kind-qualified identity such as 'model:name': {}", + shown(value) + ), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs index 083babc7d0..8ce06eaa13 100644 --- a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -1,13 +1,18 @@ pub mod build_scope_index; pub mod classify_report_resource; pub mod classify_resource; +pub mod declared_scope_path_message; pub mod explain_report_declaration; pub mod format_identity; +pub mod identity_failure_message; pub mod identity_repr; pub mod macro_source_digest; pub mod normalize_scope_path; pub mod parse_identity; pub mod path_visible; +pub mod posix_parent_text; +pub mod posix_path_text; +pub mod rejected_path_message; pub mod relationship_grants; pub mod report_lookup; pub mod report_target; diff --git a/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs b/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs new file mode 100644 index 0000000000..4f6e6c99da --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/posix_parent_text.rs @@ -0,0 +1,13 @@ +//! The parent of a `/`-separated path as Python's `Path.as_posix` prints it. + +use crate::scope_index::constants::{CURRENT_PATH_COMPONENT, PATH_SEPARATOR}; +use crate::scope_index::main::posix_path_text::posix_path_text; + +/// `Path(path).parent.as_posix()` of a normalized relative path. +#[must_use] +pub fn posix_parent_text(path: &str) -> String { + match path.rfind(PATH_SEPARATOR) { + Some(index) => posix_path_text(&path[..index]), + None => CURRENT_PATH_COMPONENT.to_owned(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs b/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs new file mode 100644 index 0000000000..1c50a740f4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/posix_path_text.rs @@ -0,0 +1,18 @@ +//! A `/`-separated path as Python's `Path.as_posix` prints it. + +use crate::scope_index::constants::{CURRENT_PATH_COMPONENT, PATH_SEPARATOR}; + +/// `Path(path).as_posix()`: empty and `.` components dropped. +#[must_use] +pub fn posix_path_text(path: &str) -> String { + let absolute: bool = path.starts_with(PATH_SEPARATOR); + let parts: Vec<&str> = path + .split(PATH_SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT_PATH_COMPONENT) + .collect(); + match (absolute, parts.is_empty()) { + (true, _) => format!("{PATH_SEPARATOR}{}", parts.join("/")), + (false, true) => CURRENT_PATH_COMPONENT.to_owned(), + (false, false) => parts.join("/"), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs b/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs new file mode 100644 index 0000000000..e8626234ec --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/rejected_path_message.rs @@ -0,0 +1,16 @@ +//! The message of a rejected scope path. + +use crate::scope_index::models::{RejectedPath, ScopePathFailure}; + +/// Python's `InvalidScopePathError` message for a rejected path. +#[must_use] +pub fn rejected_path_message(rejected: &RejectedPath) -> String { + match rejected.failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {}", rejected.path) + } + ScopePathFailure::EscapesProject => { + format!("Scope path escapes the project: {}", rejected.path) + } + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs index 4bfad639e1..77f150b393 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -10,6 +10,8 @@ mod macro_source_digest; mod parse_identity; #[path = "tests/test_path_visible.rs"] mod path_visible; +#[path = "tests/test_posix_path_text.rs"] +mod posix_path_text; #[path = "tests/test_relationship_grants.rs"] mod relationship_grants; #[path = "tests/test_report_lookup.rs"] diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs new file mode 100644 index 0000000000..dad0d3258f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_posix_path_text.rs @@ -0,0 +1,41 @@ +use crate::scope_index::main::posix_parent_text::posix_parent_text; +use crate::scope_index::main::posix_path_text::posix_path_text; +use crate::scope_index::tests::test_types::PosixPathTestCase; + +#[test] +fn given_slash_separated_paths_when_printed_then_they_match_python_as_posix() { + let test_cases = [ + PosixPathTestCase { + description: "empty and current components are dropped", + path: "models//./orders.sql", + expected_text: "models/orders.sql", + expected_parent: "models", + }, + PosixPathTestCase { + description: "an empty path prints as the current folder", + path: "", + expected_text: ".", + expected_parent: ".", + }, + PosixPathTestCase { + description: "an absolute path keeps its root", + path: "/models/orders.sql", + expected_text: "/models/orders.sql", + expected_parent: "/models", + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + posix_path_text(test_case.path), + posix_parent_text(test_case.path) + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_parent.to_owned() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs index 25c945a4cc..f98a132691 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -115,3 +115,11 @@ pub(super) struct MacroDigestTestCase { pub(super) source: &'static str, pub(super) expected_digest: &'static str, } + +/// A `/`-separated path and how Python prints it and its parent. +pub(super) struct PosixPathTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) expected_text: &'static str, + pub(super) expected_parent: &'static str, +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs new file mode 100644 index 0000000000..675c7dd26c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_completeness.rs @@ -0,0 +1,16 @@ +//! The completeness sections of a cached index. + +use crate::scope_report::models::Completeness; + +/// The six completeness sections as the report engine reads them. +#[must_use] +pub fn cached_completeness(done: [bool; 6]) -> Completeness { + Completeness { + discovery: done[0], + static_visibility: done[1], + runtime_usage: done[2], + relationships: done[3], + placement: done[4], + promotion_impact: done[5], + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs new file mode 100644 index 0000000000..0145b81061 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_declaration_values.rs @@ -0,0 +1,27 @@ +//! One cached declaration's report values. + +use crate::scope_cache::models::CachedDeclaration; +use crate::scope_report::models::{ConstantValues, DeclarationValues, EnumValues, MacroValues}; + +/// One declaration's report values. +#[must_use] +pub fn cached_declaration_values(item: &CachedDeclaration) -> DeclarationValues { + DeclarationValues { + macro_values: item.macro_facts.as_ref().map(|facts| MacroValues { + parameters: facts.parameters.clone(), + dependencies: facts.dependencies.clone(), + source_digest: facts.source_digest.clone(), + }), + enum_values: item.enum_facts.as_ref().map(|facts| EnumValues { + members: facts.members.clone(), + scalar_type: facts.scalar_type.clone(), + }), + constant_values: item.constant_facts.as_ref().map(|facts| ConstantValues { + logical_type: facts.logical_type.clone(), + collection_kind: facts.collection_kind.clone(), + item_count: facts.item_count, + nullable: facts.nullable, + render_as: facts.render_as.clone(), + }), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs new file mode 100644 index 0000000000..823b475e41 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_diagnostic_facts.rs @@ -0,0 +1,25 @@ +//! One cached diagnostic's report facts. + +use crate::scope_cache::models::CachedDiagnostic; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::QualifiedIdentity; +use crate::scope_report::models::DiagnosticFacts; + +/// A diagnostic's facts, with its identities as qualified text. +#[must_use] +pub fn cached_diagnostic_facts(item: CachedDiagnostic) -> DiagnosticFacts { + DiagnosticFacts { + code: item.code, + message: item.message, + severity: item.severity, + path: item.path, + line: item.line, + column: item.column, + declaration: item + .declaration + .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity))), + resource: item + .resource + .map(|identity| format_identity(&QualifiedIdentity::Resource(identity))), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs new file mode 100644 index 0000000000..929ae3d66e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_projection_index.rs @@ -0,0 +1,77 @@ +//! The metadata projection's view of a cached full index. + +use crate::scope_cache::models::CachedIndex; +use crate::scope_report::main::cached_completeness::cached_completeness; +use crate::scope_report::main::cached_declaration_values::cached_declaration_values; +use crate::scope_report::main::cached_diagnostic_facts::cached_diagnostic_facts; +use crate::scope_report::models::{ + ProjectionGrant, ProjectionIndex, ProjectionStanding, ProjectionUsage, ReportValues, +}; + +/// The metadata projection's view of a full index. +#[must_use] +pub fn cached_projection_index(index: CachedIndex) -> ProjectionIndex { + ProjectionIndex { + roots: index.roots, + resources: index.resources, + values: ReportValues { + declarations: index + .declarations + .iter() + .map(cached_declaration_values) + .collect(), + completeness: cached_completeness(index.completeness), + }, + declarations: index + .declarations + .into_iter() + .map(|item| item.record) + .collect(), + usages: index + .usages + .into_iter() + .map(|item| ProjectionUsage { + consumer: item.consumer, + declaration: item.declaration, + kind: item.kind, + through: item.through, + enum_member: item.enum_member, + }) + .collect(), + grants: index + .grants + .into_iter() + .map(|item| ProjectionGrant { + resource: item.resource, + declaration: item.declaration, + through: item.through, + kind: item.kind, + }) + .collect(), + visibility: index + .visibility + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: item.through, + }) + .collect(), + inaccessible: index + .inaccessible + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: None, + }) + .collect(), + diagnostics: index + .diagnostics + .into_iter() + .map(cached_diagnostic_facts) + .collect(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs b/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs new file mode 100644 index 0000000000..c5885e1871 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/cached_report_records.rs @@ -0,0 +1,39 @@ +//! The lookup records of a cached full index. + +use crate::scope_cache::models::CachedIndex; +use crate::scope_index::models::{ReportDiagnosticKey, ReportRecords, ReportUsage}; + +/// The lookup's records of a full index. +#[must_use] +pub fn cached_report_records(index: &CachedIndex) -> ReportRecords { + ReportRecords { + roots: index.roots.clone(), + resources: index.resources.clone(), + declarations: index + .declarations + .iter() + .map(|item| item.record.clone()) + .collect(), + usages: index + .usages + .iter() + .map(|item| ReportUsage { + consumer: item.consumer.clone(), + declaration: item.declaration.clone(), + kind: item.kind.clone(), + through: item.through.clone(), + }) + .collect(), + grants: index.grants.clone(), + diagnostics: index + .diagnostics + .iter() + .map(|item| ReportDiagnosticKey { + path: item.path.clone(), + line: item.line, + column: item.column, + code: item.code.clone(), + }) + .collect(), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/mod.rs b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs index ed557b8a5e..3d0b07f168 100644 --- a/crates/sqlbuild-scopes/src/scope_report/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs @@ -1,5 +1,10 @@ pub mod browse_scope_folders; pub mod build_scope_report; +pub mod cached_completeness; +pub mod cached_declaration_values; +pub mod cached_diagnostic_facts; +pub mod cached_projection_index; +pub mod cached_report_records; pub mod explain_scope_declaration; pub mod list_scope_declarations; pub mod preview_scope_move; From e7d8e51b874705e6317dcad01de9609e3b39f97c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:04:14 +0100 Subject: [PATCH 3/9] feat(native): add a typed model config value and layer described models with it `sqlbuild-model-config` gains `config_values`: `ConfigNode` holds the builtin Python values config is made of (None, bool, int, float, str, list, tuple, dict) as Rust data, and every other value as an opaque host value carrying its handle and the facts config logic reads of it, taken once with the same rules as before. It implements `AuthoredNode`, so the existing config scans read it unchanged, and `layered_config` merges one layer over another as `_merged_with_tag_union` does. The bindings convert Python values into nodes and project nodes back; opaque values project to the very same object. Externally described models now layer defaults, path defaults and headers through the typed merge, the first consumer. --- .../src/config_values/_helpers/authored.rs | 91 +++++++ .../src/config_values/_helpers/entries.rs | 71 ++++++ .../src/config_values/_helpers/mod.rs | 4 + .../src/config_values/_helpers/reprs.rs | 44 ++++ .../src/config_values/_helpers/unions.rs | 71 ++++++ .../src/config_values/constants.rs | 10 + .../src/config_values/main/config_entry.rs | 13 + .../src/config_values/main/layered_config.rs | 53 ++++ .../src/config_values/main/mod.rs | 2 + .../src/config_values/mod.rs | 10 + .../src/config_values/models.rs | 64 +++++ .../src/config_values/tests.rs | 10 + .../src/config_values/tests/helpers.rs | 33 +++ .../config_values/tests/test_config_entry.rs | 36 +++ .../tests/test_layered_config.rs | 102 ++++++++ .../config_values/tests/test_python_text.rs | 59 +++++ .../src/config_values/tests/test_types.rs | 26 ++ .../src/config_values/types.rs | 6 + crates/sqlbuild-model-config/src/lib.rs | 1 + .../_helpers/model_config/config_nodes.rs | 238 ++++++++++++++++++ .../_helpers/model_config/described_models.rs | 64 +++-- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../model_config/model_config_builder.rs | 12 - fensu.toml | 2 +- 24 files changed, 986 insertions(+), 37 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/constants.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/config_entry.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/layered_config.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/mod.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/models.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/helpers.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs new file mode 100644 index 0000000000..665252c953 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs @@ -0,0 +1,91 @@ +//! Config values read as authored values, as `PyNode` reads the Python objects they mirror. + +use crate::config_values::_helpers::reprs::{python_repr, python_str}; +use crate::config_values::models::{ConfigNode, HostChildren}; +use crate::types::{AuthoredNode, NodeKind}; + +impl AuthoredNode for ConfigNode { + fn kind(&self) -> NodeKind { + match self { + Self::Null => NodeKind::Null, + Self::Bool(flag) => NodeKind::Bool(*flag), + Self::Int(number) => NodeKind::Int { + negative: *number < 0, + }, + Self::Float(_) => NodeKind::Other, + Self::Str(_) => NodeKind::Str, + Self::List(_) => NodeKind::List, + Self::Tuple(_) => NodeKind::Tuple, + Self::Map(_) => NodeKind::Map, + Self::Host(host) => host.facts.kind, + } + } + + fn integer(&self) -> Option { + match self { + Self::Int(number) => Some(*number), + Self::Host(host) => host.facts.integer, + _ => None, + } + } + + /// Python's `float(int)`, which rounds the decimal value to the nearest float, as parsing does. + fn number(&self) -> Option { + match self { + Self::Int(number) => Some(*number as f64), + Self::Float(number) => Some(*number), + Self::Host(host) => host.facts.number, + _ => None, + } + } + + fn text(&self) -> Option { + self.with_text(str::to_owned) + } + + fn is_text(&self, text: &str) -> bool { + self.with_text(|value| value == text).unwrap_or(false) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + match self { + Self::Str(text) => Some(read(text)), + Self::Host(host) => host.facts.text.as_deref().map(read), + _ => None, + } + } + + fn python_str(&self) -> String { + python_str(self) + } + + fn python_repr(&self) -> String { + python_repr(self) + } + + fn is_date_like(&self) -> bool { + matches!(self, Self::Host(host) if host.facts.date_like) + } + + fn items(&self) -> Vec { + match self { + Self::List(items) | Self::Tuple(items) => items.clone(), + Self::Host(host) => match &host.children { + HostChildren::Items(items) => items.clone(), + _ => Vec::new(), + }, + _ => Vec::new(), + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + match self { + Self::Map(entries) => entries.clone(), + Self::Host(host) => match &host.children { + HostChildren::Entries(entries) => entries.clone(), + _ => Vec::new(), + }, + _ => Vec::new(), + } + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs new file mode 100644 index 0000000000..2b04f00cc2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs @@ -0,0 +1,71 @@ +//! A config mapping read and written as a Python `dict`. + +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// Python's `==` between two dict keys: equal text, equal numbers, or equal tuples. +pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { + let text = |node: &ConfigNode| match node { + ConfigNode::Str(text) => Some(text.as_str().to_owned()), + ConfigNode::Host(host) if host.facts.kind == NodeKind::Str && host.facts.strict_text => { + host.facts.text.clone() + } + _ => None, + }; + match (left, right) { + (ConfigNode::Null, ConfigNode::Null) => true, + (ConfigNode::Tuple(left), ConfigNode::Tuple(right)) => { + left.len() == right.len() && left.iter().zip(right).all(|(a, b)| same_key(a, b)) + } + _ => match (text(left), text(right)) { + (Some(left), Some(right)) => left == right, + (None, None) => match (numeric(left), numeric(right)) { + (Some(left), Some(right)) => left == right, + _ => false, + }, + _ => false, + }, + } +} + +/// A boolean, integer or float key's numeric value, as Python compares keys. +fn numeric(node: &ConfigNode) -> Option { + match node { + ConfigNode::Bool(flag) => Some(f64::from(u8::from(*flag))), + ConfigNode::Int(_) | ConfigNode::Float(_) => node.number(), + _ => None, + } +} + +/// `mapping.get(key)` for a string key. +pub(crate) fn entry<'a, H: Clone>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + let key: ConfigNode = ConfigNode::Str(key.to_owned()); + entries + .iter() + .find(|(name, _)| same_key(name, &key)) + .map(|(_, value)| value) +} + +/// `mapping.get(key)` unless absent or `None`. +pub(crate) fn set_entry<'a, H: Clone>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + entry(entries, key).filter(|value| !matches!(value, ConfigNode::Null)) +} + +/// `entries` after `mapping[key] = value`: an equal key keeps its place and its key, else the entry is appended. +pub(crate) fn inserted( + mut entries: Vec<(ConfigNode, ConfigNode)>, + key: ConfigNode, + value: ConfigNode, +) -> Vec<(ConfigNode, ConfigNode)> { + match entries.iter().position(|(name, _)| same_key(name, &key)) { + Some(position) => entries[position].1 = value, + None => entries.push((key, value)), + } + entries +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs new file mode 100644 index 0000000000..a86c4d913f --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod authored; +pub(crate) mod entries; +pub(crate) mod reprs; +pub(crate) mod unions; diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs new file mode 100644 index 0000000000..d28ece997b --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs @@ -0,0 +1,44 @@ +//! Python's `str()` and `repr()` of config values. + +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::config_values::constants::{FALSE_TEXT, NONE_TEXT, TRUE_TEXT}; +use crate::config_values::models::ConfigNode; + +fn joined(items: &[ConfigNode]) -> String { + items.iter().map(python_repr).collect::>().join(", ") +} + +/// Python's `repr(value)`. +pub(crate) fn python_repr(value: &ConfigNode) -> String { + match value { + ConfigNode::Null => NONE_TEXT.to_owned(), + ConfigNode::Bool(true) => TRUE_TEXT.to_owned(), + ConfigNode::Bool(false) => FALSE_TEXT.to_owned(), + ConfigNode::Int(number) => number.to_string(), + ConfigNode::Float(number) => python_float_repr(*number), + ConfigNode::Str(text) => python_str_repr(text), + ConfigNode::List(items) => format!("[{}]", joined(items)), + ConfigNode::Tuple(items) if items.len() == 1 => format!("({},)", joined(items)), + ConfigNode::Tuple(items) => format!("({})", joined(items)), + ConfigNode::Map(entries) => format!( + "{{{}}}", + entries + .iter() + .map(|(key, item)| format!("{}: {}", python_repr(key), python_repr(item))) + .collect::>() + .join(", ") + ), + ConfigNode::Host(host) => host.facts.python_repr.clone(), + } +} + +/// Python's `str(value)`: a string's own text, an opaque value's `str()`, else its `repr()`. +pub(crate) fn python_str(value: &ConfigNode) -> String { + match value { + ConfigNode::Str(text) => text.clone(), + ConfigNode::Host(host) => host.facts.python_str.clone(), + _ => python_repr(value), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs new file mode 100644 index 0000000000..29fef8dbd7 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs @@ -0,0 +1,71 @@ +//! The unions `_merged_with_tag_union` makes of string sequences and row-diff tolerances. + +use crate::config_values::_helpers::entries::{inserted, set_entry}; +use crate::config_values::constants::AUDIT_OVERRIDE_SECTIONS; +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// `_as_string_list(value)`: `str()` of each list or tuple item. +fn string_items(value: &ConfigNode) -> Vec { + match value.kind() { + NodeKind::List | NodeKind::Tuple => { + value.items().iter().map(AuthoredNode::python_str).collect() + } + _ => Vec::new(), + } +} + +/// The strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. +pub(crate) fn merged_strings( + base: &ConfigNode, + overlay: &ConfigNode, +) -> Vec> { + let mut merged: Vec = string_items(base); + for item in string_items(overlay) { + if !merged.contains(&item) { + merged.push(item); + } + } + merged.into_iter().map(ConfigNode::Str).collect() +} + +/// Row-diff tolerances deep-merged by section, as `_merge_row_diff_tolerances_mapping`. +pub(crate) fn merged_tolerances( + base: &ConfigNode, + overlay: &ConfigNode, +) -> ConfigNode { + if base.kind() != NodeKind::Map || overlay.kind() != NodeKind::Map { + return overlay.clone(); + } + let (base, overlay) = (base.entries(), overlay.entries()); + let mut merged = base.clone(); + for section in AUDIT_OVERRIDE_SECTIONS { + let Some(overlay_section) = set_entry(&overlay, section) else { + continue; + }; + let value: ConfigNode = match set_entry(&base, section) { + Some(base_section) + if base_section.kind() == NodeKind::Map + && overlay_section.kind() == NodeKind::Map => + { + let mut values = base_section.entries(); + for (key, item) in overlay_section.entries() { + values = inserted(values, key, item); + } + ConfigNode::Map(values) + } + _ => overlay_section.clone(), + }; + merged = inserted(merged, ConfigNode::Str(section.to_owned()), value); + } + for (key, item) in &overlay { + if key.kind() == NodeKind::Str + && !AUDIT_OVERRIDE_SECTIONS + .iter() + .any(|section| key.is_text(section)) + { + merged = inserted(merged, key.clone(), item.clone()); + } + } + ConfigNode::Map(merged) +} diff --git a/crates/sqlbuild-model-config/src/config_values/constants.rs b/crates/sqlbuild-model-config/src/config_values/constants.rs new file mode 100644 index 0000000000..8dda7c39eb --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/constants.rs @@ -0,0 +1,10 @@ +//! Python's literal spellings of the plain config values. + +pub(crate) const NONE_TEXT: &str = "None"; +pub(crate) const TRUE_TEXT: &str = "True"; +pub(crate) const FALSE_TEXT: &str = "False"; +pub(crate) const TAGS_KEY: &str = "tags"; +pub(crate) const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; +pub(crate) const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; +/// The `row_diff_tolerances` sections merged one level deeper than the rest. +pub(crate) const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; diff --git a/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs new file mode 100644 index 0000000000..b971020c87 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs @@ -0,0 +1,13 @@ +//! One entry of a config mapping. + +use crate::config_values::_helpers::entries::entry; +use crate::config_values::models::ConfigNode; + +/// Python's `mapping.get(key)` for a string `key`. +#[must_use] +pub fn config_entry<'a, H: Clone>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + entry(entries, key) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs new file mode 100644 index 0000000000..bb83fd7ba0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs @@ -0,0 +1,53 @@ +//! One config layer merged over another, as `_merged_with_tag_union` merges them. + +use crate::config_values::_helpers::entries::{inserted, set_entry}; +use crate::config_values::_helpers::unions::{merged_strings, merged_tolerances}; +use crate::config_values::constants::{ + ROW_DIFF_EXCLUDE_COLUMNS_KEY, ROW_DIFF_TOLERANCES_KEY, TAGS_KEY, +}; +use crate::config_values::models::{ConfigNode, LayeredConfig}; +use crate::types::{AuthoredNode, NodeKind}; + +/// `overlay` merged over a copy of `base`, with tags, row-diff exclusions and tolerances unioned. +#[must_use] +pub fn layered_config( + base: &[(ConfigNode, ConfigNode)], + overlay: &[(ConfigNode, ConfigNode)], +) -> LayeredConfig { + let mut entries = base.to_vec(); + for (key, value) in overlay { + entries = inserted(entries, key.clone(), value.clone()); + } + let mut unioned_keys: Vec<&'static str> = Vec::new(); + for (key, tuple) in [(TAGS_KEY, false), (ROW_DIFF_EXCLUDE_COLUMNS_KEY, true)] { + if let (Some(base_items), Some(overlay_items)) = + (set_entry(base, key), set_entry(overlay, key)) + { + let items: Vec> = merged_strings(base_items, overlay_items); + let merged: ConfigNode = if tuple { + ConfigNode::Tuple(items) + } else { + ConfigNode::List(items) + }; + entries = inserted(entries, ConfigNode::Str(key.to_owned()), merged); + unioned_keys.push(key); + } + } + if let (Some(base_tolerances), Some(overlay_tolerances)) = ( + set_entry(base, ROW_DIFF_TOLERANCES_KEY), + set_entry(overlay, ROW_DIFF_TOLERANCES_KEY), + ) { + entries = inserted( + entries, + ConfigNode::Str(ROW_DIFF_TOLERANCES_KEY.to_owned()), + merged_tolerances(base_tolerances, overlay_tolerances), + ); + if base_tolerances.kind() == NodeKind::Map && overlay_tolerances.kind() == NodeKind::Map { + unioned_keys.push(ROW_DIFF_TOLERANCES_KEY); + } + } + LayeredConfig { + entries, + unioned_keys, + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/mod.rs b/crates/sqlbuild-model-config/src/config_values/main/mod.rs new file mode 100644 index 0000000000..1ed5f0b2af --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/mod.rs @@ -0,0 +1,2 @@ +pub mod config_entry; +pub mod layered_config; diff --git a/crates/sqlbuild-model-config/src/config_values/mod.rs b/crates/sqlbuild-model-config/src/config_values/mod.rs new file mode 100644 index 0000000000..4d2841b6a6 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/mod.rs @@ -0,0 +1,10 @@ +//! Model config values as Rust data: plain Python builtins typed, every other value an opaque host value. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/config_values/models.rs b/crates/sqlbuild-model-config/src/config_values/models.rs new file mode 100644 index 0000000000..acf20575de --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/models.rs @@ -0,0 +1,64 @@ +//! One model config value. + +use crate::types::NodeKind; + +/// A config value: the builtin Python types it is made of, or a value only its host understands. +#[derive(Clone, Debug, PartialEq)] +pub enum ConfigNode { + Null, + Bool(bool), + /// An `int` within `i64`. + Int(i64), + Float(f64), + /// A `str` that is valid UTF-8. + Str(String), + List(Vec), + Tuple(Vec), + /// A `dict`'s entries in insertion order. + Map(Vec<(Self, Self)>), + /// Any other value, carried by its host handle and read through the facts taken from it. + Host(HostValue), +} + +/// An opaque value: its host handle, what it reads as, and its items when it is a container. +#[derive(Clone, Debug, PartialEq)] +pub struct HostValue { + pub handle: H, + pub facts: HostFacts, + pub children: HostChildren, +} + +/// What config logic may read of an opaque value, read once when the value was converted. +#[derive(Clone, Debug, PartialEq)] +pub struct HostFacts { + pub kind: NodeKind, + /// The text of a string value, with lone surrogates read as U+FFFD. + pub text: Option, + /// Whether `text` is the value's exact text. + pub strict_text: bool, + pub integer: Option, + pub number: Option, + pub python_str: String, + /// Whether `python_str` is the value's exact `str()`. + pub strict_str: bool, + pub python_repr: String, + pub date_like: bool, + /// Whether the value is a typed hook entry. + pub hook_entry: bool, +} + +/// The converted items of an opaque list, tuple or mapping, which config logic walks. +#[derive(Clone, Debug, PartialEq)] +pub enum HostChildren { + None, + Items(Vec>), + Entries(Vec<(ConfigNode, ConfigNode)>), +} + +/// One layer merged over another, and which keys a union replaced rather than overrode. +#[derive(Clone, Debug, PartialEq)] +pub struct LayeredConfig { + pub entries: Vec<(ConfigNode, ConfigNode)>, + /// The keys both layers set whose merged value is a union of the two. + pub unioned_keys: Vec<&'static str>, +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests.rs b/crates/sqlbuild-model-config/src/config_values/tests.rs new file mode 100644 index 0000000000..b3a5a4a454 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/test_config_entry.rs"] +mod config_entry; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_layered_config.rs"] +mod layered_config; +#[path = "tests/test_python_text.rs"] +mod python_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs new file mode 100644 index 0000000000..b32589f495 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs @@ -0,0 +1,33 @@ +use crate::config_values::models::{ConfigNode, HostChildren, HostFacts, HostValue}; +use crate::config_values::tests::test_types::Node; +use crate::types::NodeKind; + +/// A string node. +pub(super) fn text(value: &str) -> Node { + ConfigNode::Str(value.to_owned()) +} + +/// A `StrEnum`-like host value whose text is `value`. +pub(super) fn host_text(value: &str) -> Node { + ConfigNode::Host(HostValue { + handle: 1, + facts: HostFacts { + kind: NodeKind::Str, + text: Some(value.to_owned()), + strict_text: true, + integer: None, + number: None, + python_str: value.to_owned(), + strict_str: true, + python_repr: format!(""), + date_like: false, + hook_entry: false, + }, + children: HostChildren::None, + }) +} + +/// A list of string nodes. +pub(super) fn texts(values: &[&str]) -> Node { + ConfigNode::List(values.iter().map(|value| text(value)).collect()) +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs new file mode 100644 index 0000000000..3b9239df64 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_config_entry.rs @@ -0,0 +1,36 @@ +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{host_text, text}; +use crate::config_values::tests::test_types::ConfigEntryTestCase; + +#[test] +fn given_config_entries_when_read_by_string_key_then_python_dict_equality_applies() { + let test_cases = [ + ConfigEntryTestCase { + description: "a string key is found", + entries: vec![(text("schema"), text("analytics"))], + key: "schema", + expected_value: Some(text("analytics")), + }, + ConfigEntryTestCase { + description: "a str-enum key equals its text", + entries: vec![(host_text("schema"), ConfigNode::Int(1))], + key: "schema", + expected_value: Some(ConfigNode::Int(1)), + }, + ConfigEntryTestCase { + description: "a number key never equals a string", + entries: vec![(ConfigNode::Int(1), text("one"))], + key: "1", + expected_value: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + config_entry(&test_case.entries, test_case.key).cloned(), + test_case.expected_value, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs new file mode 100644 index 0000000000..640030f654 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs @@ -0,0 +1,102 @@ +use crate::config_values::main::layered_config::layered_config; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{text, texts}; +use crate::config_values::tests::test_types::LayeredConfigTestCase; + +#[test] +fn given_two_layers_when_merged_then_overrides_keep_order_and_unions_merge() { + let test_cases = [ + LayeredConfigTestCase { + description: "an override keeps the key's place; a new key is appended", + base: vec![ + (text("materialized"), text("view")), + (text("schema"), text("raw")), + ], + overlay: vec![ + (text("alias"), text("orders")), + (text("materialized"), text("table")), + ], + expected_entries: vec![ + (text("materialized"), text("table")), + (text("schema"), text("raw")), + (text("alias"), text("orders")), + ], + expected_unioned_keys: vec![], + }, + LayeredConfigTestCase { + description: "tags and row-diff exclusions union as list and tuple", + base: vec![ + (text("tags"), texts(&["daily", "core"])), + ( + text("row_diff_exclude_columns"), + ConfigNode::Tuple(vec![text("updated_at")]), + ), + ], + overlay: vec![ + (text("tags"), texts(&["core", "finance"])), + (text("row_diff_exclude_columns"), texts(&["loaded_at"])), + ], + expected_entries: vec![ + (text("tags"), texts(&["daily", "core", "finance"])), + ( + text("row_diff_exclude_columns"), + ConfigNode::Tuple(vec![text("updated_at"), text("loaded_at")]), + ), + ], + expected_unioned_keys: vec!["tags", "row_diff_exclude_columns"], + }, + LayeredConfigTestCase { + description: "a null overlay tag list overrides without a union", + base: vec![(text("tags"), texts(&["daily"]))], + overlay: vec![(text("tags"), ConfigNode::Null)], + expected_entries: vec![(text("tags"), ConfigNode::Null)], + expected_unioned_keys: vec![], + }, + LayeredConfigTestCase { + description: "tolerance sections merge one level deeper", + base: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![(text("float"), ConfigNode::Float(0.1))]), + ), + (text("default"), ConfigNode::Int(0)), + ]), + )], + overlay: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![(text("decimal"), ConfigNode::Int(1))]), + ), + (text("default"), ConfigNode::Int(2)), + ]), + )], + expected_entries: vec![( + text("row_diff_tolerances"), + ConfigNode::Map(vec![ + ( + text("by_type"), + ConfigNode::Map(vec![ + (text("float"), ConfigNode::Float(0.1)), + (text("decimal"), ConfigNode::Int(1)), + ]), + ), + (text("default"), ConfigNode::Int(2)), + ]), + )], + expected_unioned_keys: vec!["row_diff_tolerances"], + }, + ]; + for test_case in test_cases { + let layered = layered_config(&test_case.base, &test_case.overlay); + assert_eq!( + (layered.entries, layered.unioned_keys), + (test_case.expected_entries, test_case.expected_unioned_keys), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs new file mode 100644 index 0000000000..f2a1bad7bb --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs @@ -0,0 +1,59 @@ +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{host_text, text}; +use crate::config_values::tests::test_types::PythonTextTestCase; +use crate::types::AuthoredNode; + +#[test] +fn given_config_values_when_printed_then_str_and_repr_match_python() { + let test_cases = [ + PythonTextTestCase { + description: "scalars print as Python literals", + value: ConfigNode::Tuple(vec![ + ConfigNode::Null, + ConfigNode::Bool(true), + ConfigNode::Int(-3), + ConfigNode::Float(1e16), + ]), + expected_str: "(None, True, -3, 1e+16)", + expected_repr: "(None, True, -3, 1e+16)", + }, + PythonTextTestCase { + description: "a string prints itself and reprs quoted", + value: text("it's"), + expected_str: "it's", + expected_repr: "\"it's\"", + }, + PythonTextTestCase { + description: "a one-item tuple keeps its comma", + value: ConfigNode::Tuple(vec![ConfigNode::Float(1.0)]), + expected_str: "(1.0,)", + expected_repr: "(1.0,)", + }, + PythonTextTestCase { + description: "mappings and lists nest item reprs, host values their own", + value: ConfigNode::Map(vec![( + text("severity"), + ConfigNode::List(vec![host_text("warn"), text("x")]), + )]), + expected_str: "{'severity': [, 'x']}", + expected_repr: "{'severity': [, 'x']}", + }, + PythonTextTestCase { + description: "a host value prints its own str", + value: host_text("warn"), + expected_str: "warn", + expected_repr: "", + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.value.python_str(), test_case.value.python_repr()), + ( + test_case.expected_str.to_owned(), + test_case.expected_repr.to_owned() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs new file mode 100644 index 0000000000..22914ef174 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs @@ -0,0 +1,26 @@ +use crate::config_values::models::ConfigNode; + +/// A config node in tests; host handles are plain numbers. +pub(super) type Node = ConfigNode; + +pub(super) struct PythonTextTestCase { + pub(super) description: &'static str, + pub(super) value: Node, + pub(super) expected_str: &'static str, + pub(super) expected_repr: &'static str, +} + +pub(super) struct LayeredConfigTestCase { + pub(super) description: &'static str, + pub(super) base: Vec<(Node, Node)>, + pub(super) overlay: Vec<(Node, Node)>, + pub(super) expected_entries: Vec<(Node, Node)>, + pub(super) expected_unioned_keys: Vec<&'static str>, +} + +pub(super) struct ConfigEntryTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec<(Node, Node)>, + pub(super) key: &'static str, + pub(super) expected_value: Option, +} diff --git a/crates/sqlbuild-model-config/src/config_values/types.rs b/crates/sqlbuild-model-config/src/config_values/types.rs new file mode 100644 index 0000000000..cb1f94fbfb --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/types.rs @@ -0,0 +1,6 @@ +//! A config mapping's entries. + +use crate::config_values::models::ConfigNode; + +/// A mapping's `(key, value)` entries in insertion order, as a Python `dict` keeps them. +pub type ConfigEntries = Vec<(ConfigNode, ConfigNode)>; diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs index 130e9d67fa..397fce3c0e 100644 --- a/crates/sqlbuild-model-config/src/lib.rs +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub mod config_presence; +pub mod config_values; pub mod declarations; pub mod errors; pub mod execution_limits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs new file mode 100644 index 0000000000..aab90f802b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs @@ -0,0 +1,238 @@ +//! Python config values converted once into `ConfigNode`s, and projected back into Python objects. + +use std::sync::Arc; + +use pyo3::IntoPyObject; +use pyo3::exceptions::{PyException, PyOverflowError, PyUnicodeEncodeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, PyTupleMethods, +}; +use sqlbuild_model_config::config_values::models::{ + ConfigNode, HostChildren, HostFacts, HostValue, +}; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::records::construct::record_class; +use crate::bindings::_helpers::records::ids::RecordId; + +/// The Python object an opaque config value stands for. +pub(crate) type HostHandle = Arc>; +/// A config value converted from Python. +pub(crate) type PyConfigNode = ConfigNode; +/// A converted config mapping's entries. +pub(crate) type PyConfigEntries = Vec<(PyConfigNode, PyConfigNode)>; + +/// The typed hook entry classes, which the config build checks hook lists against. +const HOOK_ENTRY_CLASSES: [RecordId; 3] = [ + RecordId::SqlHookEntry, + RecordId::NamedSqlHookEntry, + RecordId::PythonHookEntry, +]; + +fn hook_entry(value: &Bound<'_, PyAny>) -> PyResult { + for class in HOOK_ENTRY_CLASSES { + if value.is_instance(&record_class(value.py(), class)?)? { + return Ok(true); + } + } + Ok(false) +} + +/// Whether a Python `str` encodes to UTF-8, which it does unless it holds a lone surrogate. +fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(_) => Ok(true), + Err(error) if error.is_instance_of::(text.py()) => Ok(false), + Err(error) => Err(error), + } +} + +/// Whether `str(value)` succeeds with UTF-8 text; any error `str()` raises makes it inexact. +fn exact_str(value: &Bound<'_, PyAny>) -> PyResult { + match value.str() { + Ok(text) => utf8_text(&text), + Err(error) if error.is_instance_of::(value.py()) => Ok(false), + Err(error) => Err(error), + } +} + +/// An opaque value with the facts `PyNode` reads of it, and its converted items. +fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResult { + let node: PyNode<'_> = PyNode(value.clone()); + let strict_text: bool = match value.downcast::() { + Ok(text) => utf8_text(text)?, + Err(_) => false, + }; + Ok(ConfigNode::Host(HostValue { + handle: Arc::new(value.clone().unbind()), + facts: HostFacts { + kind: node.kind(), + text: node.text(), + strict_text, + integer: node.integer(), + number: node.number(), + python_str: node.python_str(), + strict_str: exact_str(value)?, + python_repr: node.python_repr(), + date_like: node.is_date_like(), + hook_entry: hook_entry(value)?, + }, + children, + })) +} + +/// Converted items, or `None` when a container holds itself; `open` holds the enclosing containers. +fn converted_items<'py>( + items: impl Iterator>, + open: &[usize], +) -> PyResult>> { + let mut converted: Vec = Vec::new(); + for item in items { + match convert(&item, open)? { + Some(node) => converted.push(node), + None => return Ok(None), + } + } + Ok(Some(converted)) +} + +fn converted_entries( + mapping: &Bound<'_, PyDict>, + open: &[usize], +) -> PyResult> { + let mut converted: PyConfigEntries = Vec::with_capacity(mapping.len()); + for (key, item) in mapping.iter() { + let (Some(key), Some(item)) = (convert(&key, open)?, convert(&item, open)?) else { + return Ok(None); + }; + converted.push((key, item)); + } + Ok(Some(converted)) +} + +/// A container's converted children, or `None` when it holds itself. +fn container( + value: &Bound<'_, PyAny>, + enclosing: &[usize], +) -> PyResult>> { + let address: usize = value.as_ptr() as usize; + if enclosing.contains(&address) { + return Ok(None); + } + let open: Vec = [enclosing, &[address]].concat(); + Ok(if let Ok(mapping) = value.downcast::() { + converted_entries(mapping, &open)?.map(HostChildren::Entries) + } else if let Ok(items) = value.downcast::() { + converted_items(items.iter(), &open)?.map(HostChildren::Items) + } else if let Ok(items) = value.downcast::() { + converted_items(items.iter(), &open)?.map(HostChildren::Items) + } else { + Some(HostChildren::None) + }) +} + +fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { + if value.is_none() { + return Ok(Some(ConfigNode::Null)); + } + if value.is_exact_instance_of::() { + return Ok(Some(ConfigNode::Bool(value.is_truthy()?))); + } + if value.is_exact_instance_of::() { + match value.extract::() { + Ok(number) => return Ok(Some(ConfigNode::Int(number))), + Err(error) if error.is_instance_of::(value.py()) => {} + Err(error) => return Err(error), + } + } + if value.is_exact_instance_of::() { + return Ok(Some(ConfigNode::Float(value.extract::()?))); + } + if let Ok(text) = value.downcast_exact::() + && utf8_text(text)? + { + return Ok(Some(ConfigNode::Str(text.to_str()?.to_owned()))); + } + let Some(children) = container(value, open)? else { + return Ok(None); + }; + let exact: bool = value.is_exact_instance_of::() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::(); + Ok(Some(match children { + HostChildren::Entries(entries) if exact => ConfigNode::Map(entries), + HostChildren::Items(items) if exact && value.is_exact_instance_of::() => { + ConfigNode::List(items) + } + HostChildren::Items(items) if exact => ConfigNode::Tuple(items), + children => host(value, children)?, + })) +} + +/// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. +pub(crate) fn config_node(value: &Bound<'_, PyAny>) -> PyResult { + match convert(value, &[])? { + Some(node) => Ok(node), + None => host(value, HostChildren::None), + } +} + +/// A Python mapping's entries as config entries. +pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult { + mapping + .iter() + .map(|(key, value)| Ok((config_node(&key)?, config_node(&value)?))) + .collect() +} + +/// The Python object of one config value: new builtins, and opaque values as themselves. +pub(crate) fn config_object<'py>( + py: Python<'py>, + node: &PyConfigNode, +) -> PyResult> { + Ok(match node { + ConfigNode::Null => py.None().into_bound(py), + ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), + ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), + ConfigNode::Str(text) => PyString::new(py, text).into_any(), + ConfigNode::List(items) => PyList::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Tuple(items) => PyTuple::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), + ConfigNode::Host(host) => host.handle.bind(py).clone(), + }) +} + +/// A new Python `dict` of config entries. +pub(crate) fn config_dict<'py>( + py: Python<'py>, + entries: &[(PyConfigNode, PyConfigNode)], +) -> PyResult> { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + mapping.set_item(config_object(py, key)?, config_object(py, value)?)?; + } + Ok(mapping) +} + +/// Whether `node` kind is a Python `str` instance, as `isinstance(value, str)` reads it. +pub(crate) fn is_text_node(node: &PyConfigNode) -> bool { + node.kind() == NodeKind::Str +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs index 3dcabf33db..9cd4868913 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -1,13 +1,20 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. +use std::collections::HashMap; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::config_values::main::config_entry::config_entry; +use sqlbuild_model_config::config_values::main::layered_config::layered_config; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_model_config::types::AuthoredNode; -use crate::bindings::_helpers::model_config::model_config_builder::{ - layered_values, raised_config_error, +use crate::bindings::_helpers::model_config::config_nodes::{ + PyConfigEntries, PyConfigNode, config_entries, config_node, config_object, is_text_node, }; +use crate::bindings::_helpers::model_config::model_config_builder::raised_config_error; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; use crate::bindings::_helpers::records::fields::field as native_field; @@ -15,11 +22,11 @@ const DESCRIPTION_KEY: &str = "description"; const MODEL_SCHEMA_KEY: &str = "model_schema"; /// Whether `value` is a string with non-whitespace text. -fn non_empty_text(value: Option>) -> PyResult { - let Some(value) = value.filter(|value| value.is_instance_of::()) else { - return Ok(false); - }; - value.call_method0("strip")?.is_truthy() +fn non_empty_text(value: Option<&PyConfigNode>) -> bool { + value + .filter(|value| is_text_node(value)) + .and_then(|value| value.with_text(|text| !python_strip(text).is_empty())) + .unwrap_or(false) } /// The file path of each model whose layered config or model schema gives it a description. @@ -29,8 +36,10 @@ pub(crate) fn externally_described_models<'py>( ) -> PyResult>> { let py = inputs.py(); let project_config: Bound<'py, PyAny> = native_field(&inputs, "project_config")?; - let defaults: Bound<'py, PyDict> = - project_defaults_mapping(&native_field(&project_config, "defaults")?)?; + let defaults: PyConfigEntries = config_entries(&project_defaults_mapping(&native_field( + &project_config, + "defaults", + )?)?)?; let path_defaults: Bound<'py, PyDict> = native_field(&project_config, "path_defaults")?.downcast_into()?; let path_keys: Vec = path_defaults @@ -38,34 +47,41 @@ pub(crate) fn externally_described_models<'py>( .iter() .map(|key| key.extract::()) .collect::>()?; + let mut layers: HashMap = HashMap::new(); let mut described: Vec> = Vec::new(); for model_file in native_field(&inputs, "model_files")?.try_iter()? { let model_file: Bound<'py, PyAny> = model_file?; let relative_path: String = native_field(&model_file, "relative_path")? .str()? .extract()?; - let mut values: Bound<'py, PyDict> = defaults.copy()?; + let mut values: PyConfigEntries = defaults.clone(); match select_path_default(&relative_path, &path_keys) { PathDefaultChoice::Selected(Some(key)) => { - let layer: Bound<'py, PyDict> = - path_defaults.as_any().get_item(&key)?.downcast_into()?; - values = layered_values(py, &values, &layer)?; + if !layers.contains_key(&key) { + let layer: Bound<'py, PyDict> = + path_defaults.as_any().get_item(&key)?.downcast_into()?; + layers.insert(key.clone(), config_entries(&layer)?); + } + values = layered_config(&values, &layers[&key]).entries; } PathDefaultChoice::Selected(None) => {} PathDefaultChoice::Conflict(error) => return Err(raised_config_error(py, error)), } let header: Bound<'py, PyDict> = native_field(&model_file, "header_values")?.downcast_into()?; - values = layered_values(py, &values, &header)?; - let schema_description: Option> = - match values.get_item(MODEL_SCHEMA_KEY)? { - Some(name) if name.is_instance_of::() => model_schemas - .get_item(name)? - .map(|schema| schema.getattr(DESCRIPTION_KEY)) - .transpose()?, - _ => None, - }; - if non_empty_text(values.get_item(DESCRIPTION_KEY)?)? || non_empty_text(schema_description)? + values = layered_config(&values, &config_entries(&header)?).entries; + let schema_description: Option = match config_entry(&values, MODEL_SCHEMA_KEY) + { + Some(name) if is_text_node(name) => model_schemas + .get_item(config_object(py, name)?)? + .map(|schema| schema.getattr(DESCRIPTION_KEY)) + .transpose()? + .map(|description| config_node(&description)) + .transpose()?, + _ => None, + }; + if non_empty_text(config_entry(&values, DESCRIPTION_KEY)) + || non_empty_text(schema_description.as_ref()) { described.push(native_field(&model_file, "file_path")?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index e428f56314..9f9c2b0494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod authored_nodes; pub(crate) mod column_migrations; pub(crate) mod config_classes; pub(crate) mod config_errors; +pub(crate) mod config_nodes; pub(crate) mod config_templates; pub(crate) mod config_values; pub(crate) mod contract_classes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 6846442d16..dfb4a278ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -575,18 +575,6 @@ impl<'py> Expansion<'_, 'py> { } } -/// `merged_layer`, with a config error raised as the Python stage raises it. -pub(crate) fn layered_values<'py>( - py: Python<'py>, - base: &Bound<'py, PyDict>, - overlay: &Bound<'py, PyDict>, -) -> PyResult> { - merged_layer(py, base, overlay).map_err(|halt| match halt { - Halt::Error(error) => error, - Halt::Config(error) => raised_config_error(py, error), - }) -} - /// The Python exception a native config error raises as. pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { let raised = || -> PyResult { diff --git a/fensu.toml b/fensu.toml index ecb8b05d64..9cd1674a94 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1942,7 +1942,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", From 96c6ddc06752a077943de155e4e728183d05d9ce Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:01:00 +0100 Subject: [PATCH 4/9] refactor(native): build model configs as typed values in sqlbuild-model-config The model config build (layering with tag, row-diff and tolerance unions, hook checks, the three model template passes, the target namespace and its templates, storage key removal, the no-macros check and header keys) now runs in `sqlbuild-model-config::model_config_build` over `ConfigNode`s. It asks its caller only for environment reads, the project-var text of non-string values and the exact error of a value that is not UTF-8 text, through `ConfigBuildHost`. `NativeModelConfigBuilder` converts the project inputs once per compile and each header once per model, and projects the built values back. Opaque values are carried as their own Python objects. Which header entries are still the header's own is tracked by provenance instead of object identity, so native header facts apply exactly where they did. Output is unchanged: the compiler differential passes, and a new integration test pins values, types, order, header keys and errors for builtin, subclassed, str-enum, date and opaque header values against the previous build. The build runs about 2% fewer instructions on a 600-model templated project. --- .../src/config_values/_helpers/authored.rs | 114 ++- .../src/config_values/_helpers/entries.rs | 21 +- .../src/config_values/main/inserted_entry.rs | 15 + .../src/config_values/main/mod.rs | 3 + .../src/config_values/main/same_config_key.rs | 10 + .../config_values/main/set_config_entry.rs | 13 + .../src/config_values/models.rs | 2 +- .../src/config_values/tests/helpers.rs | 4 +- crates/sqlbuild-model-config/src/lib.rs | 1 + .../src/model_config_build/_helpers/checks.rs | 111 +++ .../model_config_build/_helpers/context.rs | 83 ++ .../model_config_build/_helpers/expansion.rs | 255 +++++ .../src/model_config_build/_helpers/layers.rs | 79 ++ .../src/model_config_build/_helpers/mod.rs | 7 + .../model_config_build/_helpers/namespace.rs | 91 ++ .../src/model_config_build/_helpers/phases.rs | 203 ++++ .../model_config_build/_helpers/storage.rs | 56 ++ .../src/model_config_build/constants.rs | 31 + .../main/build_model_config.rs | 37 + .../main/inherited_config.rs | 19 + .../src/model_config_build/main/mod.rs | 3 + .../main/model_context_values.rs | 17 + .../src/model_config_build/mod.rs | 10 + .../src/model_config_build/models.rs | 63 ++ .../src/model_config_build/tests.rs | 6 + .../src/model_config_build/tests/helpers.rs | 41 + .../tests/test_build_model_config.rs | 91 ++ .../model_config_build/tests/test_types.rs | 46 + .../src/model_config_build/types.rs | 31 + .../_helpers/model_config/config_nodes.rs | 4 +- .../model_config/model_config_builder.rs | 891 ++++-------------- fensu.toml | 2 +- .../compiler/model_config/_test_types.py | 29 + .../sqlbuild/compiler/model_config/helpers.py | 104 +- .../model_config/test_config_values.py | 213 +++++ 35 files changed, 1958 insertions(+), 748 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/constants.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/main/mod.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/mod.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/models.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/tests.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/model_config_build/types.rs create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs index 665252c953..7c558ad618 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs @@ -4,37 +4,37 @@ use crate::config_values::_helpers::reprs::{python_repr, python_str}; use crate::config_values::models::{ConfigNode, HostChildren}; use crate::types::{AuthoredNode, NodeKind}; -impl AuthoredNode for ConfigNode { +impl<'a, H: Clone> AuthoredNode for &'a ConfigNode { fn kind(&self) -> NodeKind { - match self { - Self::Null => NodeKind::Null, - Self::Bool(flag) => NodeKind::Bool(*flag), - Self::Int(number) => NodeKind::Int { + match *self { + ConfigNode::Null => NodeKind::Null, + ConfigNode::Bool(flag) => NodeKind::Bool(*flag), + ConfigNode::Int(number) => NodeKind::Int { negative: *number < 0, }, - Self::Float(_) => NodeKind::Other, - Self::Str(_) => NodeKind::Str, - Self::List(_) => NodeKind::List, - Self::Tuple(_) => NodeKind::Tuple, - Self::Map(_) => NodeKind::Map, - Self::Host(host) => host.facts.kind, + ConfigNode::Float(_) => NodeKind::Other, + ConfigNode::Str(_) => NodeKind::Str, + ConfigNode::List(_) => NodeKind::List, + ConfigNode::Tuple(_) => NodeKind::Tuple, + ConfigNode::Map(_) => NodeKind::Map, + ConfigNode::Host(host) => host.facts.kind, } } fn integer(&self) -> Option { - match self { - Self::Int(number) => Some(*number), - Self::Host(host) => host.facts.integer, + match *self { + ConfigNode::Int(number) => Some(*number), + ConfigNode::Host(host) => host.facts.integer, _ => None, } } /// Python's `float(int)`, which rounds the decimal value to the nearest float, as parsing does. fn number(&self) -> Option { - match self { - Self::Int(number) => Some(*number as f64), - Self::Float(number) => Some(*number), - Self::Host(host) => host.facts.number, + match *self { + ConfigNode::Int(number) => Some(*number as f64), + ConfigNode::Float(number) => Some(*number), + ConfigNode::Host(host) => host.facts.number, _ => None, } } @@ -48,9 +48,9 @@ impl AuthoredNode for ConfigNode { } fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { - match self { - Self::Str(text) => Some(read(text)), - Self::Host(host) => host.facts.text.as_deref().map(read), + match *self { + ConfigNode::Str(text) => Some(read(text)), + ConfigNode::Host(host) => host.facts.text.as_deref().map(read), _ => None, } } @@ -64,14 +64,15 @@ impl AuthoredNode for ConfigNode { } fn is_date_like(&self) -> bool { - matches!(self, Self::Host(host) if host.facts.date_like) + matches!(self, ConfigNode::Host(host) if host.facts.date_like) } fn items(&self) -> Vec { - match self { - Self::List(items) | Self::Tuple(items) => items.clone(), - Self::Host(host) => match &host.children { - HostChildren::Items(items) => items.clone(), + let node: &'a ConfigNode = self; + match node { + ConfigNode::List(items) | ConfigNode::Tuple(items) => items.iter().collect(), + ConfigNode::Host(host) => match &host.children { + HostChildren::Items(items) => items.iter().collect(), _ => Vec::new(), }, _ => Vec::new(), @@ -79,13 +80,66 @@ impl AuthoredNode for ConfigNode { } fn entries(&self) -> Vec<(Self, Self)> { - match self { - Self::Map(entries) => entries.clone(), - Self::Host(host) => match &host.children { - HostChildren::Entries(entries) => entries.clone(), + let node: &'a ConfigNode = self; + match node { + ConfigNode::Map(entries) => entries.iter().map(|(key, value)| (key, value)).collect(), + ConfigNode::Host(host) => match &host.children { + HostChildren::Entries(entries) => { + entries.iter().map(|(key, value)| (key, value)).collect() + } _ => Vec::new(), }, _ => Vec::new(), } } } + +impl AuthoredNode for ConfigNode { + fn kind(&self) -> NodeKind { + (&self).kind() + } + + fn integer(&self) -> Option { + (&self).integer() + } + + fn number(&self) -> Option { + (&self).number() + } + + fn text(&self) -> Option { + (&self).text() + } + + fn is_text(&self, text: &str) -> bool { + (&self).is_text(text) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + (&self).with_text(read) + } + + fn python_str(&self) -> String { + python_str(self) + } + + fn python_repr(&self) -> String { + python_repr(self) + } + + fn is_date_like(&self) -> bool { + (&self).is_date_like() + } + + fn items(&self) -> Vec { + (&self).items().into_iter().cloned().collect() + } + + fn entries(&self) -> Vec<(Self, Self)> { + (&self) + .entries() + .into_iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs index 2b04f00cc2..7fb9e76757 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs @@ -3,21 +3,25 @@ use crate::config_values::models::ConfigNode; use crate::types::{AuthoredNode, NodeKind}; -/// Python's `==` between two dict keys: equal text, equal numbers, or equal tuples. -pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { - let text = |node: &ConfigNode| match node { - ConfigNode::Str(text) => Some(text.as_str().to_owned()), +/// The exact text of a string key, borrowed. +fn key_text(node: &ConfigNode) -> Option<&str> { + match node { + ConfigNode::Str(text) => Some(text), ConfigNode::Host(host) if host.facts.kind == NodeKind::Str && host.facts.strict_text => { - host.facts.text.clone() + host.facts.text.as_deref() } _ => None, - }; + } +} + +/// Python's `==` between two dict keys: equal text, equal numbers, or equal tuples. +pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { match (left, right) { (ConfigNode::Null, ConfigNode::Null) => true, (ConfigNode::Tuple(left), ConfigNode::Tuple(right)) => { left.len() == right.len() && left.iter().zip(right).all(|(a, b)| same_key(a, b)) } - _ => match (text(left), text(right)) { + _ => match (key_text(left), key_text(right)) { (Some(left), Some(right)) => left == right, (None, None) => match (numeric(left), numeric(right)) { (Some(left), Some(right)) => left == right, @@ -42,10 +46,9 @@ pub(crate) fn entry<'a, H: Clone>( entries: &'a [(ConfigNode, ConfigNode)], key: &str, ) -> Option<&'a ConfigNode> { - let key: ConfigNode = ConfigNode::Str(key.to_owned()); entries .iter() - .find(|(name, _)| same_key(name, &key)) + .find(|(name, _)| key_text(name) == Some(key)) .map(|(_, value)| value) } diff --git a/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs new file mode 100644 index 0000000000..5c58750d58 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs @@ -0,0 +1,15 @@ +//! A config mapping with one entry set. + +use crate::config_values::_helpers::entries::inserted; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; + +/// `entries` after `mapping[key] = value`: an equal key keeps its place, else the entry is appended. +#[must_use] +pub fn inserted_entry( + entries: ConfigEntries, + key: ConfigNode, + value: ConfigNode, +) -> ConfigEntries { + inserted(entries, key, value) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/mod.rs b/crates/sqlbuild-model-config/src/config_values/main/mod.rs index 1ed5f0b2af..e4d117250c 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/mod.rs @@ -1,2 +1,5 @@ pub mod config_entry; +pub mod inserted_entry; pub mod layered_config; +pub mod same_config_key; +pub mod set_config_entry; diff --git a/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs new file mode 100644 index 0000000000..38cab0227a --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs @@ -0,0 +1,10 @@ +//! Python's equality between two config mapping keys. + +use crate::config_values::_helpers::entries::same_key; +use crate::config_values::models::ConfigNode; + +/// Whether two dict keys are equal: equal text, equal numbers, or equal tuples. +#[must_use] +pub fn same_config_key(left: &ConfigNode, right: &ConfigNode) -> bool { + same_key(left, right) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs new file mode 100644 index 0000000000..89b0e75ec0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs @@ -0,0 +1,13 @@ +//! One set entry of a config mapping. + +use crate::config_values::_helpers::entries::set_entry; +use crate::config_values::models::ConfigNode; + +/// Python's `mapping.get(key)` for a string `key`, unless it is absent or `None`. +#[must_use] +pub fn set_config_entry<'a, H: Clone>( + entries: &'a [(ConfigNode, ConfigNode)], + key: &str, +) -> Option<&'a ConfigNode> { + set_entry(entries, key) +} diff --git a/crates/sqlbuild-model-config/src/config_values/models.rs b/crates/sqlbuild-model-config/src/config_values/models.rs index acf20575de..b0e9e6f3bb 100644 --- a/crates/sqlbuild-model-config/src/config_values/models.rs +++ b/crates/sqlbuild-model-config/src/config_values/models.rs @@ -17,7 +17,7 @@ pub enum ConfigNode { /// A `dict`'s entries in insertion order. Map(Vec<(Self, Self)>), /// Any other value, carried by its host handle and read through the facts taken from it. - Host(HostValue), + Host(Box>), } /// An opaque value: its host handle, what it reads as, and its items when it is a container. diff --git a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs index b32589f495..c5ecb1a8df 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs @@ -9,7 +9,7 @@ pub(super) fn text(value: &str) -> Node { /// A `StrEnum`-like host value whose text is `value`. pub(super) fn host_text(value: &str) -> Node { - ConfigNode::Host(HostValue { + ConfigNode::Host(Box::new(HostValue { handle: 1, facts: HostFacts { kind: NodeKind::Str, @@ -24,7 +24,7 @@ pub(super) fn host_text(value: &str) -> Node { hook_entry: false, }, children: HostChildren::None, - }) + })) } /// A list of string nodes. diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs index 397fce3c0e..c991f8c080 100644 --- a/crates/sqlbuild-model-config/src/lib.rs +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -8,6 +8,7 @@ pub mod declarations; pub mod errors; pub mod execution_limits; pub mod header_metadata; +pub mod model_config_build; pub mod model_validation; pub mod name_patterns; pub mod path_defaults; diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs new file mode 100644 index 0000000000..788508070e --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs @@ -0,0 +1,111 @@ +//! The header, hook and macro checks of one model's config build. + +use crate::config_presence::main::entry_macro_path::entry_macro_path; +use crate::config_presence::models::MacroPath; +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::set_config_entry::set_config_entry; +use crate::config_values::models::ConfigNode; +use crate::errors::ConfigError; +use crate::model_config_build::constants::{HOOK_KEYS, SORTED_HOOK_KEYS, TAGS_KEY}; +use crate::model_config_build::models::HeaderEntryFacts; +use crate::types::{AuthoredNode, NodeKind}; + +/// The `CompileInputError` `model '' `. +pub(crate) fn model_error(model_name: &str, text: &str) -> ConfigError { + ConfigError::compile(format!("model '{model_name}' {text}")) +} + +/// `_validate_model_header_tags`. +pub(crate) fn check_header_tags( + header: &[(ConfigNode, ConfigNode)], + model_name: &str, +) -> Result<(), ConfigError> { + let Some(tags) = set_config_entry(header, TAGS_KEY) else { + return Ok(()); + }; + if tags.kind() != NodeKind::List { + return Err(model_error(model_name, "tags must be a list")); + } + if tags.items().iter().all(|tag| tag.kind() == NodeKind::Str) { + Ok(()) + } else { + Err(model_error(model_name, "tags entries must be strings")) + } +} + +/// `validate_model_hook_config`: every hook list holds typed hook entries only. +pub(crate) fn check_hooks( + values: &[(ConfigNode, ConfigNode)], + model_name: &str, +) -> Result<(), ConfigError> { + for key in SORTED_HOOK_KEYS { + let Some(value) = config_entry(values, key) else { + continue; + }; + if !matches!(value.kind(), NodeKind::List | NodeKind::Tuple) { + return Err(model_error( + model_name, + &format!("{key} must be a list of typed hook entries"), + )); + } + let typed = + |item: &ConfigNode| matches!(item, ConfigNode::Host(host) if host.facts.hook_entry); + if !value.items().iter().all(typed) { + return Err(model_error( + model_name, + &format!( + "{key} entries must use typed inline_sql(...), sql(...), or python(...) hook syntax" + ), + )); + } + } + Ok(()) +} + +/// The facts of `key` when its value is still the header's own. +pub(crate) fn own_facts<'a>( + facts: Option<&'a [(String, HeaderEntryFacts)]>, + own: &[String], + key: &str, +) -> Option<&'a HeaderEntryFacts> { + if !own.iter().any(|name| name == key) { + return None; + } + facts? + .iter() + .find(|(name, _)| name == key) + .map(|(_, facts)| facts) +} + +/// The exact text of a string key the header facts are found by. +pub(crate) fn key_text(key: &ConfigNode) -> Option { + match key { + ConfigNode::Str(text) => Some(text.clone()), + ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.clone(), + _ => None, + } +} + +/// The first config field outside hooks holding a macro call, as Python's walk reports it. +pub(crate) fn check_no_config_macros( + values: &[(ConfigNode, ConfigNode)], + facts: Option<&[(String, HeaderEntryFacts)]>, + own: &[String], +) -> Result<(), ConfigError> { + for (key, value) in values { + if HOOK_KEYS.iter().any(|name| key.is_text(name)) { + continue; + } + let found: MacroPath = match key_text(key).and_then(|name| own_facts(facts, own, &name)) { + Some(facts) => facts.macro_path.clone(), + None => entry_macro_path(&key, &value), + }; + if let MacroPath::Found(path) = found { + return Err(ConfigError::compile(format!( + "model config field '{}' does not allow macros", + path.join(".") + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs new file mode 100644 index 0000000000..7bdb39265e --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs @@ -0,0 +1,83 @@ +//! The run and model context values templates read through `${ctx.*}`. + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::model_config_build::constants::{ + ALIAS_KEY, DATABASE_KEY, DESTINATION_DATABASE_CONTEXT, DESTINATION_QUALIFIED_CONTEXT, + DESTINATION_SCHEMA_CONTEXT, DESTINATION_TABLE_CONTEXT, MODEL_ALIAS_CONTEXT, + MODEL_DATABASE_CONTEXT, MODEL_NAME_CONTEXT, MODEL_SCHEMA_CONTEXT, RUN_ID_CONTEXT, + RUN_TARGET_CONTEXT, SCHEMA_KEY, +}; +use crate::model_config_build::models::ConfigBuildInputs; +use crate::types::{AuthoredNode, NodeKind}; + +/// Context entries. +type Entries = ConfigEntries; + +fn named(name: &str, value: ConfigNode) -> (ConfigNode, ConfigNode) { + (ConfigNode::Str(name.to_owned()), value) +} + +/// The value of `key` when it is a string, as `isinstance(raw, str)` keeps it. +pub(crate) fn string_entry( + values: &[(ConfigNode, ConfigNode)], + key: &str, +) -> Option> { + config_entry(values, key) + .filter(|value| value.kind() == NodeKind::Str) + .cloned() +} + +/// `build_run_context_values`: the run's id and target. +pub(crate) fn run_context(inputs: &ConfigBuildInputs) -> Entries { + vec![ + named(RUN_ID_CONTEXT, inputs.run_id.clone()), + named(RUN_TARGET_CONTEXT, inputs.target_name.clone()), + ] +} + +fn text_of(value: &ConfigNode) -> String { + value.text().unwrap_or_default() +} + +/// `build_model_context_values` for a model's config values. +pub(crate) fn model_context( + inputs: &ConfigBuildInputs, + values: &[(ConfigNode, ConfigNode)], + model_name: &str, + include_target_values: bool, +) -> Entries { + let database: Option> = string_entry(values, DATABASE_KEY); + let schema: Option> = string_entry(values, SCHEMA_KEY); + let alias: ConfigNode = + string_entry(values, ALIAS_KEY).unwrap_or_else(|| ConfigNode::Str(model_name.to_owned())); + let or_null = |value: &Option>| value.clone().unwrap_or(ConfigNode::Null); + let mut context: Entries = run_context(inputs); + context.extend([ + named(MODEL_NAME_CONTEXT, ConfigNode::Str(model_name.to_owned())), + named(MODEL_DATABASE_CONTEXT, or_null(&database)), + named(MODEL_SCHEMA_CONTEXT, or_null(&schema)), + named(MODEL_ALIAS_CONTEXT, alias.clone()), + ]); + if !include_target_values { + return context; + } + let qualified: ConfigNode = match (&database, &schema) { + (Some(database), Some(schema)) => ConfigNode::Str(format!( + "{}.{}.{}", + text_of(database), + text_of(schema), + text_of(&alias) + )), + (None, Some(schema)) => ConfigNode::Str(format!("{}.{}", text_of(schema), text_of(&alias))), + _ => ConfigNode::Null, + }; + context.extend([ + named(DESTINATION_DATABASE_CONTEXT, or_null(&database)), + named(DESTINATION_SCHEMA_CONTEXT, or_null(&schema)), + named(DESTINATION_TABLE_CONTEXT, alias), + named(DESTINATION_QUALIFIED_CONTEXT, qualified), + ]); + context +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs new file mode 100644 index 0000000000..32b22852c8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs @@ -0,0 +1,255 @@ +//! `${...}` templates in config values expanded over the build's variables, environment and context. + +use std::cell::RefCell; + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::models::{ConfigNode, HostChildren}; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_config_build::constants::{CONTEXT_READ, ENVIRONMENT_READ, TEMPLATE_OPEN_TOKEN}; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; +use crate::templates::main::expand_template_string::expand_template_string; +use crate::templates::main::template_error_message::template_error_message; +use crate::templates::models::{ + ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, +}; +use crate::templates::types::TemplateHost; +use crate::types::{AuthoredNode, NodeKind}; + +/// What one expansion reads: variables, the caller's environment and context values. +pub(crate) struct ExpansionSources<'a, B: ConfigBuildHost> { + pub(crate) host: &'a B, + pub(crate) variables: &'a [(HostNode, HostNode)], + pub(crate) context: ConfigEntries, +} + +/// One expansion's template host, holding the first caller error and recording reads. +struct BuildTemplateHost<'a, B: ConfigBuildHost> { + sources: ExpansionSources<'a, B>, + label: &'a str, + reads: RefCell>, + raised: RefCell>, +} + +impl BuildTemplateHost<'_, B> { + fn hold(&self, error: B::Error) -> TemplateFailure { + let _ = self.raised.borrow_mut().get_or_insert(error); + TemplateFailure::Host + } + + /// The exact text of a string value, else the caller's error. + fn exact_text<'v>(&self, value: &'v ConfigNode) -> Result<&'v str, TemplateFailure> { + match value { + ConfigNode::Str(text) => Ok(text), + ConfigNode::Host(host) if host.facts.strict_text => { + host.facts.text.as_deref().ok_or(TemplateFailure::Host) + } + _ => Err(self.hold(self.sources.host.text_error(value))), + } + } + + fn halt(&self, failure: TemplateFailure) -> HostHalt { + match failure { + TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { + ConfigBuildHalt::Config(ConfigError::compile(template_error_message( + &error, self.label, + ))) + } + TemplateFailure::Host => match self.raised.borrow_mut().take() { + Some(error) => ConfigBuildHalt::Host(error), + None => ConfigBuildHalt::Config(ConfigError::compile( + "template expansion stopped without an error".to_owned(), + )), + }, + } + } +} + +impl TemplateHost for BuildTemplateHost<'_, B> { + type Value = ConfigNode; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + Ok(config_entry(self.sources.variables, name).cloned()) + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((ENVIRONMENT_READ, name.to_owned())); + self.sources + .host + .environment(name) + .map_err(|error| self.hold(error)) + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((CONTEXT_READ, name.to_owned())); + Ok(match config_entry(&self.sources.context, name) { + None => ContextValue::Unknown, + Some(ConfigNode::Null) => ContextValue::Unavailable, + Some(value) => ContextValue::Value(value.clone()), + }) + } + + fn text(&self, text: &str) -> Result { + Ok(ConfigNode::Str(text.to_owned())) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(ConfigNode::Bool(flag)) + } + + fn null(&self) -> Self::Value { + ConfigNode::Null + } + + fn scalar(&self, value: &Self::Value) -> Result { + match value { + ConfigNode::Null => Ok(Scalar::Null), + ConfigNode::Bool(flag) => Ok(Scalar::Bool(*flag)), + ConfigNode::Host(host) if !host.facts.strict_str => { + Err(self.hold(self.sources.host.text_error(value))) + } + _ => Ok(Scalar::Text(value.python_str())), + } + } + + fn render(&self, value: &Self::Value, label: &str) -> Result { + if let ConfigNode::Str(text) = value { + return Ok(text.clone()); + } + match self.sources.host.render(value, label) { + Ok(Ok(text)) => Ok(text), + Ok(Err(message)) => Err(TemplateFailure::Invalid( + crate::templates::errors::TemplateError::Message(message), + )), + Err(error) => Err(self.hold(error)), + } + } + + fn label(&self) -> &str { + self.label + } +} + +/// Each item expanded. +fn expanded_items( + host: &BuildTemplateHost<'_, B>, + values: &[HostNode], + options: TemplateOptions, +) -> Result>, TemplateFailure> { + values + .iter() + .map(|item| expanded(host, item, options)) + .collect() +} + +/// Each entry's value expanded under its key. +fn expanded_entries( + host: &BuildTemplateHost<'_, B>, + mapping: &[(HostNode, HostNode)], + options: TemplateOptions, +) -> Result, TemplateFailure> { + let mut expanded_mapping: HostEntries = Vec::with_capacity(mapping.len()); + for (key, item) in mapping { + expanded_mapping.push((key.clone(), expanded(host, item, options)?)); + } + Ok(expanded_mapping) +} + +/// Every template in `value` and the containers inside it expanded, as `expanded()` does. +fn expanded( + host: &BuildTemplateHost<'_, B>, + value: &ConfigNode, + options: TemplateOptions, +) -> Result, TemplateFailure> { + let items = |values: &[HostNode]| expanded_items(host, values, options); + let entries = |mapping: &[(HostNode, HostNode)]| expanded_entries(host, mapping, options); + match (value, value.kind()) { + (_, NodeKind::Str) => { + let text: &str = host.exact_text(value)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + Ok(match expand_template_string(host, text, options)? { + StringExpansion::Unchanged => ConfigNode::Str(text.to_owned()), + StringExpansion::Value(result) => result, + StringExpansion::Text(result) => ConfigNode::Str(result), + }) + } + (ConfigNode::Map(mapping), _) => Ok(ConfigNode::Map(entries(mapping)?)), + (ConfigNode::List(values), _) => Ok(ConfigNode::List(items(values)?)), + (ConfigNode::Tuple(values), _) => Ok(ConfigNode::Tuple(items(values)?)), + (ConfigNode::Host(opaque), kind) => match (&opaque.children, kind) { + (HostChildren::Entries(mapping), NodeKind::Map) => { + Ok(ConfigNode::Map(entries(mapping)?)) + } + (HostChildren::Items(values), NodeKind::List) => Ok(ConfigNode::List(items(values)?)), + (HostChildren::Items(values), NodeKind::Tuple) => Ok(ConfigNode::Tuple(items(values)?)), + _ => Ok(value.clone()), + }, + _ => Ok(value.clone()), + } +} + +/// An expansion's result and the `(kind, name)` reads it made, in order. +pub(crate) type Expanded = ( + Result::Handle>, HostHalt>, + Vec<(&'static str, String)>, +); + +/// A result and the `(kind, name)` reads made on the way to it. +type Tracked = (Result>, Vec<(&'static str, String)>); + +/// `expand` run with a template host over `sources`, with the reads it made. +fn with_host( + sources: ExpansionSources<'_, B>, + scope: (bool, &str), + expand: impl FnOnce(&BuildTemplateHost<'_, B>, TemplateOptions) -> Result, +) -> Tracked { + let (preserve_unknown_context, label) = scope; + let host: BuildTemplateHost<'_, B> = BuildTemplateHost { + sources, + label, + reads: RefCell::new(Vec::new()), + raised: RefCell::new(None), + }; + let options: TemplateOptions = TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context, + }; + let result = expand(&host, options).map_err(|failure| host.halt(failure)); + (result, host.reads.into_inner()) +} + +/// `value` expanded, with its reads; `scope` is `preserve_unknown_context` and the context label. +pub(crate) fn expand_value( + sources: ExpansionSources<'_, B>, + value: &ConfigNode, + scope: (bool, &str), +) -> Expanded { + with_host(sources, scope, |host, options| { + expanded(host, value, options) + }) +} + +/// Expanded entries and the `(kind, name)` reads the expansion made. +pub(crate) type ExpandedEntries = ( + Result, HostHalt>, + Vec<(&'static str, String)>, +); + +/// A mapping's entries expanded, with the reads. +pub(crate) fn expand_entries( + sources: ExpansionSources<'_, B>, + entries: &[(HostNode, HostNode)], + scope: (bool, &str), +) -> ExpandedEntries { + with_host(sources, scope, |host, options| { + expanded_entries(host, entries, options) + }) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs new file mode 100644 index 0000000000..7b223de378 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs @@ -0,0 +1,79 @@ +//! A model's layers merged, with which header entries are still the header's own values. + +use crate::config_values::main::layered_config::layered_config; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::model_config_build::_helpers::checks::key_text; +use crate::model_config_build::constants::{TAGS_KEY, TEMPLATE_OPEN_TOKEN}; +use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::types::{AuthoredNode, NodeKind}; + +/// Config entries. +pub(crate) type Entries = ConfigEntries; + +/// A fresh copy of the project defaults, with its own `tags` list. +pub(crate) fn project_defaults(inputs: &ConfigBuildInputs) -> Entries { + inputs + .defaults + .iter() + .map(|(key, value)| match value { + ConfigNode::List(items) if key.is_text(TAGS_KEY) => { + (key.clone(), ConfigNode::List(items.clone())) + } + _ => (key.clone(), value.clone()), + }) + .collect() +} + +/// The path default `key`'s entries. +pub(crate) fn path_default_values<'a, H: Clone, E>( + inputs: &'a ConfigBuildInputs, + key: &str, +) -> Result<&'a Entries, ConfigBuildHalt> { + inputs + .path_defaults + .iter() + .find(|(name, _)| name == key) + .map(|(_, values)| values) + .ok_or_else(|| ConfigBuildHalt::MissingPathDefault(key.to_owned())) +} + +/// `overlay` over `base`, and the overlay's keys whose merged value is still the overlay's own. +pub(crate) fn layered_own( + base: &Entries, + overlay: &[(ConfigNode, ConfigNode)], +) -> (Entries, Vec) { + let layered = layered_config(base, overlay); + let own: Vec = overlay + .iter() + .filter_map(|(key, _)| key_text(key)) + .filter(|name| !layered.unioned_keys.contains(&name.as_str())) + .collect(); + (layered.entries, own) +} + +/// Whether expansion keeps `value` as the same object: a string without a template, or a non-container. +pub(crate) fn kept_by_expansion(value: &ConfigNode) -> bool { + match value.kind() { + NodeKind::Str => !value + .with_text(|text| text.contains(TEMPLATE_OPEN_TOKEN)) + .unwrap_or(false), + NodeKind::Map | NodeKind::List | NodeKind::Tuple => false, + _ => true, + } +} + +/// Whether the entry named `name` in `values` keeps its value through an expansion. +fn kept_entry(values: &Entries, name: &str) -> bool { + values + .iter() + .find(|(key, _)| key_text(key).as_deref() == Some(name)) + .is_some_and(|(_, value)| kept_by_expansion(value)) +} + +/// The own keys whose values an expansion of `values` keeps. +pub(crate) fn kept_own(values: &Entries, own: Vec) -> Vec { + own.into_iter() + .filter(|name| kept_entry(values, name)) + .collect() +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs new file mode 100644 index 0000000000..7250eb231f --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod checks; +pub(crate) mod context; +pub(crate) mod expansion; +pub(crate) mod layers; +pub(crate) mod namespace; +pub(crate) mod phases; +pub(crate) mod storage; diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs new file mode 100644 index 0000000000..858bbd444e --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/namespace.rs @@ -0,0 +1,91 @@ +//! The target namespace applied over a model's values, as `apply_environment_database_schema_overrides`. + +use crate::config_values::main::inserted_entry::inserted_entry; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_config_build::_helpers::context::model_context; +use crate::model_config_build::_helpers::expansion::{ExpansionSources, expand_value}; +use crate::model_config_build::constants::{ + DATABASE_KEY, ENVIRONMENT_DATABASE_LABEL, ENVIRONMENT_SCHEMA_LABEL, PRESERVE_TARGET_VALUE, + SCHEMA_KEY, +}; +use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt}; +use crate::types::AuthoredNode; + +/// Config entries. +type Entries = ConfigEntries; + +/// The namespaced values, the keys the target replaced, and the reads its expansions made. +pub(crate) struct Namespaced { + pub(crate) values: Entries, + pub(crate) replaced: Vec<&'static str>, + pub(crate) reads: Vec<(&'static str, String)>, +} + +/// One model's values with the target's database and schema expanded over them. +pub(crate) fn namespaced_values( + (host, inputs): (&B, &ConfigBuildInputs), + values: &Entries, + logical: (bool, bool), + model_name: &str, +) -> Result, HostHalt> { + let mut namespaced = Namespaced { + values: values.clone(), + replaced: Vec::new(), + reads: Vec::new(), + }; + let Some((database, schema)) = &inputs.target_namespace else { + return Ok(namespaced); + }; + let (logical_database, logical_schema) = logical; + let namespace = [ + ( + DATABASE_KEY, + database, + logical_database, + ENVIRONMENT_DATABASE_LABEL, + ), + (SCHEMA_KEY, schema, logical_schema, ENVIRONMENT_SCHEMA_LABEL), + ]; + let preserved = |value: &ConfigNode| value.is_text(PRESERVE_TARGET_VALUE); + let missing: Vec<&str> = namespace + .iter() + .filter(|(_, value, logical, _)| preserved(value) && !logical) + .map(|(key, _, _, _)| *key) + .collect(); + if !missing.is_empty() { + let dimensions: String = missing.join(" and "); + return Err(ConfigBuildHalt::Config( + ConfigError::compile(format!( + "Model '{model_name}' has no logical {dimensions}, but the selected target sets \ + {dimensions} to 'preserve'" + )) + .with_help(format!( + "Set {dimensions} on the resource or its defaults, or configure a literal \ + target {dimensions}." + )), + )); + } + let context: Entries = model_context(inputs, values, model_name, false); + for (key, value, _, label) in namespace { + if matches!(value, ConfigNode::Null) || preserved(value) { + continue; + } + let sources: ExpansionSources<'_, B> = ExpansionSources { + host, + variables: &inputs.variables, + context: context.clone(), + }; + let (expanded, reads) = expand_value(sources, value, (false, label)); + namespaced.reads.extend(reads); + namespaced.values = inserted_entry( + namespaced.values, + ConfigNode::Str(key.to_owned()), + expanded?, + ); + namespaced.replaced.push(key); + } + Ok(namespaced) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs new file mode 100644 index 0000000000..a81551add4 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs @@ -0,0 +1,203 @@ +//! The phases of one model's config build: layering, model templates, target namespace, result. + +use crate::config_presence::main::contains_template::contains_template; +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::inserted_entry::inserted_entry; +use crate::config_values::models::ConfigNode; +use crate::model_config_build::_helpers::checks::{ + check_hooks, check_no_config_macros, key_text, own_facts, +}; +use crate::model_config_build::_helpers::context::{model_context, run_context, string_entry}; +use crate::model_config_build::_helpers::expansion::{ExpansionSources, expand_entries}; +use crate::model_config_build::_helpers::layers::{ + Entries, kept_own, layered_own, path_default_values, project_defaults, +}; +use crate::model_config_build::_helpers::namespace::{Namespaced, namespaced_values}; +use crate::model_config_build::_helpers::storage::{sorted_header_keys, storage_free_values}; +use crate::model_config_build::constants::{ + DATABASE_KEY, HOOK_KEYS, MODEL_CONFIG_LABEL, RETENTION_KEY, SCHEMA_KEY, TABLE_TYPE_KEY, +}; +use crate::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, ModelHeader, +}; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; +use crate::model_validation::main::retention_override::retention_override; +use crate::model_validation::main::table_type_override::table_type_override; +use crate::types::AuthoredNode; + +/// What one model's build reads: its host and project inputs, its header and its name. +pub(crate) struct BuildModel<'a, B: ConfigBuildHost> { + pub(crate) host: &'a B, + pub(crate) inputs: &'a ConfigBuildInputs, + pub(crate) header: ModelHeader<'a, B::Handle>, + pub(crate) name: &'a str, +} + +/// The values so far, the header keys still holding the header's own values, and the reads made. +pub(crate) struct Phase { + pub(crate) values: Entries, + pub(crate) own: Vec, + pub(crate) reads: Vec<(&'static str, String)>, +} + +/// The layered values with hooks set aside, and whether a layer configures the schema. +pub(crate) struct Layered { + pub(crate) phase: Phase, + pub(crate) hooks: Entries, + pub(crate) layer_schema_configured: bool, + pub(crate) has_templates: bool, +} + +/// The namespaced values and the logical database and schema the model set itself. +pub(crate) struct Targeted { + pub(crate) phase: Phase, + pub(crate) logical_schema: Option>, + pub(crate) logical_database: Option>, +} + +/// The project defaults, the matched path default and the header merged, with hooks set aside. +pub(crate) fn layered( + model: &BuildModel<'_, B>, + matched_path_default: Option<&str>, +) -> Result, HostHalt> { + let path_values: Option<&Entries> = matched_path_default + .map(|key| path_default_values(model.inputs, key)) + .transpose()?; + let mut defaults: Entries = project_defaults(model.inputs); + if let Some(path_values) = path_values { + defaults = layered_own(&defaults, path_values).0; + } + let (mut values, own) = layered_own(&defaults, model.header.entries); + check_hooks(&values, model.name).map_err(ConfigBuildHalt::Config)?; + let mut hooks: Entries = Vec::new(); + for key in HOOK_KEYS { + if let Some(position) = values.iter().position(|(name, _)| name.is_text(key)) { + hooks.push(values.remove(position)); + } + } + let has_templates: bool = values.iter().any(|(key, value)| { + let facts = key_text(key).and_then(|name| own_facts(model.header.facts, &own, &name)); + facts.map_or_else(|| contains_template(&value), |facts| facts.template) + }); + let layer_schema_configured: bool = config_entry(model.header.entries, SCHEMA_KEY).is_some() + || path_values.is_some_and(|values| config_entry(values, SCHEMA_KEY).is_some()); + Ok(Layered { + phase: Phase { + values, + own, + reads: Vec::new(), + }, + hooks, + layer_schema_configured, + has_templates, + }) +} + +/// `phase`'s values expanded once over `sources`, its reads extended. +fn expanded_phase( + mut phase: Phase, + sources: ExpansionSources<'_, B>, + preserve_unknown_context: bool, +) -> Result, HostHalt> { + let scope = (preserve_unknown_context, MODEL_CONFIG_LABEL); + let (expanded, reads) = expand_entries(sources, &phase.values, scope); + phase.reads.extend(reads); + phase.values = expanded?; + Ok(phase) +} + +/// Sources of `model`'s host over `variables` and `context`. +fn sources<'a, B: ConfigBuildHost>( + model: &BuildModel<'a, B>, + variables: &'a [(HostNode, HostNode)], + context: HostEntries, +) -> ExpansionSources<'a, B> { + ExpansionSources { + host: model.host, + variables, + context, + } +} + +/// The model's templates expanded against the run, then twice against the model context. +pub(crate) fn model_templates( + model: &BuildModel<'_, B>, + mut phase: Phase, +) -> Result, HostHalt> { + phase.own = kept_own(&phase.values, phase.own); + let run = run_context(model.inputs); + phase = expanded_phase(phase, sources(model, &model.inputs.variables, run), true)?; + for _ in 0..2 { + let context = model_context(model.inputs, &phase.values, model.name, false); + phase = expanded_phase(phase, sources(model, &[], context), true)?; + } + Ok(phase) +} + +/// The target's namespace applied, then target templates expanded when any are left. +pub(crate) fn targeted( + model: &BuildModel<'_, B>, + mut phase: Phase, + has_templates: bool, +) -> Result, HostHalt> { + let logical_schema = string_entry(&phase.values, SCHEMA_KEY); + let logical_database = string_entry(&phase.values, DATABASE_KEY); + let logical = (logical_database.is_some(), logical_schema.is_some()); + let Namespaced { + values, + replaced, + reads, + } = namespaced_values( + (model.host, model.inputs), + &phase.values, + logical, + model.name, + )?; + phase.values = values; + phase.reads.extend(reads); + phase.own.retain(|name| !replaced.contains(&name.as_str())); + let needs_target_templates: bool = has_templates + || [DATABASE_KEY, SCHEMA_KEY].iter().any(|key| { + config_entry(&phase.values, key).is_some_and(|value| contains_template(&value)) + }); + if needs_target_templates { + phase.own = kept_own(&phase.values, phase.own); + let context = model_context(model.inputs, &phase.values, model.name, true); + phase = expanded_phase(phase, sources(model, &[], context), false)?; + } + Ok(Targeted { + phase, + logical_schema, + logical_database, + }) +} + +/// The built config: hooks restored, storage keys dropped, overrides read and macros rejected. +pub(crate) fn finished( + model: &BuildModel<'_, B>, + layered: (Entries, bool), + targeted: Targeted, +) -> Result, HostHalt> { + let (hooks, layer_schema_configured) = layered; + let (header, config) = (&model.header, ConfigBuildHalt::Config); + let mut resolved: Entries = targeted.phase.values; + for (key, value) in hooks { + resolved = inserted_entry(resolved, key, value); + } + let values: Entries = storage_free_values(resolved, header.entries); + let retention_value = config_entry(header.entries, RETENTION_KEY); + let retention = retention_override(retention_value, model.name).map_err(config)?; + let table_type_value = config_entry(header.entries, TABLE_TYPE_KEY); + let table_type = table_type_override(table_type_value, model.name).map_err(config)?; + check_no_config_macros(&values, header.facts, &targeted.phase.own).map_err(config)?; + Ok(BuiltModelConfig { + values, + header_keys: sorted_header_keys(header.entries)?, + logical_schema: targeted.logical_schema, + layer_schema_configured, + logical_database: targeted.logical_database, + retention, + table_type, + reads: targeted.phase.reads, + }) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs new file mode 100644 index 0000000000..d1db8e8219 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs @@ -0,0 +1,56 @@ +//! The built values without storage keys, and the header's keys. + +use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::same_config_key::same_config_key; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::model_config_build::_helpers::checks::key_text; +use crate::model_config_build::constants::{ + FULL_REFRESH_KEY, INCREMENTAL_MATERIALIZATION, MATERIALIZED_KEY, RETENTION_KEY, TABLE_TYPE_KEY, +}; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::types::AuthoredNode; + +/// Config entries. +type Entries = ConfigEntries; + +/// Whether `key` names a storage setting, which the config values leave out. +fn storage_key(key: &ConfigNode) -> bool { + key.is_text(RETENTION_KEY) || key.is_text(TABLE_TYPE_KEY) +} + +/// Drop storage keys, then `full_refresh` where `build_model_config` drops it. +pub(crate) fn storage_free_values( + resolved: Entries, + header: &[(ConfigNode, ConfigNode)], +) -> Entries { + let mut values: Entries = resolved + .into_iter() + .filter(|(key, _)| !storage_key(key)) + .collect(); + let incremental: bool = config_entry(&values, MATERIALIZED_KEY) + .is_some_and(|value| value.is_text(INCREMENTAL_MATERIALIZATION)); + if config_entry(header, FULL_REFRESH_KEY).is_none() && !incremental { + let refresh = ConfigNode::Str(FULL_REFRESH_KEY.to_owned()); + values.retain(|(key, _)| !same_config_key(key, &refresh)); + } + values +} + +/// The header's keys sorted by code point; a key that is not a string halts. +pub(crate) fn sorted_header_keys( + header: &[(ConfigNode, ConfigNode)], +) -> Result, ConfigBuildHalt> { + let mut keys: Vec = Vec::with_capacity(header.len()); + for (key, _) in header { + match (key_text(key), key) { + (Some(text), _) => keys.push(text), + (None, ConfigNode::Host(host)) if host.facts.text.is_some() => { + keys.push(host.facts.text.clone().unwrap_or_default()); + } + (None, _) => return Err(ConfigBuildHalt::KeyNotText(key.clone())), + } + } + keys.sort(); + Ok(keys) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/constants.rs b/crates/sqlbuild-model-config/src/model_config_build/constants.rs new file mode 100644 index 0000000000..0260df4a21 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/constants.rs @@ -0,0 +1,31 @@ +//! Config keys, labels and context names the model config build reads and writes. + +pub(crate) const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +/// Hook keys in the sorted order `validate_model_hook_config` checks them. +pub(crate) const SORTED_HOOK_KEYS: [&str; 2] = ["post_hooks", "pre_hooks"]; +pub(crate) const MODEL_CONFIG_LABEL: &str = "model config"; +pub(crate) const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; +pub(crate) const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; +pub(crate) const TAGS_KEY: &str = "tags"; +pub(crate) const DATABASE_KEY: &str = "database"; +pub(crate) const SCHEMA_KEY: &str = "schema"; +pub(crate) const ALIAS_KEY: &str = "alias"; +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const FULL_REFRESH_KEY: &str = "full_refresh"; +pub(crate) const RETENTION_KEY: &str = "time_travel_retention"; +pub(crate) const TABLE_TYPE_KEY: &str = "table_type"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const PRESERVE_TARGET_VALUE: &str = "preserve"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const ENVIRONMENT_READ: &str = "env"; +pub(crate) const CONTEXT_READ: &str = "ctx"; +pub(crate) const RUN_ID_CONTEXT: &str = "run.id"; +pub(crate) const RUN_TARGET_CONTEXT: &str = "run.target"; +pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; +pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; +pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; +pub(crate) const MODEL_ALIAS_CONTEXT: &str = "model.alias"; +pub(crate) const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; +pub(crate) const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; +pub(crate) const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; +pub(crate) const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs new file mode 100644 index 0000000000..b2115d2048 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs @@ -0,0 +1,37 @@ +//! One model's effective config built from its layers, as `build_model_config` builds it. + +use crate::model_config_build::_helpers::checks::check_header_tags; +use crate::model_config_build::_helpers::phases::{ + BuildModel, Layered, finished, layered, model_templates, targeted, +}; +use crate::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, ModelHeader, +}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt}; + +/// One model's effective config: its layers merged, then expanded for the model and its target. +pub fn build_model_config( + (host, inputs): (&B, &ConfigBuildInputs), + header: ModelHeader<'_, B::Handle>, + matched_path_default: Option<&str>, + name: &str, +) -> Result, HostHalt> { + check_header_tags(header.entries, name).map_err(ConfigBuildHalt::Config)?; + let model: BuildModel<'_, B> = BuildModel { + host, + inputs, + header, + name, + }; + let Layered { + mut phase, + hooks, + layer_schema_configured, + has_templates, + } = layered(&model, matched_path_default)?; + if has_templates { + phase = model_templates(&model, phase)?; + } + let targeted = targeted(&model, phase, has_templates)?; + finished(&model, (hooks, layer_schema_configured), targeted) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs new file mode 100644 index 0000000000..8a29ef96a7 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs @@ -0,0 +1,19 @@ +//! What a path default's models inherit. + +use crate::config_values::types::ConfigEntries; +use crate::model_config_build::_helpers::layers::{ + layered_own, path_default_values, project_defaults, +}; +use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; + +/// `build_layered_model_values` with an empty header: the defaults under the matched path default. +pub fn inherited_config( + inputs: &ConfigBuildInputs, + matched_path_default: Option<&str>, +) -> Result, ConfigBuildHalt> { + let mut layered = project_defaults(inputs); + if let Some(key) = matched_path_default { + layered = layered_own(&layered, path_default_values(inputs, key)?).0; + } + Ok(layered) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs new file mode 100644 index 0000000000..d48f05004a --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/mod.rs @@ -0,0 +1,3 @@ +pub mod build_model_config; +pub mod inherited_config; +pub mod model_context_values; diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs new file mode 100644 index 0000000000..ee7d66dfc5 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs @@ -0,0 +1,17 @@ +//! The context values hook templates read for one model's built config. + +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::model_config_build::_helpers::context::model_context; +use crate::model_config_build::models::ConfigBuildInputs; + +/// `build_model_context_values` for a model's config values. +#[must_use] +pub fn model_context_values( + inputs: &ConfigBuildInputs, + values: &[(ConfigNode, ConfigNode)], + model_name: &str, + include_target_values: bool, +) -> ConfigEntries { + model_context(inputs, values, model_name, include_target_values) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/mod.rs b/crates/sqlbuild-model-config/src/model_config_build/mod.rs new file mode 100644 index 0000000000..39e299a85d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/mod.rs @@ -0,0 +1,10 @@ +//! A model's effective config: its layers merged, templated and checked, as `build_model_config` builds it. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/model_config_build/models.rs b/crates/sqlbuild-model-config/src/model_config_build/models.rs new file mode 100644 index 0000000000..d8bb0bf2f8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/models.rs @@ -0,0 +1,63 @@ +//! The inputs and result of one model's config build. + +use crate::config_presence::models::MacroPath; +use crate::config_values::models::ConfigNode; +use crate::config_values::types::ConfigEntries; +use crate::errors::ConfigError; +use crate::model_validation::models::{RetentionOverride, TableTypeOverride}; + +/// The project-wide inputs every model's config build reads, captured once per compile. +#[derive(Clone, Debug)] +pub struct ConfigBuildInputs { + /// `project_defaults_to_mapping`'s entries. + pub defaults: ConfigEntries, + /// Each path default's entries by its key. + pub path_defaults: Vec<(String, ConfigEntries)>, + /// The effective vars. + pub variables: ConfigEntries, + pub run_id: ConfigNode, + pub target_name: ConfigNode, + /// The active target's `(database, schema)`, without a target `None`. + pub target_namespace: Option<(ConfigNode, ConfigNode)>, +} + +/// What the native header parse found in one header entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderEntryFacts { + /// Whether a string in the value holds a `${...}` template. + pub template: bool, + /// Where the config macro walk finds a macro call. + pub macro_path: MacroPath, +} + +/// One model's authored header and, when it was parsed natively, each entry's facts by key. +#[derive(Clone, Copy, Debug)] +pub struct ModelHeader<'a, H> { + pub entries: &'a [(ConfigNode, ConfigNode)], + pub facts: Option<&'a [(String, HeaderEntryFacts)]>, +} + +/// One model's built config, before the caller wraps it in `CompileModelConfig`. +#[derive(Clone, Debug)] +pub struct BuiltModelConfig { + pub values: ConfigEntries, + pub header_keys: Vec, + pub logical_schema: Option>, + pub layer_schema_configured: bool, + pub logical_database: Option>, + pub retention: RetentionOverride, + pub table_type: TableTypeOverride, + /// The `(kind, name)` environment and context reads, in order. + pub reads: Vec<(&'static str, String)>, +} + +/// Why a build stops: the config error it raises, a caller error, or a header key that is not text. +#[derive(Debug, PartialEq)] +pub enum ConfigBuildHalt { + Config(ConfigError), + Host(E), + /// A header key that is not a string, which Python fails to read as one. + KeyNotText(ConfigNode), + /// A path default the inputs do not hold. + MissingPathDefault(String), +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests.rs b/crates/sqlbuild-model-config/src/model_config_build/tests.rs new file mode 100644 index 0000000000..ce6598d186 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_model_config.rs"] +mod build_model_config; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs new file mode 100644 index 0000000000..4715d4af52 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/helpers.rs @@ -0,0 +1,41 @@ +use crate::config_values::models::ConfigNode; +use crate::model_config_build::main::build_model_config::build_model_config; +use crate::model_config_build::models::{ConfigBuildInputs, ModelHeader}; +use crate::model_config_build::tests::test_types::{BuildOutcome, BuildTestCase, FixedHost, Node}; + +/// A string node. +pub(super) fn text(value: &str) -> Node { + ConfigNode::Str(value.to_owned()) +} + +/// A string-keyed entry. +pub(super) fn named(key: &str, value: Node) -> (Node, Node) { + (text(key), value) +} + +/// The values and reads of building `orders` from the case's header, or the error text. +pub(super) fn built(test_case: &BuildTestCase) -> BuildOutcome { + let inputs: ConfigBuildInputs = ConfigBuildInputs { + defaults: vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + ], + path_defaults: Vec::new(), + variables: vec![ + named("region", text("north")), + named("count", ConfigNode::Int(3)), + ], + run_id: text("run_1"), + target_name: text("dev"), + target_namespace: test_case.target_namespace.clone(), + }; + let host = FixedHost { + environment: vec![("SCHEMA_SUFFIX", "eu")], + }; + let header = ModelHeader { + entries: &test_case.header, + facts: None, + }; + build_model_config((&host, &inputs), header, None, "orders") + .map(|built| (built.values, built.reads)) +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs new file mode 100644 index 0000000000..04afb9130d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/test_build_model_config.rs @@ -0,0 +1,91 @@ +use crate::config_values::models::ConfigNode; +use crate::errors::ConfigError; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::tests::helpers::{built, named, text}; +use crate::model_config_build::tests::test_types::BuildTestCase; + +#[test] +fn given_model_headers_when_building_config_then_values_expand_and_layer_as_python_does() { + let test_cases = [ + BuildTestCase { + description: "the header overrides defaults and tags union in place", + header: vec![ + named("tags", ConfigNode::List(vec![text("orders")])), + named("materialized", text("table")), + ], + target_namespace: None, + expected_outcome: Ok(( + vec![ + named("materialized", text("table")), + named("tags", ConfigNode::List(vec![text("core"), text("orders")])), + ], + vec![], + )), + }, + BuildTestCase { + description: "templates read variables, the environment and the model context", + header: vec![named( + "alias", + text("${CTX:model.name}_${region}_${ENV:SCHEMA_SUFFIX}_${count}"), + )], + target_namespace: None, + expected_outcome: Ok(( + vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + named("alias", text("orders_north_eu_3")), + ], + vec![ + ("ctx", "model.name".to_owned()), + ("env", "SCHEMA_SUFFIX".to_owned()), + ("ctx", "model.name".to_owned()), + ], + )), + }, + BuildTestCase { + description: "the target schema expands with the model context", + header: vec![], + target_namespace: Some((ConfigNode::Null, text("${region}_${CTX:model.name}"))), + expected_outcome: Ok(( + vec![ + named("materialized", text("view")), + named("tags", ConfigNode::List(vec![text("core")])), + named("schema", text("north_orders")), + ], + vec![("ctx", "model.name".to_owned())], + )), + }, + BuildTestCase { + description: "a preserved target schema needs a logical schema", + header: vec![], + target_namespace: Some((ConfigNode::Null, text("preserve"))), + expected_outcome: Err(ConfigBuildHalt::Config( + ConfigError::compile( + "Model 'orders' has no logical schema, but the selected target sets schema \ + to 'preserve'" + .to_owned(), + ) + .with_help( + "Set schema on the resource or its defaults, or configure a literal target \ + schema.", + ), + )), + }, + BuildTestCase { + description: "a macro call in a config value is rejected", + header: vec![named("description", text("@note()"))], + target_namespace: None, + expected_outcome: Err(ConfigBuildHalt::Config(ConfigError::compile( + "model config field 'description' does not allow macros".to_owned(), + ))), + }, + ]; + for test_case in test_cases { + assert_eq!( + built(&test_case), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs new file mode 100644 index 0000000000..06dcd266ac --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs @@ -0,0 +1,46 @@ +use crate::config_values::models::ConfigNode; +use crate::model_config_build::models::ConfigBuildHalt; +use crate::model_config_build::types::ConfigBuildHost; + +/// A config node in tests; host handles are plain numbers. +pub(super) type Node = ConfigNode; + +/// An environment of fixed variables; rendering takes `str()` of numbers. +pub(super) struct FixedHost { + pub(super) environment: Vec<(&'static str, &'static str)>, +} + +impl ConfigBuildHost for FixedHost { + type Handle = u8; + type Error = String; + + fn environment(&self, name: &str) -> Result, String> { + Ok(self + .environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| ConfigNode::Str((*value).to_owned()))) + } + + fn render(&self, value: &Node, _label: &str) -> Result, String> { + Ok(match value { + ConfigNode::Int(number) => Ok(number.to_string()), + _ => Err("cannot be interpolated".to_owned()), + }) + } + + fn text_error(&self, _value: &Node) -> String { + "not text".to_owned() + } +} + +/// The built values and reads, or why the build stops. +pub(super) type BuildOutcome = + Result<(Vec<(Node, Node)>, Vec<(&'static str, String)>), ConfigBuildHalt>; + +pub(super) struct BuildTestCase { + pub(super) description: &'static str, + pub(super) header: Vec<(Node, Node)>, + pub(super) target_namespace: Option<(Node, Node)>, + pub(super) expected_outcome: BuildOutcome, +} diff --git a/crates/sqlbuild-model-config/src/model_config_build/types.rs b/crates/sqlbuild-model-config/src/model_config_build/types.rs new file mode 100644 index 0000000000..5444ec214a --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_config_build/types.rs @@ -0,0 +1,31 @@ +//! What the model config build asks of its caller. + +use crate::config_values::models::ConfigNode; +use crate::model_config_build::models::ConfigBuildHalt; + +/// The process environment, project-var rendering and exact text errors, owned by the caller. +pub trait ConfigBuildHost { + /// The caller's handle of an opaque value. + type Handle: Clone; + /// An error the caller raises, held until the build stops. + type Error; + + /// The environment variable `name`, or `None` when it is not set. + fn environment(&self, name: &str) -> Result>, Self::Error>; + /// A non-string value as interpolated text, or the message of a value that cannot be. + fn render( + &self, + value: &ConfigNode, + label: &str, + ) -> Result, Self::Error>; + /// The error reading `value` as exact text raises. + fn text_error(&self, value: &ConfigNode) -> Self::Error; +} + +/// A config value of the build host `B`. +pub type HostNode = ConfigNode<::Handle>; +/// Config entries of the build host `B`. +pub type HostEntries = Vec<(HostNode, HostNode)>; +/// Why a build over the host `B` stops. +pub type HostHalt = + ConfigBuildHalt<::Handle, ::Error>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs index aab90f802b..a59dc54979 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs @@ -66,7 +66,7 @@ fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResul Ok(text) => utf8_text(text)?, Err(_) => false, }; - Ok(ConfigNode::Host(HostValue { + Ok(ConfigNode::Host(Box::new(HostValue { handle: Arc::new(value.clone().unbind()), facts: HostFacts { kind: node.kind(), @@ -81,7 +81,7 @@ fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResul hook_entry: hook_entry(value)?, }, children, - })) + }))) } /// Converted items, or `None` when a container holds itself; `open` holds the enclosing containers. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index dfb4a278ad..88b983f712 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -1,103 +1,43 @@ -//! Effective model config layered and templated natively, as `build_model_config` builds it. +//! The native model config build handed Python inputs once, and its results projected for Python. -use pyo3::exceptions::PyKeyError; +use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, - PyTupleMethods, -}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; -use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; -use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::model_validation::main::retention_override::retention_override; -use sqlbuild_model_config::model_validation::main::table_type_override::table_type_override; +use sqlbuild_model_config::model_config_build::main::build_model_config::build_model_config; +use sqlbuild_model_config::model_config_build::main::inherited_config::inherited_config; +use sqlbuild_model_config::model_config_build::main::model_context_values::model_context_values; +use sqlbuild_model_config::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, HeaderEntryFacts, ModelHeader, +}; +use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; -use sqlbuild_model_config::templates::models::TemplateOptions; -use sqlbuild_model_config::types::AuthoredNode; +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_classes::{ConfigClasses, built_config}; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::config_templates::{ - PythonHost, Stop, TemplateSources, expanded, template_error, +use crate::bindings::_helpers::model_config::config_nodes::{ + HostHandle, PyConfigEntries, PyConfigNode, config_dict, config_entries, config_node, + config_object, }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; -const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; -/// Hook keys in the sorted order `validate_model_hook_config` checks them. -const SORTED_HOOK_KEYS: [&str; 2] = ["post_hooks", "pre_hooks"]; -const MODEL_CONFIG_LABEL: &str = "model config"; -const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; -const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; -const TAGS_KEY: &str = "tags"; -const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; -const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; -const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; -const DATABASE_KEY: &str = "database"; -const SCHEMA_KEY: &str = "schema"; -const ALIAS_KEY: &str = "alias"; -const MATERIALIZED_KEY: &str = "materialized"; -const FULL_REFRESH_KEY: &str = "full_refresh"; -const RETENTION_KEY: &str = "time_travel_retention"; -const TABLE_TYPE_KEY: &str = "table_type"; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -const PRESERVE_TARGET_VALUE: &str = "preserve"; -const RUN_ID_CONTEXT: &str = "run.id"; -const RUN_TARGET_CONTEXT: &str = "run.target"; -const MODEL_NAME_CONTEXT: &str = "model.name"; -const MODEL_DATABASE_CONTEXT: &str = "model.database"; -const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; -const MODEL_ALIAS_CONTEXT: &str = "model.alias"; -const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; -const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; -const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; -const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; - -/// Why the native build stops: the config error, or a Python error. -enum Halt { - Config(ConfigError), - Error(PyErr), -} - -impl From for Halt { - fn from(error: PyErr) -> Self { - Self::Error(error) - } -} - -impl From for Halt { - fn from(error: ConfigError) -> Self { - Self::Config(error) - } -} - -/// The `CompileInputError` `model '' `, raised by the model config build. -fn model_error(model_name: &str, text: &str) -> Halt { - Halt::Config(ConfigError::compile(format!("model '{model_name}' {text}"))) -} - -type Built = Result; +const BUILD_STOPPED: &str = "a config value read as text without an error"; -/// The project-wide inputs every model's config build reads, captured once per compile. +/// The project-wide inputs every model's config build reads, converted once per compile. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct NativeModelConfigBuilder { - defaults: Py, - path_defaults: Py, + inputs: ConfigBuildInputs, path_keys: Vec, - hook_entry_types: Py, - variables: Py, environment: Py, - target_name: Option>, - run_id: Py, - target_namespace: Option<(Py, Py)>, } /// One model's built config before Python wraps it in `CompileModelConfig`. @@ -124,6 +64,71 @@ pub(crate) type TargetInputs<'py> = ( Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, ); +/// The process environment and project-var rendering, read through Python. +struct PythonConfigHost<'py> { + environment: Bound<'py, PyAny>, +} + +impl ConfigBuildHost for PythonConfigHost<'_> { + type Handle = HostHandle; + type Error = PyErr; + + fn environment(&self, name: &str) -> PyResult> { + if !self.environment.contains(name)? { + return Ok(None); + } + Ok(Some(config_node(&self.environment.get_item(name)?)?)) + } + + fn render(&self, value: &PyConfigNode, label: &str) -> PyResult> { + let py: Python<'_> = self.environment.py(); + let object: Bound<'_, PyAny> = config_object(py, value)?; + if let Ok(text) = object.downcast_exact::() { + return Ok(Ok(text.to_str()?.to_owned())); + } + match project_var_text(&object, label) { + Ok(text) => Ok(Ok(text)), + Err(error) if error.is_instance_of::(py) => { + Ok(Err(error.value(py).to_string())) + } + Err(error) => Err(error), + } + } + + fn text_error(&self, value: &PyConfigNode) -> PyErr { + let py: Python<'_> = self.environment.py(); + let read = || -> PyResult<()> { + let object: Bound<'_, PyAny> = config_object(py, value)?; + let text: Bound<'_, PyString> = match object.downcast::() { + Ok(text) => text.clone(), + Err(_) => object.str()?, + }; + let _ = text.to_str()?; + Ok(()) + }; + match read() { + Ok(()) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error, + } + } +} + +/// One header entry's parsed facts as the build reads them. +fn entry_facts((key, entry): &(String, EntryFacts)) -> (String, HeaderEntryFacts) { + let facts: HeaderEntryFacts = HeaderEntryFacts { + template: entry.template, + macro_path: entry.macro_path.clone(), + }; + (key.clone(), facts) +} + +fn optional_node(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + config_node(&value).map(Some) +} + #[pymethods] impl NativeModelConfigBuilder { #[new] @@ -133,25 +138,31 @@ impl NativeModelConfigBuilder { target: TargetInputs<'_>, ) -> PyResult { let (run, target_namespace) = target; - let (defaults, path_defaults, hook_entry_types) = layers; - let defaults: Bound<'_, PyDict> = project_defaults_mapping(&defaults)?; - let path_keys = path_defaults - .keys() - .iter() - .map(|key| key.extract::()) - .collect::>>()?; + let (defaults, path_defaults, _hook_entry_types) = layers; + let mut path_keys: Vec = Vec::with_capacity(path_defaults.len()); + let mut path_values: Vec<(String, PyConfigEntries)> = + Vec::with_capacity(path_defaults.len()); + for (key, values) in path_defaults.iter() { + let key: String = key.extract()?; + path_values.push((key.clone(), config_entries(values.downcast::()?)?)); + path_keys.push(key); + } let (target_name, run_id) = run; Ok(Self { - defaults: defaults.unbind(), - path_defaults: path_defaults.unbind(), + inputs: ConfigBuildInputs { + defaults: config_entries(&project_defaults_mapping(&defaults)?)?, + path_defaults: path_values, + variables: config_entries(&sources.0)?, + run_id: config_node(&run_id)?, + target_name: optional_node(target_name)?.unwrap_or(ConfigNode::Null), + target_namespace: target_namespace + .map(|(database, schema)| -> PyResult<_> { + Ok((config_node(&database)?, config_node(&schema)?)) + }) + .transpose()?, + }, path_keys, - hook_entry_types: hook_entry_types.unbind(), - variables: sources.0.unbind(), environment: sources.1.unbind(), - target_name: (!target_name.is_none()).then(|| target_name.unbind()), - run_id: run_id.unbind(), - target_namespace: target_namespace - .map(|(database, schema)| (database.unbind(), schema.unbind())), }) } @@ -176,7 +187,38 @@ impl NativeModelConfigBuilder { } } +/// The Python error a halt raises, or the config error it is rejected with. +fn halted<'py>( + py: Python<'py>, + halt: ConfigBuildHalt, +) -> PyResult> { + Ok(match halt { + ConfigBuildHalt::Config(error) => Ok(error), + ConfigBuildHalt::Host(error) => Err(error), + ConfigBuildHalt::MissingPathDefault(key) => Err(PyKeyError::new_err(key)), + ConfigBuildHalt::KeyNotText(key) => { + Err(match config_object(py, &key)?.downcast::() { + Ok(_) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error.into(), + }) + } + }) +} + +fn projected<'py>( + py: Python<'py>, + value: Option, +) -> PyResult>> { + value.map(|value| config_object(py, &value)).transpose() +} + impl NativeModelConfigBuilder { + fn host<'py>(&self, py: Python<'py>) -> PythonConfigHost<'py> { + PythonConfigHost { + environment: self.environment.bind(py).clone(), + } + } + /// The selected path-default key, or the conflict error Python raises. pub(crate) fn selected_path_default( &self, @@ -197,46 +239,60 @@ impl NativeModelConfigBuilder { matched_path_default: Option<&str>, model_name: &str, ) -> PyResult> { - match self.build_config(py, (header, facts), matched_path_default, model_name) { - Ok(built) => Ok(ConfigBuild::Built(built)), - Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), - Err(Halt::Error(error)) => Err(error), + let entries: PyConfigEntries = config_entries(header)?; + let facts: Option> = + facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); + let model_header = ModelHeader { + entries: &entries, + facts: facts.as_deref(), + }; + let host: PythonConfigHost<'py> = self.host(py); + match build_model_config( + (&host, &self.inputs), + model_header, + matched_path_default, + model_name, + ) { + Ok(built) => Ok(ConfigBuild::Built(self.projected_build(py, built)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), + Err(error) => Err(error), + }, } } + fn projected_build<'py>( + &self, + py: Python<'py>, + built: BuiltModelConfig, + ) -> PyResult> { + Ok(BuiltConfig { + values: config_dict(py, &built.values)?, + header_keys: built.header_keys, + logical_schema: projected(py, built.logical_schema)?, + layer_schema_configured: built.layer_schema_configured, + logical_database: projected(py, built.logical_database)?, + retention: built.retention, + table_type: built.table_type, + reads: built.reads, + }) + } + /// `build_layered_model_values` with an empty header: what a path default's models inherit. pub(crate) fn inherited_values<'py>( &self, py: Python<'py>, matched_path_default: Option<&str>, ) -> PyResult, Py>> { - let layered = (|| -> Built> { - let mut layered = self.project_defaults(py)?; - if let Some(key) = matched_path_default { - layered = merged_layer(py, &layered, &self.path_default_values(py, key)?)?; - } - merged_layer(py, &layered, &PyDict::new(py)) - })(); - match layered { - Ok(values) => Ok(Ok(values)), - Err(Halt::Config(error)) => Ok(Err(native_config_error(py, error)?)), - Err(Halt::Error(error)) => Err(error), + match inherited_config::(&self.inputs, matched_path_default) { + Ok(values) => Ok(Ok(config_dict(py, &values)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(Err(native_config_error(py, error)?)), + Err(error) => Err(error), + }, } } - /// `build_run_context_values`: the run's id and target. - fn run_context_values<'py>(&self, py: Python<'py>) -> PyResult> { - let context = PyDict::new(py); - context.set_item(RUN_ID_CONTEXT, self.run_id.bind(py))?; - context.set_item( - RUN_TARGET_CONTEXT, - self.target_name - .as_ref() - .map_or_else(|| py.None().into_bound(py), |item| item.bind(py).clone()), - )?; - Ok(context) - } - /// `build_model_context_values` for a model's config values. pub(crate) fn model_context_values<'py>( &self, @@ -244,334 +300,24 @@ impl NativeModelConfigBuilder { model_name: &str, include_target_values: bool, ) -> PyResult> { - let py: Python<'py> = values.py(); - let none = py.None().into_bound(py); - let database = string_entry(values, DATABASE_KEY)?; - let schema = string_entry(values, SCHEMA_KEY)?; - let alias = match string_entry(values, ALIAS_KEY)? { - Some(alias) => alias, - None => PyString::new(py, model_name).into_any(), - }; - let context = self.run_context_values(py)?; - context.set_item(MODEL_NAME_CONTEXT, model_name)?; - context.set_item(MODEL_DATABASE_CONTEXT, database.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_ALIAS_CONTEXT, &alias)?; - if !include_target_values { - return Ok(context); - } - let qualified = match (&database, &schema) { - (Some(database), Some(schema)) => Some(format!( - "{}.{}.{}", - text_of(database)?, - text_of(schema)?, - text_of(&alias)? - )), - (None, Some(schema)) => Some(format!("{}.{}", text_of(schema)?, text_of(&alias)?)), - _ => None, - }; - context.set_item( - DESTINATION_DATABASE_CONTEXT, - database.as_ref().unwrap_or(&none), - )?; - context.set_item(DESTINATION_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(DESTINATION_TABLE_CONTEXT, &alias)?; - context.set_item(DESTINATION_QUALIFIED_CONTEXT, qualified)?; - Ok(context) - } - - /// The active target's `(database, schema)`, or `(None, None)` without a target. - pub(crate) fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { - match &self.target_namespace { - Some((database, schema)) => PyTuple::new(py, [database.bind(py), schema.bind(py)]), - None => PyTuple::new(py, [py.None(), py.None()]), - } - } - - fn build_config<'py>( - &self, - py: Python<'py>, - input: HeaderInput<'_, 'py>, - matched_path_default: Option<&str>, - model_name: &str, - ) -> Built> { - let header: &Bound<'py, PyDict> = input.0; - check_header_tags(header, model_name)?; - let path_values = matched_path_default - .map(|key| self.path_default_values(py, key)) - .transpose()?; - let mut layered = self.project_defaults(py)?; - if let Some(path_values) = &path_values { - layered = merged_layer(py, &layered, path_values)?; - } - let layered = merged_layer(py, &layered, header)?; - self.check_hooks(py, &layered, model_name)?; - let mut hooks: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); - for key in HOOK_KEYS { - if let Some(value) = layered.get_item(key)? { - hooks.push((key, value)); - layered.del_item(key)?; - } - } - let has_templates = templated(input, &layered)?; - let mut expansion = Expansion { - builder: self, - py, + let context: PyConfigEntries = model_context_values( + &self.inputs, + &config_entries(values)?, model_name, - reads: Vec::new(), - }; - let model_values = if has_templates { - let early = expansion.expand( - &layered, - &self.variables.bind(py).clone(), - expansion.run_context()?, - (true, MODEL_CONFIG_LABEL), - )?; - let empty = PyDict::new(py); - let first = expansion.expand( - &early, - &empty, - expansion.model_context(&early, false)?, - (true, MODEL_CONFIG_LABEL), - )?; - expansion.expand( - &first, - &empty, - expansion.model_context(&first, false)?, - (true, MODEL_CONFIG_LABEL), - )? - } else { - layered - }; - let logical_schema = string_entry(&model_values, SCHEMA_KEY)?; - let logical_database = string_entry(&model_values, DATABASE_KEY)?; - let layer_schema_configured = header.contains(SCHEMA_KEY)? - || path_values - .as_ref() - .map(|values| values.contains(SCHEMA_KEY)) - .transpose()? - .unwrap_or(false); - let namespaced = - expansion.namespaced_values(&model_values, &logical_database, &logical_schema)?; - let needs_target_templates = has_templates - || [DATABASE_KEY, SCHEMA_KEY] - .iter() - .map(|key| entry_has_template(&namespaced, key)) - .collect::>>()? - .contains(&true); - let resolved = if needs_target_templates { - let empty = PyDict::new(py); - expansion.expand( - &namespaced, - &empty, - expansion.model_context(&namespaced, true)?, - (false, MODEL_CONFIG_LABEL), - )? - } else { - namespaced - }; - for (key, value) in hooks { - resolved.set_item(key, value)?; - } - let values = storage_free_values(py, &resolved, header)?; - let retention = retention_override( - header.get_item(RETENTION_KEY)?.map(PyNode).as_ref(), - model_name, - )?; - let table_type = table_type_override( - header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref(), - model_name, - )?; - check_no_config_macros(input, &values)?; - Ok(BuiltConfig { - values, - header_keys: sorted_keys(header)?, - logical_schema, - layer_schema_configured, - logical_database, - retention, - table_type, - reads: expansion.reads, - }) - } - - fn path_default_values<'py>(&self, py: Python<'py>, key: &str) -> Built> { - let values = self - .path_defaults - .bind(py) - .get_item(key)? - .ok_or_else(|| PyKeyError::new_err(key.to_owned()))?; - Ok(values.downcast_into::().map_err(PyErr::from)?) - } - - /// Return a fresh `project_defaults_to_mapping` result, with its own `tags` list. - fn project_defaults<'py>(&self, py: Python<'py>) -> Built> { - let defaults = self.defaults.bind(py).copy()?; - if let Some(tags) = defaults.get_item(TAGS_KEY)? { - let tags = tags.downcast_into::().map_err(PyErr::from)?; - defaults.set_item(TAGS_KEY, PyList::new(py, tags.iter())?)?; - } - Ok(defaults) - } - - /// Check hook keys as `validate_model_hook_config` does. - fn check_hooks( - &self, - py: Python<'_>, - values: &Bound<'_, PyDict>, - model_name: &str, - ) -> Built<()> { - let entry_types = self.hook_entry_types.bind(py); - for key in SORTED_HOOK_KEYS { - let Some(value) = values.get_item(key)? else { - continue; - }; - let Some(entries) = sequence_items(&value) else { - return Err(model_error( - model_name, - &format!("{key} must be a list of typed hook entries"), - )); - }; - for entry in entries { - if !entry.is_instance(entry_types)? { - return Err(model_error( - model_name, - &format!( - "{key} entries must use typed inline_sql(...), sql(...), or \ - python(...) hook syntax" - ), - )); - } - } - } - Ok(()) - } -} - -/// The template expansions of one model's build and the reads they recorded. -struct Expansion<'a, 'py> { - builder: &'a NativeModelConfigBuilder, - py: Python<'py>, - model_name: &'a str, - reads: Vec<(&'static str, String)>, -} - -impl<'py> Expansion<'_, 'py> { - /// Expand `values`; `scope` is `preserve_unknown_context` and the context label. - fn expand( - &mut self, - values: &Bound<'py, PyDict>, - variables: &Bound<'py, PyDict>, - context: Bound<'py, PyDict>, - scope: (bool, &str), - ) -> Built> { - let result = self.expand_value(values.as_any(), variables, context, scope)?; - Ok(result.downcast_into::().map_err(PyErr::from)?) - } - - fn expand_value( - &mut self, - value: &Bound<'py, PyAny>, - variables: &Bound<'py, PyDict>, - context: Bound<'py, PyDict>, - scope: (bool, &str), - ) -> Built> { - let (preserve_unknown_context, label) = scope; - let environment = self.builder.environment.bind(self.py).clone(); - let host = PythonHost::new( - self.py, - TemplateSources(variables.clone(), environment, context), - label, + include_target_values, ); - let options = TemplateOptions { - allow_context: true, - preserve_context_tokens: false, - preserve_unknown_context, - }; - let result = expanded(&host, value, options); - self.reads.extend(host.into_reads()); - result.map_err(|stop| match stop { - Stop::Failure(error) => Halt::Config(template_error(&error, label)), - Stop::Python(error) => Halt::Error(error), - }) + config_dict(values.py(), &context) } - fn run_context(&self) -> Built> { - Ok(self.builder.run_context_values(self.py)?) - } - - /// Return `build_model_context_values` for these values. - fn model_context( - &self, - values: &Bound<'py, PyDict>, - include_target_values: bool, - ) -> Built> { - Ok(self - .builder - .model_context_values(values, self.model_name, include_target_values)?) - } - - /// Check preserved namespaces and apply `apply_environment_database_schema_overrides`. - fn namespaced_values( - &mut self, - values: &Bound<'py, PyDict>, - logical_database: &Option>, - logical_schema: &Option>, - ) -> Built> { - let overridden = values.copy()?; - let Some((database, schema)) = &self.builder.target_namespace else { - return Ok(overridden); - }; - let py = self.py; - let namespace = [ - ( - DATABASE_KEY, - database.bind(py).clone(), - logical_database.is_some(), - ), - ( - SCHEMA_KEY, - schema.bind(py).clone(), - logical_schema.is_some(), + /// The active target's `(database, schema)`, or `(None, None)` without a target. + pub(crate) fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.inputs.target_namespace { + Some((database, schema)) => PyTuple::new( + py, + [config_object(py, database)?, config_object(py, schema)?], ), - ]; - let preserved = - |value: &Bound<'py, PyAny>| PyNode(value.clone()).is_text(PRESERVE_TARGET_VALUE); - let missing: Vec<&str> = namespace - .iter() - .filter(|(_, value, logical)| preserved(value) && !logical) - .map(|(key, _, _)| *key) - .collect(); - if !missing.is_empty() { - let dimensions = missing.join(" and "); - return Err(Halt::Config( - ConfigError::compile(format!( - "Model '{}' has no logical {dimensions}, but the selected target sets \ - {dimensions} to 'preserve'", - self.model_name - )) - .with_help(format!( - "Set {dimensions} on the resource or its defaults, or configure a literal \ - target {dimensions}." - )), - )); - } - let context = self.model_context(values, false)?; - for (key, value, _) in namespace { - if value.is_none() || preserved(&value) { - continue; - } - let label = if key == DATABASE_KEY { - ENVIRONMENT_DATABASE_LABEL - } else { - ENVIRONMENT_SCHEMA_LABEL - }; - let variables = self.builder.variables.bind(py).clone(); - let expanded_value = - self.expand_value(&value, &variables, context.copy()?, (false, label))?; - overridden.set_item(key, expanded_value)?; + None => PyTuple::new(py, [py.None(), py.None()]), } - Ok(overridden) } } @@ -587,266 +333,7 @@ pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { raised().unwrap_or_else(|error| error) } -/// Merge `overlay` over a copy of `base` as `_merged_with_tag_union` does. -fn merged_layer<'py>( - py: Python<'py>, - base: &Bound<'py, PyDict>, - overlay: &Bound<'py, PyDict>, -) -> Built> { - let result = base.copy()?; - result.update(overlay.as_mapping())?; - if let (Some(base_tags), Some(overlay_tags)) = - (set_entry(base, TAGS_KEY)?, set_entry(overlay, TAGS_KEY)?) - { - let tags = merged_strings(&base_tags, &overlay_tags)?; - result.set_item(TAGS_KEY, PyList::new(py, tags)?)?; - } - if let (Some(base_columns), Some(overlay_columns)) = ( - set_entry(base, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, - set_entry(overlay, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, - ) { - let columns = merged_strings(&base_columns, &overlay_columns)?; - result.set_item(ROW_DIFF_EXCLUDE_COLUMNS_KEY, PyTuple::new(py, columns)?)?; - } - if let (Some(base_tolerances), Some(overlay_tolerances)) = ( - set_entry(base, ROW_DIFF_TOLERANCES_KEY)?, - set_entry(overlay, ROW_DIFF_TOLERANCES_KEY)?, - ) { - result.set_item( - ROW_DIFF_TOLERANCES_KEY, - merged_tolerances(&base_tolerances, &overlay_tolerances)?, - )?; - } - Ok(result) -} - -/// Return the strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. -fn merged_strings(base: &Bound<'_, PyAny>, overlay: &Bound<'_, PyAny>) -> Built> { - let mut merged = string_items(base)?; - for item in string_items(overlay)? { - if !merged.contains(&item) { - merged.push(item); - } - } - Ok(merged) -} - -/// Return `_as_string_list(value)`: `str()` of each list or tuple item. -fn string_items(value: &Bound<'_, PyAny>) -> Built> { - let Some(items) = sequence_items(value) else { - return Ok(Vec::new()); - }; - items - .iter() - .map(|item| Ok(item.str()?.to_string_lossy().into_owned())) - .collect() -} - -/// Deep-merge row-diff tolerances by section as `_merge_row_diff_tolerances_mapping` does. -fn merged_tolerances<'py>( - base: &Bound<'py, PyAny>, - overlay: &Bound<'py, PyAny>, -) -> Built> { - let (Ok(base), Ok(overlay)) = (base.downcast::(), overlay.downcast::()) else { - return Ok(overlay.clone()); - }; - let merged = base.copy()?; - for section in AUDIT_OVERRIDE_SECTIONS { - let Some(overlay_section) = set_entry(overlay, section)? else { - continue; - }; - let base_section = set_entry(base, section)?; - if let Some(base_section) = &base_section - && let (Ok(base_section), Ok(overlay_mapping)) = ( - base_section.downcast::(), - overlay_section.downcast::(), - ) - { - let section_values = base_section.copy()?; - section_values.update(overlay_mapping.as_mapping())?; - merged.set_item(section, section_values)?; - } else { - merged.set_item(section, &overlay_section)?; - } - } - for (key, value) in overlay.iter() { - if key.is_instance_of::() - && !AUDIT_OVERRIDE_SECTIONS - .iter() - .any(|section| PyNode(key.clone()).is_text(section)) - { - merged.set_item(key, value)?; - } - } - Ok(merged.into_any()) -} - -/// Return `values.get(key)` unless it is absent or `None`. -fn set_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built>> { - Ok(values.get_item(key)?.filter(|value| !value.is_none())) -} - -/// Return the value when it is a string, as `isinstance(raw, str)` keeps it. -fn string_entry<'py>( - values: &Bound<'py, PyDict>, - key: &str, -) -> PyResult>> { - Ok(values - .get_item(key)? - .filter(pyo3::types::PyAnyMethods::is_instance_of::)) -} - -fn text_of(value: &Bound<'_, PyAny>) -> PyResult { - Ok(value.downcast::()?.to_string_lossy().into_owned()) -} - -fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option>> { - if let Ok(items) = value.downcast::() { - Some(items.iter().collect()) - } else if let Ok(items) = value.downcast::() { - Some(items.iter().collect()) - } else { - None - } -} - -fn entry_has_template(values: &Bound<'_, PyDict>, key: &str) -> Built { - Ok(values - .get_item(key)? - .is_some_and(|value| contains_template(&PyNode(value)))) -} - -/// Check header tags as `_validate_model_header_tags` does. -fn check_header_tags(header: &Bound<'_, PyDict>, model_name: &str) -> Built<()> { - let Some(tags) = set_entry(header, TAGS_KEY)? else { - return Ok(()); - }; - let Ok(tags) = tags.downcast::() else { - return Err(model_error(model_name, "tags must be a list")); - }; - if tags.iter().all(|tag| tag.is_instance_of::()) { - Ok(()) - } else { - Err(model_error(model_name, "tags entries must be strings")) - } -} - -/// Drop storage keys, then `full_refresh` where `build_model_config` drops it. -fn storage_free_values<'py>( - py: Python<'py>, - resolved: &Bound<'py, PyDict>, - header: &Bound<'py, PyDict>, -) -> Built> { - let values = PyDict::new(py); - for (key, value) in resolved.iter() { - let storage = [RETENTION_KEY, TABLE_TYPE_KEY] - .iter() - .any(|name| PyNode(key.clone()).is_text(name)); - if !storage { - values.set_item(key, value)?; - } - } - let incremental = values - .get_item(MATERIALIZED_KEY)? - .is_some_and(|value| PyNode(value).is_text(INCREMENTAL_MATERIALIZATION)); - if !header.contains(FULL_REFRESH_KEY)? && !incremental && values.contains(FULL_REFRESH_KEY)? { - values.del_item(FULL_REFRESH_KEY)?; - } - Ok(values) -} - -/// A model's authored header and, when it was read natively, the header's facts. -type HeaderInput<'a, 'py> = (&'a Bound<'py, PyDict>, Option<&'a HeaderFacts>); - -/// The facts of a config entry whose value is the header's own. -fn header_entry<'a, 'py>( - (header, facts): HeaderInput<'a, 'py>, - entry: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - match facts { - Some(facts) => own_entry(facts, header, entry), - None => Ok(None), - } -} - -/// `contains_template_data(values)`, reading the header's own entries from their facts. -fn templated<'py>(input: HeaderInput<'_, 'py>, values: &Bound<'py, PyDict>) -> Built { - for (key, value) in values.iter() { - let found: bool = match header_entry(input, (&key, &value))? { - Some(facts) => facts.template, - None => contains_template(&PyNode(value)), - }; - if found { - return Ok(true); - } - } - Ok(false) -} - -/// Raise for the first config field outside hooks holding a macro call, as Python's walk does. -fn check_no_config_macros<'py>( - input: HeaderInput<'_, 'py>, - values: &Bound<'py, PyDict>, -) -> Built<()> { - for (key, value) in values.iter() { - if HOOK_KEYS - .iter() - .any(|name| PyNode(key.clone()).is_text(name)) - { - continue; - } - let found: MacroPath = match header_entry(input, (&key, &value))? { - Some(facts) => facts.macro_path.clone(), - None => entry_macro_path(&PyNode(key), &PyNode(value)), - }; - if let MacroPath::Found(path) = found { - return Err(Halt::Config(ConfigError::compile(format!( - "model config field '{}' does not allow macros", - path.join(".") - )))); - } - } - Ok(()) -} - -fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { - let mut keys = header - .keys() - .iter() - .map(|key| { - Ok(key - .downcast::() - .map_err(PyErr::from)? - .to_string_lossy() - .into_owned()) - }) - .collect::>>()?; - keys.sort(); - Ok(keys) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) } - -/// An entry's facts only when its value is the header's own, so replaced values read from Python. -pub(crate) fn own_entry<'a, 'py>( - facts: &'a HeaderFacts, - header: &Bound<'py, PyDict>, - (key, value): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let Ok(name) = key.downcast::() else { - return Ok(None); - }; - let Ok(name) = name.to_str() else { - return Ok(None); - }; - let Some(entry) = facts.entry(name) else { - return Ok(None); - }; - Ok(header - .get_item(name)? - .filter(|own| own.is(value)) - .map(|_| entry)) -} diff --git a/fensu.toml b/fensu.toml index 9cd1674a94..7031d9fdb7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2379,7 +2379,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index b510731f3e..fe784b2954 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -33,3 +33,32 @@ class ColumnMetadataExpansionTestCase: header_column: str expected_type: str | None expected_error: str | None + + +@dataclass(frozen=True) +class ConfigValueBuildTestCase: + """A Python-given MODEL header and the config values the build gives the model.""" + + description: str + header: dict[object, object] + expected_values: tuple[tuple[str, str, str], ...] + expected_header_keys: tuple[str, ...] + + +@dataclass(frozen=True) +class ConfigValueRejectionTestCase: + """A Python-given MODEL header the config build rejects, and the error it raises.""" + + description: str + header: dict[object, object] + expected_error: tuple[str, str] + + +@dataclass(frozen=True) +class OpaqueConfigValueTestCase: + """A MODEL header holding an opaque value under `key`, which reaches the config unchanged.""" + + description: str + header: dict[object, object] + key: object + expected_same_object: bool diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 34f74aa3be..289a38dc0d 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -6,6 +6,7 @@ import re from collections.abc import Callable, Iterator from dataclasses import dataclass, fields, replace +from enum import StrEnum from pathlib import Path from typing import TYPE_CHECKING, TypeGuard, cast @@ -29,7 +30,7 @@ if TYPE_CHECKING: from _typeshed import DataclassInstance - from sqlbuild._native import CompileInputsView + from sqlbuild._native import CompileInputsView, CompileModelConfig _TEXTS: tuple[object, ...] = ( "INTEGER", @@ -533,3 +534,104 @@ def column_metadata_outcome(*, project_dir: Path, header_column: str) -> tuple[o tuple[object, ...], _entry_metadata(inputs.model_inputs[0].schema_entry)[0] ) return getattr(columns[0], "type", None), None + + +CONFIG_VALUE_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = "orders.duckdb"\n\n' + '[vars]\nregion = "north"\nratio = "0.5"\n\n' + '[defaults]\nmaterialized = "table"\ncontract = "enforced"\ntags = ["core"]\n\n' + '[path_defaults."marts"]\nrow_diff_exclude_columns = ["loaded_at"]\ntags = ["mart"]\n\n' + '[targets.dev]\nconnection = "local"\nschema = "${region}_${CTX:model.name}"\n' + ), + "models/marts/orders.sql": 'MODEL (description "Orders.");\n\nSELECT 1 AS order_id\n', +} + + +class Severity(StrEnum): + """A string enum whose members config values and keys hold as themselves.""" + + WARN = "warn" + + +class TagList(list[object]): + """A list subclass a Python caller may pass as a config value.""" + + +class SettingMap(dict[str, object]): + """A dict subclass a Python caller may pass as a config value.""" + + +class Marker: + """An opaque object the build carries through without reading it.""" + + def __repr__(self) -> str: + return "Marker()" + + +MARKER: Marker = Marker() + + +def config_value_inputs(*, project_dir: Path, header: dict[object, object]) -> CompileInputsView: + """Build compile inputs for the config value project with its one model's header replaced.""" + + for relative_path, contents in CONFIG_VALUE_PROJECT.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + model_file: DiscoveredSqlModelFile = discovered.model_files[0] + given: DiscoveredSqlModelFile = DiscoveredSqlModelFile( + file_path=model_file.file_path, + relative_path=model_file.relative_path, + contents=model_file.contents, + header_values=cast("dict[str, object]", header), + header_column_locations={}, + output_column_locations={}, + query_sql=model_file.query_sql, + ) + adapter: DuckDbAdapter = DuckDbAdapter() + return build_compile_inputs( + discovered_inputs=replace(discovered, model_files=(given,)), + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="20261007T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + + +def config_value_outcome( + *, project_dir: Path, header: dict[object, object] +) -> tuple[tuple[tuple[str, str, str], ...], tuple[str, ...]]: + """The model's config values as `(key repr, value type, value repr)` and its header keys.""" + + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header).model_inputs[0].config + ) + return ( + tuple( + (repr(key), type(value).__name__, repr(value)) for key, value in config.values.items() + ), + config.model_header_keys, + ) + + +def marker_kept(*, project_dir: Path, header: dict[object, object], key: object) -> bool: + """Whether the opaque value under `key` reaches the model's config values as the same object.""" + + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header).model_inputs[0].config + ) + return config.values[cast("str", key)] is MARKER diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py b/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py new file mode 100644 index 0000000000..5d98dc6ff1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_config_values.py @@ -0,0 +1,213 @@ +"""Config values the native build gives models: builtin, subclassed and opaque Python values.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + ConfigValueBuildTestCase, + ConfigValueRejectionTestCase, + OpaqueConfigValueTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( + MARKER, + SettingMap, + Severity, + TagList, + config_value_inputs, + config_value_outcome, + marker_kept, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigValueBuildTestCase( + description="builtin values, unions and templates", + header={ + "description": "Orders.", + "tags": ["${region}-${ratio}", "core"], + "row_diff_exclude_columns": ["updated_at"], + "table_settings": {"owner": "${CTX:model.name}", "weight": 10**30, "enabled": True}, + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart', 'north-0.5']"), + ("'row_diff_exclude_columns'", "tuple", "('loaded_at', 'updated_at')"), + ( + "'table_settings'", + "dict", + "{'owner': 'orders', 'weight': 1000000000000000000000000000000, 'enabled': True}", + ), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=( + "description", + "row_diff_exclude_columns", + "table_settings", + "tags", + ), + ), + ConfigValueBuildTestCase( + description="subclasses, str enums, dates and opaque values", + header={ + "description": "Orders.", + "tags": TagList(["${region}", Severity.WARN]), + "table_settings": SettingMap(owner="${CTX:model.name}", level=Severity.WARN), + "row_diff_tolerances": {"by_type": {"DOUBLE": 0.5}, "default": date(2024, 1, 2)}, + Severity.WARN: MARKER, + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart', 'north', 'warn']"), + ("'row_diff_exclude_columns'", "list", "['loaded_at']"), + ( + "'table_settings'", + "dict", + "{'owner': 'orders', 'level': }", + ), + ( + "'row_diff_tolerances'", + "dict", + "{'by_type': {'DOUBLE': 0.5}, 'default': datetime.date(2024, 1, 2)}", + ), + ("", "Marker", "Marker()"), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=( + "description", + "row_diff_tolerances", + "table_settings", + "tags", + "warn", + ), + ), + ConfigValueBuildTestCase( + description="target context and whole-value templates", + header={ + "database": "db", + "alias": "${CTX:destination.qualified}", + "row_diff_sample_seed": "${ratio}", + }, + expected_values=( + ("'materialized'", "str", "'table'"), + ("'contract'", "str", "'enforced'"), + ("'tags'", "list", "['core', 'mart']"), + ("'row_diff_exclude_columns'", "list", "['loaded_at']"), + ("'database'", "str", "'db'"), + ("'alias'", "str", "'db.north_orders.${CTX:destination.qualified}'"), + ("'row_diff_sample_seed'", "str", "'0.5'"), + ("'schema'", "str", "'north_orders'"), + ), + expected_header_keys=("alias", "database", "row_diff_sample_seed"), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_given_header_when_building_config_then_values_keep_types_and_order( + test_case: ConfigValueBuildTestCase, tmp_path: Path +) -> None: + outcome: tuple[tuple[tuple[str, str, str], ...], tuple[str, ...]] = config_value_outcome( + project_dir=tmp_path / "orders", header=test_case.header + ) + + assert outcome == (test_case.expected_values, test_case.expected_header_keys) + + +@pytest.mark.parametrize( + "test_case", + [ + ConfigValueRejectionTestCase( + description="a lone surrogate in a templated config", + header={"description": "x\udc80", "alias": "${region}"}, + expected_error=( + "UnicodeEncodeError", + "'utf-8' codec can't encode character '\\udc80' in position 1: surrogates not allowed", + ), + ), + ConfigValueRejectionTestCase( + description="a hook list of plain strings", + header={"pre_hooks": ["select 1"]}, + expected_error=( + "CompileInputError", + "model 'orders' pre_hooks entries must use typed inline_sql(...), sql(...), or python(...) hook syntax", + ), + ), + ConfigValueRejectionTestCase( + description="a missing hook list", + header={"post_hooks": None}, + expected_error=( + "CompileInputError", + "model 'orders' post_hooks must be a list of typed hook entries", + ), + ), + ConfigValueRejectionTestCase( + description="a non-string tag", + header={"tags": ["a", 1]}, + expected_error=("CompileInputError", "model 'orders' tags entries must be strings"), + ), + ConfigValueRejectionTestCase( + description="a non-string header key", + header={1: "x"}, + expected_error=("TypeError", "'int' object cannot be converted to 'PyString'"), + ), + ConfigValueRejectionTestCase( + description="a nested macro call", + header={"table_settings": {"a": ["@x ("]}}, + expected_error=( + "CompileInputError", + "model config field 'table_settings.a' does not allow macros", + ), + ), + ConfigValueRejectionTestCase( + description="an unknown variable", + header={"alias": "${missing}"}, + expected_error=( + "CompileInputError", + "model config references unknown variable 'missing'", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_header_when_building_config_then_the_build_raises_as_before( + test_case: ConfigValueRejectionTestCase, tmp_path: Path +) -> None: + with pytest.raises(Exception) as raised: + config_value_inputs(project_dir=tmp_path / "orders", header=test_case.header) + + assert (type(raised.value).__name__, str(raised.value)) == test_case.expected_error + + +@pytest.mark.parametrize( + "test_case", + [ + OpaqueConfigValueTestCase( + description="an object under a str-enum key", + header={"description": "x", Severity.WARN: MARKER}, + key=Severity.WARN, + expected_same_object=True, + ), + OpaqueConfigValueTestCase( + description="an object beside templated values", + header={"alias": "${region}", "owner": MARKER}, + key="owner", + expected_same_object=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_opaque_config_value_when_building_config_then_the_same_object_reaches_the_model( + test_case: OpaqueConfigValueTestCase, tmp_path: Path +) -> None: + kept: bool = marker_kept( + project_dir=tmp_path / "orders", header=test_case.header, key=test_case.key + ) + + assert kept == test_case.expected_same_object From d71a88f864d4ab75fae8bdbd24dd5358a5bccf2f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:05:11 +0100 Subject: [PATCH 5/9] refactor(native): read planner config values as typed nodes `sqlbuild-model-config` gains the typed config reads (`checked_text`, `checked_flag`, `checked_integer`, `checked_texts`, `checked_cursor_bound`) with the type rules and error contents of the `get_config_*` readers; the Python-facing readers now check through them and still return the caller's own objects. A config record keeps its values as typed nodes, converted from `values` on first read and cached on the record. The planner's phase objects, change detection, display plan, column migrations, lineage and SQL test cursor window read through it instead of the Python dict. --- .../src/config_values/_helpers/checks.rs | 22 +++ .../src/config_values/_helpers/mod.rs | 1 + .../src/config_values/constants.rs | 5 + .../src/config_values/errors.rs | 11 ++ .../main/checked_cursor_bound.rs | 24 ++++ .../src/config_values/main/checked_flag.rs | 21 +++ .../src/config_values/main/checked_integer.rs | 21 +++ .../src/config_values/main/checked_text.rs | 21 +++ .../src/config_values/main/checked_texts.rs | 26 ++++ .../src/config_values/main/mod.rs | 5 + .../src/config_values/mod.rs | 1 + .../src/config_values/models.rs | 7 + .../src/config_values/tests.rs | 2 + .../src/config_values/tests/helpers.rs | 54 +++++++- .../config_values/tests/test_checked_reads.rs | 118 ++++++++++++++++ .../src/config_values/tests/test_types.rs | 11 ++ .../_helpers/model_config/config_values.rs | 111 ++++++--------- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../_helpers/model_config/typed_config.rs | 130 ++++++++++++++++++ .../_helpers/planner_changes/detect.rs | 9 +- .../_helpers/planner_entries/display_plan.rs | 10 +- .../planner_migrations/column_objects.rs | 10 +- .../_helpers/planner_phases/objects.rs | 9 +- .../_helpers/planner_strategy/lineage.rs | 9 +- .../_helpers/sql_test_glue/cursor_window.rs | 8 +- fensu.toml | 3 +- 26 files changed, 546 insertions(+), 104 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/errors.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/checked_text.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs create mode 100644 crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs new file mode 100644 index 0000000000..98fb958caf --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/checks.rs @@ -0,0 +1,22 @@ +//! The authored value of a config key and the type error a reader raises for it. + +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; + +/// `values.get(key)` unless it is `None`. +pub(crate) fn authored(value: Option<&ConfigNode>) -> Option<&ConfigNode> { + value.filter(|value| !matches!(value, ConfigNode::Null)) +} + +/// The error naming `key`, the `expected` kind and the value found. +pub(crate) fn wrong_type<'a, H>( + key: &str, + expected: &'static str, + value: &'a ConfigNode, +) -> ConfigTypeError<'a, H> { + ConfigTypeError { + key: key.to_owned(), + expected, + value, + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs index a86c4d913f..42fe2ead92 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod authored; +pub(crate) mod checks; pub(crate) mod entries; pub(crate) mod reprs; pub(crate) mod unions; diff --git a/crates/sqlbuild-model-config/src/config_values/constants.rs b/crates/sqlbuild-model-config/src/config_values/constants.rs index 8dda7c39eb..7eff286ac1 100644 --- a/crates/sqlbuild-model-config/src/config_values/constants.rs +++ b/crates/sqlbuild-model-config/src/config_values/constants.rs @@ -8,3 +8,8 @@ pub(crate) const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns" pub(crate) const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; /// The `row_diff_tolerances` sections merged one level deeper than the rest. pub(crate) const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; +pub(crate) const EXPECTED_TEXT: &str = "a string"; +pub(crate) const EXPECTED_FLAG: &str = "a boolean"; +pub(crate) const EXPECTED_TEXTS: &str = "a list or tuple of strings"; +pub(crate) const EXPECTED_INTEGER: &str = "an integer"; +pub(crate) const EXPECTED_CURSOR_BOUND: &str = "a string, integer, date, or datetime"; diff --git a/crates/sqlbuild-model-config/src/config_values/errors.rs b/crates/sqlbuild-model-config/src/config_values/errors.rs new file mode 100644 index 0000000000..389606e98b --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/errors.rs @@ -0,0 +1,11 @@ +//! The error a typed config read returns. + +use crate::config_values::models::ConfigNode; + +/// A config value of the wrong type: its key, the kind expected, and the value found. +#[derive(Clone, Debug, PartialEq)] +pub struct ConfigTypeError<'a, H> { + pub key: String, + pub expected: &'static str, + pub value: &'a ConfigNode, +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs new file mode 100644 index 0000000000..d22d400e09 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs @@ -0,0 +1,24 @@ +//! A config cursor bound read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_CURSOR_BOUND; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::{ConfigNode, CursorBound}; +use crate::types::{AuthoredNode, NodeKind}; + +/// How `key` reads as a cursor bound: a string, a non-boolean integer, a date or a datetime. +pub fn checked_cursor_bound<'a, H: Clone>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + if value.is_date_like() { + return Ok(Some(CursorBound::Dated(value))); + } + match value.kind() { + NodeKind::Str | NodeKind::Int { .. } => Ok(Some(CursorBound::Stringified(value))), + _ => Err(wrong_type(key, EXPECTED_CURSOR_BOUND, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs new file mode 100644 index 0000000000..f964a8cfde --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs @@ -0,0 +1,21 @@ +//! A config boolean read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_FLAG; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// The boolean value of `key`, or `None` when absent or `None`. +pub fn checked_flag<'a, H: Clone>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Bool(flag) => Ok(Some(flag)), + _ => Err(wrong_type(key, EXPECTED_FLAG, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs new file mode 100644 index 0000000000..15c107a95e --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs @@ -0,0 +1,21 @@ +//! A config integer read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_INTEGER; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// The value of `key` when it is an integer and not a boolean, or `None` when absent or `None`. +pub fn checked_integer<'a, H: Clone>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Int { .. } => Ok(Some(value)), + _ => Err(wrong_type(key, EXPECTED_INTEGER, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs new file mode 100644 index 0000000000..eeb5c359cb --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs @@ -0,0 +1,21 @@ +//! A config string read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_TEXT; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// The value of `key` when it is a string (a present empty string included), or `None` when absent or `None`. +pub fn checked_text<'a, H: Clone>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + match value.kind() { + NodeKind::Str => Ok(Some(value)), + _ => Err(wrong_type(key, EXPECTED_TEXT, value)), + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs new file mode 100644 index 0000000000..5e2dd5a2de --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs @@ -0,0 +1,26 @@ +//! A config sequence of strings read with its type checked. + +use crate::config_values::_helpers::checks::{authored, wrong_type}; +use crate::config_values::constants::EXPECTED_TEXTS; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::ConfigNode; +use crate::types::{AuthoredNode, NodeKind}; + +/// The string items of `key` when it is a list or tuple of strings, or `None` when absent or `None`. +pub fn checked_texts<'a, H: Clone>( + key: &str, + value: Option<&'a ConfigNode>, +) -> Result>>, ConfigTypeError<'a, H>> { + let Some(value) = authored(value) else { + return Ok(None); + }; + let items: Vec<&ConfigNode> = match value.kind() { + NodeKind::List | NodeKind::Tuple => (&value).items(), + _ => return Err(wrong_type(key, EXPECTED_TEXTS, value)), + }; + if items.iter().all(|item| item.kind() == NodeKind::Str) { + Ok(Some(items)) + } else { + Err(wrong_type(key, EXPECTED_TEXTS, value)) + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/mod.rs b/crates/sqlbuild-model-config/src/config_values/main/mod.rs index e4d117250c..b424fdd584 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/mod.rs @@ -1,3 +1,8 @@ +pub mod checked_cursor_bound; +pub mod checked_flag; +pub mod checked_integer; +pub mod checked_text; +pub mod checked_texts; pub mod config_entry; pub mod inserted_entry; pub mod layered_config; diff --git a/crates/sqlbuild-model-config/src/config_values/mod.rs b/crates/sqlbuild-model-config/src/config_values/mod.rs index 4d2841b6a6..b326b8b35a 100644 --- a/crates/sqlbuild-model-config/src/config_values/mod.rs +++ b/crates/sqlbuild-model-config/src/config_values/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; +pub mod errors; pub mod main; pub mod models; pub mod types; diff --git a/crates/sqlbuild-model-config/src/config_values/models.rs b/crates/sqlbuild-model-config/src/config_values/models.rs index b0e9e6f3bb..db0a1ed00c 100644 --- a/crates/sqlbuild-model-config/src/config_values/models.rs +++ b/crates/sqlbuild-model-config/src/config_values/models.rs @@ -62,3 +62,10 @@ pub struct LayeredConfig { /// The keys both layers set whose merged value is a union of the two. pub unioned_keys: Vec<&'static str>, } + +/// How a cursor bound reads as text: its `str()`, or its `isoformat()` for a date or datetime. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum CursorBound<'a, H> { + Stringified(&'a ConfigNode), + Dated(&'a ConfigNode), +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests.rs b/crates/sqlbuild-model-config/src/config_values/tests.rs index b3a5a4a454..8613a52d31 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_checked_reads.rs"] +mod checked_reads; #[path = "tests/test_config_entry.rs"] mod config_entry; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs index c5ecb1a8df..120e2a05cf 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs @@ -1,4 +1,5 @@ -use crate::config_values::models::{ConfigNode, HostChildren, HostFacts, HostValue}; +use crate::config_values::errors::ConfigTypeError; +use crate::config_values::models::{ConfigNode, CursorBound, HostChildren, HostFacts, HostValue}; use crate::config_values::tests::test_types::Node; use crate::types::NodeKind; @@ -31,3 +32,54 @@ pub(super) fn host_text(value: &str) -> Node { pub(super) fn texts(values: &[&str]) -> Node { ConfigNode::List(values.iter().map(|value| text(value)).collect()) } + +/// A `datetime.date` host value. +pub(super) fn host_date() -> Node { + ConfigNode::Host(Box::new(HostValue { + handle: 2, + facts: HostFacts { + kind: NodeKind::Other, + text: None, + strict_text: false, + integer: None, + number: None, + python_str: "2024-01-02".to_owned(), + strict_str: true, + python_repr: "datetime.date(2024, 1, 2)".to_owned(), + date_like: true, + hook_entry: false, + }, + children: HostChildren::None, + })) +} + +/// `value`, `none`, or the expected kind of a typed read's outcome. +pub(super) fn outcome(result: Result, ConfigTypeError<'_, u8>>) -> String { + result.map_or_else( + |error| error.expected.to_owned(), + |value| value.map_or("none", |_| "value").to_owned(), + ) +} + +/// `dated`, `stringified`, `none`, or the expected kind of a cursor bound read. +pub(super) fn cursor_outcome( + result: Result>, ConfigTypeError<'_, u8>>, +) -> String { + result.map_or_else( + |error| error.expected.to_owned(), + |bound| { + let kinds = [ + ("dated", matches!(bound, Some(CursorBound::Dated(_)))), + ( + "stringified", + matches!(bound, Some(CursorBound::Stringified(_))), + ), + ]; + kinds + .iter() + .find(|(_, found)| *found) + .map_or("none", |(kind, _)| kind) + .to_owned() + }, + ) +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs new file mode 100644 index 0000000000..7839c18087 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_checked_reads.rs @@ -0,0 +1,118 @@ +use crate::config_values::main::checked_cursor_bound::checked_cursor_bound; +use crate::config_values::main::checked_flag::checked_flag; +use crate::config_values::main::checked_integer::checked_integer; +use crate::config_values::main::checked_text::checked_text; +use crate::config_values::main::checked_texts::checked_texts; +use crate::config_values::models::ConfigNode; +use crate::config_values::tests::helpers::{ + cursor_outcome, host_date, host_text, outcome, text, texts, +}; +use crate::config_values::tests::test_types::CheckedReadTestCase; + +#[test] +fn given_config_values_when_read_as_typed_values_then_types_are_checked_as_python_does() { + let test_cases = [ + CheckedReadTestCase { + description: "an absent or null value reads as none", + value: Some(ConfigNode::Null), + expected_text: "none", + expected_flag: "none", + expected_integer: "none", + expected_texts: "none", + expected_cursor_bound: "none", + }, + CheckedReadTestCase { + description: "a present empty string is a string", + value: Some(text("")), + expected_text: "value", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a str-enum member is a string", + value: Some(host_text("daily")), + expected_text: "value", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a boolean is not an integer or a cursor bound", + value: Some(ConfigNode::Bool(true)), + expected_text: "a string", + expected_flag: "value", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "an integer reads as an integer and a stringified bound", + value: Some(ConfigNode::Int(-4)), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "value", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "stringified", + }, + CheckedReadTestCase { + description: "a date is a dated cursor bound only", + value: Some(host_date()), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "dated", + }, + CheckedReadTestCase { + description: "a tuple of strings is a string sequence", + value: Some(ConfigNode::Tuple(vec![text("a"), host_text("b")])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "value", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "a list holding a non-string is not a string sequence", + value: Some(ConfigNode::List(vec![text("a"), ConfigNode::Int(1)])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "a list or tuple of strings", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + CheckedReadTestCase { + description: "a list of strings is a string sequence", + value: Some(texts(&["a"])), + expected_text: "a string", + expected_flag: "a boolean", + expected_integer: "an integer", + expected_texts: "value", + expected_cursor_bound: "a string, integer, date, or datetime", + }, + ]; + for test_case in test_cases { + let value = test_case.value.as_ref(); + assert_eq!( + [ + outcome(checked_text("key", value)), + outcome(checked_flag("key", value)), + outcome(checked_integer("key", value)), + outcome(checked_texts("key", value)), + cursor_outcome(checked_cursor_bound("key", value)), + ], + [ + test_case.expected_text, + test_case.expected_flag, + test_case.expected_integer, + test_case.expected_texts, + test_case.expected_cursor_bound, + ], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs index 22914ef174..8e08ddb0e6 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs @@ -24,3 +24,14 @@ pub(super) struct ConfigEntryTestCase { pub(super) key: &'static str, pub(super) expected_value: Option, } + +/// One config value and how each typed reader reads it: `value`, `none`, or the expected kind. +pub(super) struct CheckedReadTestCase { + pub(super) description: &'static str, + pub(super) value: Option, + pub(super) expected_text: &'static str, + pub(super) expected_flag: &'static str, + pub(super) expected_integer: &'static str, + pub(super) expected_texts: &'static str, + pub(super) expected_cursor_bound: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index 2bc2ff7200..cc1da29c7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -1,29 +1,23 @@ //! Typed values read from raw authored configuration mappings, rejecting the wrong types. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyBool, PyInt, PyList, PyString, PyTuple}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; +use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; +use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; +use sqlbuild_model_config::config_values::models::CursorBound; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_node}; +use crate::bindings::_helpers::model_config::typed_config::type_error; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::ErrorId; -const DATETIME_MODULE: &str = "datetime"; - -/// `values.get(key)`, or `None` when the value is absent or `None`. -fn authored<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { - let value: Bound<'py, PyAny> = mapping_get(values, key)?; - Ok(Some(value).filter(|value| !value.is_none())) -} - -/// `ConfigValueTypeError(key=key, expected=expected, actual_type=type(value))`. -fn wrong_type(key: &str, expected: &str, value: &Bound<'_, PyAny>) -> PyErr { - match config_value_type_error(key, expected, &value.get_type().into_any()) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} +const ISOFORMAT_METHOD: &str = "isoformat"; /// The `ConfigValueTypeError` naming `key`, the `expected` kind and the value's `actual_type`. pub(crate) fn config_value_type_error<'py>( @@ -51,6 +45,15 @@ pub(crate) fn config_value_type_error<'py>( ) } +/// `values.get(key)` as a config node. +fn authored_node<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult<(PyConfigNode, Bound<'py, PyAny>)> { + let value = mapping_get(values, key)?; + Ok((config_node(&value)?, value)) +} + /// An authored string, preserving a present empty string. #[pyfunction(name = "get_config_str")] #[pyo3(signature = (*, values, key))] @@ -58,13 +61,10 @@ pub(crate) fn config_str<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if !value.is_instance_of::() { - return Err(wrong_type(key, "a string", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// An authored boolean, or `None` when the key is absent or `None`. @@ -74,13 +74,10 @@ pub(crate) fn config_bool<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if !value.is_instance_of::() { - return Err(wrong_type(key, "a boolean", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_flag(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// An authored sequence of strings as a tuple, or `None` when absent or `None`. @@ -90,16 +87,13 @@ pub(crate) fn config_string_tuple<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { + let (node, value) = authored_node(values, key)?; + let checked = + checked_texts(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + if checked.is_none() { return Ok(None); - }; - if !(value.is_instance_of::() || value.is_instance_of::()) { - return Err(wrong_type(key, "a list or tuple of strings", &value)); } let items: Vec> = value.try_iter()?.collect::>()?; - if !items.iter().all(|item| item.is_instance_of::()) { - return Err(wrong_type(key, "a list or tuple of strings", &value)); - } PyTuple::new(values.py(), items).map(Some) } @@ -110,13 +104,10 @@ pub(crate) fn config_int<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - if value.is_instance_of::() || !value.is_instance_of::() { - return Err(wrong_type(key, "an integer", &value)); - } - Ok(Some(value)) + let (node, value) = authored_node(values, key)?; + let checked = + checked_integer(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } /// A cursor bound as text from a string, integer, date or datetime. @@ -126,26 +117,14 @@ pub(crate) fn config_cursor_bound<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - let Some(value) = authored(values, key)? else { - return Ok(None); - }; - let py = values.py(); - let date: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "date")?; - let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - let dated: bool = value.is_instance(&date)? || value.is_instance(&datetime)?; - if value.is_instance_of::() - || !(dated || value.is_instance_of::() || value.is_instance_of::()) - { - return Err(wrong_type( - key, - "a string, integer, date, or datetime", - &value, - )); - } - if dated { - return value.call_method0("isoformat").map(Some); - } - value.str().map(|text| Some(text.into_any())) + let (node, value) = authored_node(values, key)?; + let checked = + checked_cursor_bound(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(match checked { + None => None, + Some(CursorBound::Dated(_)) => Some(value.call_method0(ISOFORMAT_METHOD)?), + Some(CursorBound::Stringified(_)) => Some(value.str()?.into_any()), + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 9f9c2b0494..4ebfd08885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -16,4 +16,5 @@ pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; +pub(crate) mod typed_config; pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs new file mode 100644 index 0000000000..997e7fcc5c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs @@ -0,0 +1,130 @@ +//! Model config values read as typed config nodes, kept on the config record once converted. + +use std::sync::Arc; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyclass; +use pyo3::types::PyDict; +use sqlbuild_model_config::config_values::errors::ConfigTypeError; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::main::config_entry::config_entry; +use sqlbuild_model_config::config_values::models::ConfigNode; + +use crate::bindings::_helpers::model_config::config_nodes::{ + HostHandle, PyConfigEntries, PyConfigNode, config_entries, config_node, config_object, +}; +use crate::bindings::_helpers::model_config::config_values::config_value_type_error; +use crate::bindings::_helpers::records::fields::{ + cached_value, field as native_field, store_cached, +}; +use crate::bindings::_helpers::records::models::NativeRecord; + +const CONFIG_FIELD: &str = "config"; +const VALUES_FIELD: &str = "values"; +/// The config record's cache slot holding its typed values; no Python attribute reads it. +const TYPED_VALUES_SLOT: &str = "__typed_values__"; + +/// A config record's values as typed config nodes. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TypedConfigValues { + entries: Arc, +} + +/// The entries of a values mapping: a `dict` read directly, any other mapping through `items()`. +fn mapping_entries(values: &Bound<'_, PyAny>) -> PyResult { + if let Ok(mapping) = values.cast::() { + return config_entries(mapping); + } + values + .call_method0("items")? + .try_iter()? + .map(|item| { + let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + Ok((config_node(&key)?, config_node(&value)?)) + }) + .collect() +} + +/// Keep `entries` as the typed values of the config record `config`. +pub(crate) fn keep_typed_values( + config: &Bound<'_, PyAny>, + entries: Arc, +) -> PyResult<()> { + let Ok(record) = config.cast::() else { + return Ok(()); + }; + let typed: Py = Py::new(config.py(), TypedConfigValues { entries })?; + store_cached(record, TYPED_VALUES_SLOT, &typed.into_any()) +} + +/// A config record's values as typed nodes, converted from its `values` once and kept. +pub(crate) fn typed_values(config: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = config.py(); + let Ok(record) = config.cast::() else { + return Ok(Arc::new(mapping_entries(&native_field( + config, + VALUES_FIELD, + )?)?)); + }; + if let Some(typed) = cached_value(record, TYPED_VALUES_SLOT)? { + let typed: Bound<'_, TypedConfigValues> = typed.into_bound(py).cast_into()?; + return Ok(Arc::clone(&typed.get().entries)); + } + let entries: Arc = + Arc::new(mapping_entries(&native_field(config, VALUES_FIELD)?)?); + keep_typed_values(config, Arc::clone(&entries))?; + Ok(entries) +} + +/// A model's config values as typed nodes. +pub(crate) fn model_values(model: &Bound<'_, PyAny>) -> PyResult> { + typed_values(&native_field(model, CONFIG_FIELD)?) +} + +/// The `ConfigValueTypeError` a typed read raises, naming the value's Python type. +pub(crate) fn type_error(py: Python<'_>, error: &ConfigTypeError<'_, HostHandle>) -> PyErr { + let raised = || -> PyResult { + let actual_type = config_object(py, error.value)?.get_type().into_any(); + Ok(PyErr::from_value(config_value_type_error( + &error.key, + error.expected, + &actual_type, + )?)) + }; + raised().unwrap_or_else(|error| error) +} + +/// The exact text of a string node; a string Python cannot encode raises as reading it does. +pub(crate) fn node_text(py: Python<'_>, node: &PyConfigNode) -> PyResult { + match node { + ConfigNode::Str(text) => Ok(text.clone()), + ConfigNode::Host(host) if host.facts.strict_text => { + Ok(host.facts.text.clone().unwrap_or_default()) + } + _ => config_object(py, node)?.extract(), + } +} + +/// `key`'s string value, or `None` when absent or `None`. +pub(crate) fn config_text( + py: Python<'_>, + entries: &[(PyConfigNode, PyConfigNode)], + key: &str, +) -> PyResult> { + checked_text(key, config_entry(entries, key)) + .map_err(|error| type_error(py, &error))? + .map(|value| node_text(py, value)) + .transpose() +} + +/// `key`'s string value as its Python object, or `None` when absent or `None`. +pub(crate) fn config_text_object<'py>( + py: Python<'py>, + entries: &[(PyConfigNode, PyConfigNode)], + key: &str, +) -> PyResult>> { + checked_text(key, config_entry(entries, key)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 0fa98dc16f..e9ca93cb6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -4,7 +4,7 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -437,12 +437,7 @@ pub(crate) fn model_change<'py>( /// The model's resolved `replay_on_change` policy. fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = model.py(); - let raw: Option = authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - REPLAY_CONFIG_KEY, - )? - .map(|value| value.extract()) - .transpose()?; + let raw: Option = config_text(py, &model_values(model)?, REPLAY_CONFIG_KEY)?; resolve_replay_on_change(raw.as_deref()).map_err(|error| replay_policy_error(py, error)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index 8c91a000be..caf8713de1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,7 +1,7 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; @@ -41,11 +41,9 @@ fn display_action( } fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - Ok(authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .unwrap_or_else(|| model.py().None().into_bound(model.py()))) + let py = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) } fn display_entry<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 58f2542027..3aaab66766 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,7 +1,7 @@ //! Boundary conversions for the column migration planner. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::records::construct::new_record_with; @@ -47,11 +47,9 @@ pub(super) fn native<'py>( } pub(super) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - Ok(authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .unwrap_or_else(|| model.py().None().into_bound(model.py()))) + let py = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) } pub(super) fn definition_hash<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index cf6f809bba..4376b8d325 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -3,7 +3,7 @@ use sqlbuild_host::entries::models::HostId; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords}; -use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{enum_member, new_record, new_record_with}; @@ -159,12 +159,7 @@ pub(crate) fn by_name<'py>( /// A model config value as text, `None` when absent or not text. pub(crate) fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { - authored_config_str( - &native_field(&native_field(model, "config")?, "values")?, - key, - )? - .map(|value| value.extract()) - .transpose() + config_text(model.py(), &model_values(model)?, key) } /// The compiled object key of the model `name`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 9976607ea7..860c4a660e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -49,11 +49,8 @@ fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult> /// Whether a model's `incremental_mode` is microbatch. fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { - let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; - Ok(match config_str(&values, "incremental_mode")? { - Some(mode) => mode.eq(MICROBATCH_MODE)?, - None => false, - }) + let mode: Option = config_text(model.py(), &model_values(model)?, "incremental_mode")?; + Ok(mode.as_deref() == Some(MICROBATCH_MODE)) } /// A model's watermark input names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index 3807ce7aab..e7976ddc8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; -use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::records::construct::{enum_from_value, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -238,8 +238,8 @@ fn ordering(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py = model.py(); - let values = native_field(&native_field(model, "config")?, "values")?; - let declared = config_str(&values, "cursor_type")?.filter(|value| { + let values = model_values(model)?; + let declared = config_text_object(py, &values, "cursor_type")?.filter(|value| { value .extract::() .is_ok_and(|value| !value.is_empty()) @@ -248,7 +248,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { declared.unwrap_or_else(|| PyString::new(py, TIMESTAMP_CURSOR_TYPE).into_any()), ))?; let type_name: String = native_field(&cursor_type, "value")?.extract()?; - let grain = match config_str(&values, "cursor_grain")? { + let grain = match config_text_object(py, &values, "cursor_grain")? { Some(grain) if type_name == TIMESTAMP_CURSOR_TYPE => { let grain = enum_from_value(py, RecordId::CursorGrain, grain)?; let session = CursorSession::new(py)?; diff --git a/fensu.toml b/fensu.toml index 7031d9fdb7..893f0d0bf7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1941,7 +1941,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", @@ -1949,6 +1949,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", From dcaa537ad909095fd51646b7b87a60e142049797 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:28:40 +0100 Subject: [PATCH 6/9] fix(native): keep header-owned config values as the header's objects Since configs are built as typed nodes, every built container was projected as a fresh Python object. The model loop reuses natively parsed header columns only when `values["columns"]` is the header's own object, so any model with `columns` beside another config key fell back to parsing its columns again in Python. The build now reports which header keys still hold the header's own values (the provenance it already tracks, with set-aside hooks included), and the projection uses the header's objects for them, so identity holds exactly where it did before. Stage counters record whether a model's header columns were reused or parsed again, and an integration test pins the native path for models with `columns` beside `materialized`, `tags` or a description. --- .../src/model_config_build/_helpers/phases.rs | 39 +++++++++-- .../main/build_model_config.rs | 5 +- .../src/model_config_build/models.rs | 2 + .../_helpers/compile_driver/stage_work.rs | 4 ++ .../_helpers/input_loops/model_loop_header.rs | 10 +++ .../model_config/model_config_builder.rs | 66 ++++++++++++++----- .../compiler/model_config/_test_types.py | 10 +++ .../sqlbuild/compiler/model_config/helpers.py | 39 +++++++++++ .../model_config/test_header_column_paths.py | 38 +++++++++++ 9 files changed, 185 insertions(+), 28 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs index a81551add4..2089b78cb2 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs @@ -40,11 +40,17 @@ pub(crate) struct Phase { pub(crate) reads: Vec<(&'static str, String)>, } -/// The layered values with hooks set aside, and whether a layer configures the schema. -pub(crate) struct Layered { - pub(crate) phase: Phase, +/// What layering sets aside until the end: hooks, which of them are the header's own, and the schema layer. +pub(crate) struct SetAside { pub(crate) hooks: Entries, + pub(crate) own_hooks: Vec, pub(crate) layer_schema_configured: bool, +} + +/// The layered values, what is set aside, and whether any value holds a template. +pub(crate) struct Layered { + pub(crate) phase: Phase, + pub(crate) set_aside: SetAside, pub(crate) has_templates: bool, } @@ -79,6 +85,11 @@ pub(crate) fn layered( let facts = key_text(key).and_then(|name| own_facts(model.header.facts, &own, &name)); facts.map_or_else(|| contains_template(&value), |facts| facts.template) }); + let own_hooks: Vec = own + .iter() + .filter(|name| HOOK_KEYS.contains(&name.as_str())) + .cloned() + .collect(); let layer_schema_configured: bool = config_entry(model.header.entries, SCHEMA_KEY).is_some() || path_values.is_some_and(|values| config_entry(values, SCHEMA_KEY).is_some()); Ok(Layered { @@ -87,8 +98,11 @@ pub(crate) fn layered( own, reads: Vec::new(), }, - hooks, - layer_schema_configured, + set_aside: SetAside { + hooks, + own_hooks, + layer_schema_configured, + }, has_templates, }) } @@ -175,10 +189,14 @@ pub(crate) fn targeted( /// The built config: hooks restored, storage keys dropped, overrides read and macros rejected. pub(crate) fn finished( model: &BuildModel<'_, B>, - layered: (Entries, bool), + set_aside: SetAside, targeted: Targeted, ) -> Result, HostHalt> { - let (hooks, layer_schema_configured) = layered; + let SetAside { + hooks, + own_hooks, + layer_schema_configured, + } = set_aside; let (header, config) = (&model.header, ConfigBuildHalt::Config); let mut resolved: Entries = targeted.phase.values; for (key, value) in hooks { @@ -190,6 +208,12 @@ pub(crate) fn finished( let table_type_value = config_entry(header.entries, TABLE_TYPE_KEY); let table_type = table_type_override(table_type_value, model.name).map_err(config)?; check_no_config_macros(&values, header.facts, &targeted.phase.own).map_err(config)?; + let mut own_header_keys: Vec = targeted.phase.own; + let restored: Vec = own_hooks + .into_iter() + .filter(|name| !own_header_keys.contains(name)) + .collect(); + own_header_keys.extend(restored); Ok(BuiltModelConfig { values, header_keys: sorted_header_keys(header.entries)?, @@ -199,5 +223,6 @@ pub(crate) fn finished( retention, table_type, reads: targeted.phase.reads, + own_header_keys, }) } diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs index b2115d2048..06d561416a 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/main/build_model_config.rs @@ -25,13 +25,12 @@ pub fn build_model_config( }; let Layered { mut phase, - hooks, - layer_schema_configured, + set_aside, has_templates, } = layered(&model, matched_path_default)?; if has_templates { phase = model_templates(&model, phase)?; } let targeted = targeted(&model, phase, has_templates)?; - finished(&model, (hooks, layer_schema_configured), targeted) + finished(&model, set_aside, targeted) } diff --git a/crates/sqlbuild-model-config/src/model_config_build/models.rs b/crates/sqlbuild-model-config/src/model_config_build/models.rs index d8bb0bf2f8..a5860bfeba 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/models.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/models.rs @@ -49,6 +49,8 @@ pub struct BuiltModelConfig { pub table_type: TableTypeOverride, /// The `(kind, name)` environment and context reads, in order. pub reads: Vec<(&'static str, String)>, + /// The header keys whose built values are still the header's own values. + pub own_header_keys: Vec, } /// Why a build stops: the config error it raises, a caller error, or a header key that is not text. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 074a44b231..53fbaa31fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -26,6 +26,10 @@ pub(crate) const FORMAT_LINT_FILES: &str = "format_lint_files"; pub(crate) const SQL_TEST_ARTIFACT_PLANS: &str = "plan_compiled_sql_test_artifacts"; pub(crate) const SQL_TEST_MODEL_CHAINS: &str = "sql_test_model_chain_names_by_key"; pub(crate) const BACKGROUND_SQL_TEST_PLANS: &str = "background_sql_test_plans"; +/// Models whose header columns are reused from the native header parse. +pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; +/// Models whose header columns are parsed again from their config values. +pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; /// Compile stage runs since the last reset, by stage. #[pyclass] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index c7a8584120..cf3ba5e5b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -15,6 +15,9 @@ use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_h use sqlbuild_model_config::header_metadata::models::{DynamicFamily, ParsedColumn}; use sqlbuild_model_config::model_validation::models::ColumnMigration; +use crate::bindings::_helpers::compile_driver::stage_work::{ + NATIVE_HEADER_COLUMNS, PARSED_HEADER_COLUMNS, count_stage, +}; use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{ @@ -107,6 +110,13 @@ pub(crate) fn header_schema<'py>( let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); let model: (&str, &str) = (source.name, source.relative_path); let native: Option<&[NativeColumn]> = native_columns(context, source, columns.as_ref())?; + if columns.is_some() { + let stage = match native { + Some(_) => NATIVE_HEADER_COLUMNS, + None => PARSED_HEADER_COLUMNS, + }; + count_stage(py, stage, 1)?; + } let python_columns: Option> = if native.is_some() { None } else { columns }; let metadata = parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 88b983f712..66ae289768 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -205,6 +205,53 @@ fn halted<'py>( }) } +/// The built values as a new `dict`; entries still the header's own keep the header's objects. +fn own_values_dict<'py>( + header: &Bound<'py, PyDict>, + built: &BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in &built.values { + let own: bool = own_key(key, &built.own_header_keys); + let key: Bound<'py, PyAny> = config_object(py, key)?; + let value: Bound<'py, PyAny> = match own.then(|| header.get_item(&key)).transpose()? { + Some(Some(value)) => value, + _ => config_object(py, value)?, + }; + mapping.set_item(key, value)?; + } + Ok(mapping) +} + +/// Whether `key` is a string key named among the header's own keys. +fn own_key(key: &PyConfigNode, own: &[String]) -> bool { + let text: Option<&str> = match key { + ConfigNode::Str(text) => Some(text), + ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.as_deref(), + _ => None, + }; + text.is_some_and(|text| own.iter().any(|name| name == text)) +} + +/// One model's build projected for Python. +fn projected_build<'py>( + header: &Bound<'py, PyDict>, + built: BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + Ok(BuiltConfig { + values: own_values_dict(header, &built)?, + header_keys: built.header_keys, + logical_schema: projected(py, built.logical_schema)?, + layer_schema_configured: built.layer_schema_configured, + logical_database: projected(py, built.logical_database)?, + retention: built.retention, + table_type: built.table_type, + reads: built.reads, + }) +} + fn projected<'py>( py: Python<'py>, value: Option, @@ -253,7 +300,7 @@ impl NativeModelConfigBuilder { matched_path_default, model_name, ) { - Ok(built) => Ok(ConfigBuild::Built(self.projected_build(py, built)?)), + Ok(built) => Ok(ConfigBuild::Built(projected_build(header, built)?)), Err(halt) => match halted(py, halt)? { Ok(error) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), Err(error) => Err(error), @@ -261,23 +308,6 @@ impl NativeModelConfigBuilder { } } - fn projected_build<'py>( - &self, - py: Python<'py>, - built: BuiltModelConfig, - ) -> PyResult> { - Ok(BuiltConfig { - values: config_dict(py, &built.values)?, - header_keys: built.header_keys, - logical_schema: projected(py, built.logical_schema)?, - layer_schema_configured: built.layer_schema_configured, - logical_database: projected(py, built.logical_database)?, - retention: built.retention, - table_type: built.table_type, - reads: built.reads, - }) - } - /// `build_layered_model_values` with an empty header: what a path default's models inherit. pub(crate) fn inherited_values<'py>( &self, diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index fe784b2954..97e321ecad 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -62,3 +62,13 @@ class OpaqueConfigValueTestCase: header: dict[object, object] key: object expected_same_object: bool + + +@dataclass(frozen=True) +class HeaderColumnPathTestCase: + """MODEL headers with columns, and how many models reuse or reparse their native columns.""" + + description: str + headers: tuple[str, ...] + expected_native: int + expected_parsed: int diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 289a38dc0d..c627503fe0 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -635,3 +635,42 @@ def marker_kept(*, project_dir: Path, header: dict[object, object], key: object) config_value_inputs(project_dir=project_dir, header=header).model_inputs[0].config ) return config.values[cast("str", key)] is MARKER + + +def header_column_paths(*, project_dir: Path, headers: tuple[str, ...]) -> tuple[int, int]: + """How many models reuse their natively parsed header columns, and how many reparse them.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": CONFIG_VALUE_PROJECT["sqlbuild_project.toml"], + **{ + f"models/marts/orders_{index}.sql": f"MODEL ({header});\n\nSELECT 1 AS order_id\n" + for index, header in enumerate(headers) + }, + } + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + adapter: DuckDbAdapter = DuckDbAdapter() + _ = _native.compile_stage_work_info(reset=True) + _ = build_compile_inputs( + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="20261007T000000Z_orders", + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + work: dict[str, int] = _native.compile_stage_work_info(reset=True) + return (work.get("native_header_columns", 0), work.get("parsed_header_columns", 0)) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py new file mode 100644 index 0000000000..d52285d004 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py @@ -0,0 +1,38 @@ +"""Natively parsed MODEL header columns are reused whenever the built config keeps them.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + HeaderColumnPathTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import header_column_paths + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderColumnPathTestCase( + description="columns beside materialized, tags or a description", + headers=( + "materialized table, columns (order_id (type INTEGER))", + 'tags ["daily"], columns (order_id (type INTEGER))', + 'description "Orders.", columns (order_id (type INTEGER))', + ), + expected_native=3, + expected_parsed=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_header_columns_when_building_model_inputs_then_native_columns_are_reused( + test_case: HeaderColumnPathTestCase, tmp_path: Path +) -> None: + paths: tuple[int, int] = header_column_paths( + project_dir=tmp_path / "orders", headers=test_case.headers + ) + + assert paths == (test_case.expected_native, test_case.expected_parsed) From 64a0669c39b1d4bb90240d3beedc68dfac64e870 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:51:08 +0100 Subject: [PATCH 7/9] fix(native): read config value text through Python when only Python prints it exactly Opaque config values were read eagerly at conversion: `str()` twice, `repr()`, a date check and three hook-entry checks, with a raising `__repr__` read as empty. Only cheap facts (kind, text, numbers) are now read when a value is converted. Date checks and display text are asked of the value's handle when a scan needs them, and the build asks its host for `str()` and hook-entry checks, so their errors raise as before. Text is produced in Rust only where it is exact: `None`, booleans, integers, floats and a string's own text. A string's repr, containers and opaque values use Python's `str()`/`repr()` of the projected object for template operands and tag/row-diff unions. That replaces an approximate Rust repr that printed some format, private-use and unassigned characters unescaped. Hook-template context values read only `database`, `schema` and `alias` instead of converting the whole values dict. Integration tests pin container text, item errors in template operands and unions, and that the build never reads an opaque value's text. --- .../src/config_values/_helpers/authored.rs | 7 +- .../src/config_values/_helpers/entries.rs | 11 +-- .../src/config_values/_helpers/reprs.rs | 50 +++++------- .../src/config_values/_helpers/unions.rs | 30 ++++--- .../main/checked_cursor_bound.rs | 3 +- .../src/config_values/main/checked_flag.rs | 3 +- .../src/config_values/main/checked_integer.rs | 3 +- .../src/config_values/main/checked_text.rs | 3 +- .../src/config_values/main/checked_texts.rs | 3 +- .../src/config_values/main/config_entry.rs | 3 +- .../src/config_values/main/inserted_entry.rs | 3 +- .../src/config_values/main/layered_config.rs | 15 ++-- .../src/config_values/main/mod.rs | 1 + .../config_values/main/plain_scalar_text.rs | 10 +++ .../src/config_values/main/same_config_key.rs | 3 +- .../config_values/main/set_config_entry.rs | 3 +- .../src/config_values/models.rs | 9 +-- .../src/config_values/tests/helpers.rs | 10 --- .../tests/test_layered_config.rs | 6 +- .../config_values/tests/test_python_text.rs | 48 +++++------ .../src/config_values/tests/test_types.rs | 21 +++++ .../src/config_values/types.rs | 11 +++ .../src/model_config_build/_helpers/checks.rs | 47 +++++++---- .../model_config_build/_helpers/context.rs | 9 ++- .../model_config_build/_helpers/expansion.rs | 14 +++- .../src/model_config_build/_helpers/layers.rs | 26 +++--- .../src/model_config_build/_helpers/phases.rs | 6 +- .../model_config_build/_helpers/storage.rs | 7 +- .../main/inherited_config.rs | 12 +-- .../main/model_context_values.rs | 3 +- .../model_config_build/tests/test_types.rs | 8 ++ .../src/model_config_build/types.rs | 11 ++- .../_helpers/model_config/config_nodes.rs | 73 +++++++++-------- .../_helpers/model_config/described_models.rs | 12 ++- .../model_config/model_config_builder.rs | 33 ++++++-- .../compiler/model_config/_test_types.py | 18 +++++ .../sqlbuild/compiler/model_config/helpers.py | 62 ++++++++++++++- .../model_config/test_config_value_text.py | 79 +++++++++++++++++++ 38 files changed, 470 insertions(+), 206 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs create mode 100644 tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs index 7c558ad618..f1c51ec966 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/authored.rs @@ -2,9 +2,10 @@ use crate::config_values::_helpers::reprs::{python_repr, python_str}; use crate::config_values::models::{ConfigNode, HostChildren}; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; -impl<'a, H: Clone> AuthoredNode for &'a ConfigNode { +impl<'a, H: HostObject> AuthoredNode for &'a ConfigNode { fn kind(&self) -> NodeKind { match *self { ConfigNode::Null => NodeKind::Null, @@ -64,7 +65,7 @@ impl<'a, H: Clone> AuthoredNode for &'a ConfigNode { } fn is_date_like(&self) -> bool { - matches!(self, ConfigNode::Host(host) if host.facts.date_like) + matches!(self, ConfigNode::Host(host) if host.handle.is_date_like()) } fn items(&self) -> Vec { @@ -94,7 +95,7 @@ impl<'a, H: Clone> AuthoredNode for &'a ConfigNode { } } -impl AuthoredNode for ConfigNode { +impl AuthoredNode for ConfigNode { fn kind(&self) -> NodeKind { (&self).kind() } diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs index 7fb9e76757..3ece308db8 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/entries.rs @@ -1,6 +1,7 @@ //! A config mapping read and written as a Python `dict`. use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// The exact text of a string key, borrowed. @@ -15,7 +16,7 @@ fn key_text(node: &ConfigNode) -> Option<&str> { } /// Python's `==` between two dict keys: equal text, equal numbers, or equal tuples. -pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { +pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> bool { match (left, right) { (ConfigNode::Null, ConfigNode::Null) => true, (ConfigNode::Tuple(left), ConfigNode::Tuple(right)) => { @@ -33,7 +34,7 @@ pub(crate) fn same_key(left: &ConfigNode, right: &ConfigNode) -> } /// A boolean, integer or float key's numeric value, as Python compares keys. -fn numeric(node: &ConfigNode) -> Option { +fn numeric(node: &ConfigNode) -> Option { match node { ConfigNode::Bool(flag) => Some(f64::from(u8::from(*flag))), ConfigNode::Int(_) | ConfigNode::Float(_) => node.number(), @@ -42,7 +43,7 @@ fn numeric(node: &ConfigNode) -> Option { } /// `mapping.get(key)` for a string key. -pub(crate) fn entry<'a, H: Clone>( +pub(crate) fn entry<'a, H: HostObject>( entries: &'a [(ConfigNode, ConfigNode)], key: &str, ) -> Option<&'a ConfigNode> { @@ -53,7 +54,7 @@ pub(crate) fn entry<'a, H: Clone>( } /// `mapping.get(key)` unless absent or `None`. -pub(crate) fn set_entry<'a, H: Clone>( +pub(crate) fn set_entry<'a, H: HostObject>( entries: &'a [(ConfigNode, ConfigNode)], key: &str, ) -> Option<&'a ConfigNode> { @@ -61,7 +62,7 @@ pub(crate) fn set_entry<'a, H: Clone>( } /// `entries` after `mapping[key] = value`: an equal key keeps its place and its key, else the entry is appended. -pub(crate) fn inserted( +pub(crate) fn inserted( mut entries: Vec<(ConfigNode, ConfigNode)>, key: ConfigNode, value: ConfigNode, diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs index d28ece997b..edabcf2939 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/reprs.rs @@ -1,44 +1,30 @@ -//! Python's `str()` and `repr()` of config values. +//! Python's `str()` and `repr()` of config values: plain scalars in Rust, the rest by Python. use sqlbuild_core::json::main::python_float_repr::python_float_repr; -use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::config_values::constants::{FALSE_TEXT, NONE_TEXT, TRUE_TEXT}; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; -fn joined(items: &[ConfigNode]) -> String { - items.iter().map(python_repr).collect::>().join(", ") +/// Python's exact `str()` (or `repr()`) of a plain scalar, or `None` for any other value. +pub(crate) fn scalar_text(value: &ConfigNode, repr: bool) -> Option { + match value { + ConfigNode::Null => Some(NONE_TEXT.to_owned()), + ConfigNode::Bool(true) => Some(TRUE_TEXT.to_owned()), + ConfigNode::Bool(false) => Some(FALSE_TEXT.to_owned()), + ConfigNode::Int(number) => Some(number.to_string()), + ConfigNode::Float(number) => Some(python_float_repr(*number)), + ConfigNode::Str(text) if !repr => Some(text.clone()), + _ => None, + } } /// Python's `repr(value)`. -pub(crate) fn python_repr(value: &ConfigNode) -> String { - match value { - ConfigNode::Null => NONE_TEXT.to_owned(), - ConfigNode::Bool(true) => TRUE_TEXT.to_owned(), - ConfigNode::Bool(false) => FALSE_TEXT.to_owned(), - ConfigNode::Int(number) => number.to_string(), - ConfigNode::Float(number) => python_float_repr(*number), - ConfigNode::Str(text) => python_str_repr(text), - ConfigNode::List(items) => format!("[{}]", joined(items)), - ConfigNode::Tuple(items) if items.len() == 1 => format!("({},)", joined(items)), - ConfigNode::Tuple(items) => format!("({})", joined(items)), - ConfigNode::Map(entries) => format!( - "{{{}}}", - entries - .iter() - .map(|(key, item)| format!("{}: {}", python_repr(key), python_repr(item))) - .collect::>() - .join(", ") - ), - ConfigNode::Host(host) => host.facts.python_repr.clone(), - } +pub(crate) fn python_repr(value: &ConfigNode) -> String { + scalar_text(value, true).unwrap_or_else(|| H::display(value, true)) } -/// Python's `str(value)`: a string's own text, an opaque value's `str()`, else its `repr()`. -pub(crate) fn python_str(value: &ConfigNode) -> String { - match value { - ConfigNode::Str(text) => text.clone(), - ConfigNode::Host(host) => host.facts.python_str.clone(), - _ => python_repr(value), - } +/// Python's `str(value)`. +pub(crate) fn python_str(value: &ConfigNode) -> String { + scalar_text(value, false).unwrap_or_else(|| H::display(value, false)) } diff --git a/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs index 29fef8dbd7..d1b29d2747 100644 --- a/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs +++ b/crates/sqlbuild-model-config/src/config_values/_helpers/unions.rs @@ -1,36 +1,44 @@ //! The unions `_merged_with_tag_union` makes of string sequences and row-diff tolerances. use crate::config_values::_helpers::entries::{inserted, set_entry}; +use crate::config_values::_helpers::reprs::scalar_text; use crate::config_values::constants::AUDIT_OVERRIDE_SECTIONS; use crate::config_values::models::ConfigNode; +use crate::config_values::types::{HostObject, ItemText}; use crate::types::{AuthoredNode, NodeKind}; /// `_as_string_list(value)`: `str()` of each list or tuple item. -fn string_items(value: &ConfigNode) -> Vec { +fn string_items( + value: &ConfigNode, + text: ItemText<'_, H, E>, +) -> Result, E> { match value.kind() { - NodeKind::List | NodeKind::Tuple => { - value.items().iter().map(AuthoredNode::python_str).collect() - } - _ => Vec::new(), + NodeKind::List | NodeKind::Tuple => (&value) + .items() + .into_iter() + .map(|item| scalar_text(item, false).map_or_else(|| text(item), Ok)) + .collect(), + _ => Ok(Vec::new()), } } /// The strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. -pub(crate) fn merged_strings( +pub(crate) fn merged_strings( base: &ConfigNode, overlay: &ConfigNode, -) -> Vec> { - let mut merged: Vec = string_items(base); - for item in string_items(overlay) { + text: ItemText<'_, H, E>, +) -> Result>, E> { + let mut merged: Vec = string_items(base, text)?; + for item in string_items(overlay, text)? { if !merged.contains(&item) { merged.push(item); } } - merged.into_iter().map(ConfigNode::Str).collect() + Ok(merged.into_iter().map(ConfigNode::Str).collect()) } /// Row-diff tolerances deep-merged by section, as `_merge_row_diff_tolerances_mapping`. -pub(crate) fn merged_tolerances( +pub(crate) fn merged_tolerances( base: &ConfigNode, overlay: &ConfigNode, ) -> ConfigNode { diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs index d22d400e09..3527ee3580 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_cursor_bound.rs @@ -4,10 +4,11 @@ use crate::config_values::_helpers::checks::{authored, wrong_type}; use crate::config_values::constants::EXPECTED_CURSOR_BOUND; use crate::config_values::errors::ConfigTypeError; use crate::config_values::models::{ConfigNode, CursorBound}; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// How `key` reads as a cursor bound: a string, a non-boolean integer, a date or a datetime. -pub fn checked_cursor_bound<'a, H: Clone>( +pub fn checked_cursor_bound<'a, H: HostObject>( key: &str, value: Option<&'a ConfigNode>, ) -> Result>, ConfigTypeError<'a, H>> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs index f964a8cfde..af12ea2733 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_flag.rs @@ -4,10 +4,11 @@ use crate::config_values::_helpers::checks::{authored, wrong_type}; use crate::config_values::constants::EXPECTED_FLAG; use crate::config_values::errors::ConfigTypeError; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// The boolean value of `key`, or `None` when absent or `None`. -pub fn checked_flag<'a, H: Clone>( +pub fn checked_flag<'a, H: HostObject>( key: &str, value: Option<&'a ConfigNode>, ) -> Result, ConfigTypeError<'a, H>> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs index 15c107a95e..577a321f84 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_integer.rs @@ -4,10 +4,11 @@ use crate::config_values::_helpers::checks::{authored, wrong_type}; use crate::config_values::constants::EXPECTED_INTEGER; use crate::config_values::errors::ConfigTypeError; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// The value of `key` when it is an integer and not a boolean, or `None` when absent or `None`. -pub fn checked_integer<'a, H: Clone>( +pub fn checked_integer<'a, H: HostObject>( key: &str, value: Option<&'a ConfigNode>, ) -> Result>, ConfigTypeError<'a, H>> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs index eeb5c359cb..81b9a3e85f 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_text.rs @@ -4,10 +4,11 @@ use crate::config_values::_helpers::checks::{authored, wrong_type}; use crate::config_values::constants::EXPECTED_TEXT; use crate::config_values::errors::ConfigTypeError; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// The value of `key` when it is a string (a present empty string included), or `None` when absent or `None`. -pub fn checked_text<'a, H: Clone>( +pub fn checked_text<'a, H: HostObject>( key: &str, value: Option<&'a ConfigNode>, ) -> Result>, ConfigTypeError<'a, H>> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs index 5e2dd5a2de..bcbbe3368d 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/checked_texts.rs @@ -4,10 +4,11 @@ use crate::config_values::_helpers::checks::{authored, wrong_type}; use crate::config_values::constants::EXPECTED_TEXTS; use crate::config_values::errors::ConfigTypeError; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::types::{AuthoredNode, NodeKind}; /// The string items of `key` when it is a list or tuple of strings, or `None` when absent or `None`. -pub fn checked_texts<'a, H: Clone>( +pub fn checked_texts<'a, H: HostObject>( key: &str, value: Option<&'a ConfigNode>, ) -> Result>>, ConfigTypeError<'a, H>> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs index b971020c87..f22b150cc8 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/config_entry.rs @@ -2,10 +2,11 @@ use crate::config_values::_helpers::entries::entry; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; /// Python's `mapping.get(key)` for a string `key`. #[must_use] -pub fn config_entry<'a, H: Clone>( +pub fn config_entry<'a, H: HostObject>( entries: &'a [(ConfigNode, ConfigNode)], key: &str, ) -> Option<&'a ConfigNode> { diff --git a/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs index 5c58750d58..f1ede71f58 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/inserted_entry.rs @@ -3,10 +3,11 @@ use crate::config_values::_helpers::entries::inserted; use crate::config_values::models::ConfigNode; use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; /// `entries` after `mapping[key] = value`: an equal key keeps its place, else the entry is appended. #[must_use] -pub fn inserted_entry( +pub fn inserted_entry( entries: ConfigEntries, key: ConfigNode, value: ConfigNode, diff --git a/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs index bb83fd7ba0..d02967215f 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/layered_config.rs @@ -6,14 +6,15 @@ use crate::config_values::constants::{ ROW_DIFF_EXCLUDE_COLUMNS_KEY, ROW_DIFF_TOLERANCES_KEY, TAGS_KEY, }; use crate::config_values::models::{ConfigNode, LayeredConfig}; +use crate::config_values::types::{HostObject, ItemText}; use crate::types::{AuthoredNode, NodeKind}; -/// `overlay` merged over a copy of `base`, with tags, row-diff exclusions and tolerances unioned. -#[must_use] -pub fn layered_config( +/// `overlay` over a copy of `base`, unioning tags, row-diff exclusions and tolerances; `text` prints non-scalar items. +pub fn layered_config( base: &[(ConfigNode, ConfigNode)], overlay: &[(ConfigNode, ConfigNode)], -) -> LayeredConfig { + text: ItemText<'_, H, E>, +) -> Result, E> { let mut entries = base.to_vec(); for (key, value) in overlay { entries = inserted(entries, key.clone(), value.clone()); @@ -23,7 +24,7 @@ pub fn layered_config( if let (Some(base_items), Some(overlay_items)) = (set_entry(base, key), set_entry(overlay, key)) { - let items: Vec> = merged_strings(base_items, overlay_items); + let items: Vec> = merged_strings(base_items, overlay_items, text)?; let merged: ConfigNode = if tuple { ConfigNode::Tuple(items) } else { @@ -46,8 +47,8 @@ pub fn layered_config( unioned_keys.push(ROW_DIFF_TOLERANCES_KEY); } } - LayeredConfig { + Ok(LayeredConfig { entries, unioned_keys, - } + }) } diff --git a/crates/sqlbuild-model-config/src/config_values/main/mod.rs b/crates/sqlbuild-model-config/src/config_values/main/mod.rs index b424fdd584..a16884acfe 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/mod.rs @@ -6,5 +6,6 @@ pub mod checked_texts; pub mod config_entry; pub mod inserted_entry; pub mod layered_config; +pub mod plain_scalar_text; pub mod same_config_key; pub mod set_config_entry; diff --git a/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs b/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs new file mode 100644 index 0000000000..4c78b6425d --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_values/main/plain_scalar_text.rs @@ -0,0 +1,10 @@ +//! Python's text of a plain scalar config value. + +use crate::config_values::_helpers::reprs::scalar_text; +use crate::config_values::models::ConfigNode; + +/// Python's exact `str()` (or `repr()`) of `None`, a boolean, an integer, a float or a string's `str()`; else `None`. +#[must_use] +pub fn plain_scalar_text(value: &ConfigNode, repr: bool) -> Option { + scalar_text(value, repr) +} diff --git a/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs index 38cab0227a..012d815e0f 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/same_config_key.rs @@ -2,9 +2,10 @@ use crate::config_values::_helpers::entries::same_key; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; /// Whether two dict keys are equal: equal text, equal numbers, or equal tuples. #[must_use] -pub fn same_config_key(left: &ConfigNode, right: &ConfigNode) -> bool { +pub fn same_config_key(left: &ConfigNode, right: &ConfigNode) -> bool { same_key(left, right) } diff --git a/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs index 89b0e75ec0..3d0d68c948 100644 --- a/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs +++ b/crates/sqlbuild-model-config/src/config_values/main/set_config_entry.rs @@ -2,10 +2,11 @@ use crate::config_values::_helpers::entries::set_entry; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; /// Python's `mapping.get(key)` for a string `key`, unless it is absent or `None`. #[must_use] -pub fn set_config_entry<'a, H: Clone>( +pub fn set_config_entry<'a, H: HostObject>( entries: &'a [(ConfigNode, ConfigNode)], key: &str, ) -> Option<&'a ConfigNode> { diff --git a/crates/sqlbuild-model-config/src/config_values/models.rs b/crates/sqlbuild-model-config/src/config_values/models.rs index db0a1ed00c..1d1f6df825 100644 --- a/crates/sqlbuild-model-config/src/config_values/models.rs +++ b/crates/sqlbuild-model-config/src/config_values/models.rs @@ -28,7 +28,7 @@ pub struct HostValue { pub children: HostChildren, } -/// What config logic may read of an opaque value, read once when the value was converted. +/// The cheap facts read of an opaque value when it is converted; anything else is asked of its handle. #[derive(Clone, Debug, PartialEq)] pub struct HostFacts { pub kind: NodeKind, @@ -38,13 +38,6 @@ pub struct HostFacts { pub strict_text: bool, pub integer: Option, pub number: Option, - pub python_str: String, - /// Whether `python_str` is the value's exact `str()`. - pub strict_str: bool, - pub python_repr: String, - pub date_like: bool, - /// Whether the value is a typed hook entry. - pub hook_entry: bool, } /// The converted items of an opaque list, tuple or mapping, which config logic walks. diff --git a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs index 120e2a05cf..8d2b3921ac 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/helpers.rs @@ -18,11 +18,6 @@ pub(super) fn host_text(value: &str) -> Node { strict_text: true, integer: None, number: None, - python_str: value.to_owned(), - strict_str: true, - python_repr: format!(""), - date_like: false, - hook_entry: false, }, children: HostChildren::None, })) @@ -43,11 +38,6 @@ pub(super) fn host_date() -> Node { strict_text: false, integer: None, number: None, - python_str: "2024-01-02".to_owned(), - strict_str: true, - python_repr: "datetime.date(2024, 1, 2)".to_owned(), - date_like: true, - hook_entry: false, }, children: HostChildren::None, })) diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs index 640030f654..feba8f1201 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_layered_config.rs @@ -1,7 +1,7 @@ use crate::config_values::main::layered_config::layered_config; use crate::config_values::models::ConfigNode; use crate::config_values::tests::helpers::{text, texts}; -use crate::config_values::tests::test_types::LayeredConfigTestCase; +use crate::config_values::tests::test_types::{LayeredConfigTestCase, Node}; #[test] fn given_two_layers_when_merged_then_overrides_keep_order_and_unions_merge() { @@ -91,7 +91,9 @@ fn given_two_layers_when_merged_then_overrides_keep_order_and_unions_merge() { }, ]; for test_case in test_cases { - let layered = layered_config(&test_case.base, &test_case.overlay); + let text = |_: &Node| -> Result { Err(()) }; + let layered = layered_config(&test_case.base, &test_case.overlay, &text) + .expect("plain scalar unions need no host text"); assert_eq!( (layered.entries, layered.unioned_keys), (test_case.expected_entries, test_case.expected_unioned_keys), diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs index f2a1bad7bb..74ee21a803 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_python_text.rs @@ -4,45 +4,37 @@ use crate::config_values::tests::test_types::PythonTextTestCase; use crate::types::AuthoredNode; #[test] -fn given_config_values_when_printed_then_str_and_repr_match_python() { +fn given_config_values_when_printed_then_scalars_print_natively_and_the_rest_by_python() { let test_cases = [ PythonTextTestCase { - description: "scalars print as Python literals", - value: ConfigNode::Tuple(vec![ - ConfigNode::Null, - ConfigNode::Bool(true), - ConfigNode::Int(-3), - ConfigNode::Float(1e16), - ]), - expected_str: "(None, True, -3, 1e+16)", - expected_repr: "(None, True, -3, 1e+16)", + description: "plain scalars print as Python literals", + value: ConfigNode::Float(1e16), + expected_str: "1e+16", + expected_repr: "1e+16", }, PythonTextTestCase { - description: "a string prints itself and reprs quoted", - value: text("it's"), - expected_str: "it's", - expected_repr: "\"it's\"", + description: "a boolean and None print as Python spells them", + value: ConfigNode::Bool(false), + expected_str: "False", + expected_repr: "False", }, PythonTextTestCase { - description: "a one-item tuple keeps its comma", - value: ConfigNode::Tuple(vec![ConfigNode::Float(1.0)]), - expected_str: "(1.0,)", - expected_repr: "(1.0,)", + description: "a string prints itself; its repr is Python's", + value: text("it's\u{200b}"), + expected_str: "it's\u{200b}", + expected_repr: "0", }, PythonTextTestCase { - description: "mappings and lists nest item reprs, host values their own", - value: ConfigNode::Map(vec![( - text("severity"), - ConfigNode::List(vec![host_text("warn"), text("x")]), - )]), - expected_str: "{'severity': [, 'x']}", - expected_repr: "{'severity': [, 'x']}", + description: "a container prints by Python, never approximated in Rust", + value: ConfigNode::List(vec![host_text("warn"), text("x")]), + expected_str: "0", + expected_repr: "0", }, PythonTextTestCase { - description: "a host value prints its own str", + description: "a host value prints by Python", value: host_text("warn"), - expected_str: "warn", - expected_repr: "", + expected_str: "1", + expected_repr: "1", }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs index 8e08ddb0e6..112ea25acc 100644 --- a/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_values/tests/test_types.rs @@ -1,4 +1,5 @@ use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; /// A config node in tests; host handles are plain numbers. pub(super) type Node = ConfigNode; @@ -35,3 +36,23 @@ pub(super) struct CheckedReadTestCase { pub(super) expected_texts: &'static str, pub(super) expected_cursor_bound: &'static str, } + +/// Test handles: `2` is a date; other values print as `` or `` with their handle. +impl HostObject for u8 { + fn is_date_like(&self) -> bool { + *self == 2 + } + + fn display(node: &ConfigNode, repr: bool) -> String { + let kind: &str = ["", ""][usize::from(repr)]; + format!("{kind}{}", node_handle(node)) + } +} + +/// The handle of a host node, or `0` for a value the crate printed nothing of. +fn node_handle(node: &Node) -> u8 { + let ConfigNode::Host(host) = node else { + return 0; + }; + host.handle +} diff --git a/crates/sqlbuild-model-config/src/config_values/types.rs b/crates/sqlbuild-model-config/src/config_values/types.rs index cb1f94fbfb..1da941f764 100644 --- a/crates/sqlbuild-model-config/src/config_values/types.rs +++ b/crates/sqlbuild-model-config/src/config_values/types.rs @@ -4,3 +4,14 @@ use crate::config_values::models::ConfigNode; /// A mapping's `(key, value)` entries in insertion order, as a Python `dict` keeps them. pub type ConfigEntries = Vec<(ConfigNode, ConfigNode)>; + +/// What config logic asks of an opaque value's handle, read only when it is needed. +pub trait HostObject: Clone { + /// Whether the value is a `datetime.date` or `datetime.datetime`. + fn is_date_like(&self) -> bool; + /// Python's `str(node)`, or `repr(node)`, of a value the crate does not print exactly itself. + fn display(node: &ConfigNode, repr: bool) -> String; +} + +/// Python's `str()` of a value that is not a plain scalar, or the error it raises. +pub type ItemText<'t, H, E> = &'t dyn Fn(&ConfigNode) -> Result; diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs index 788508070e..821b0fad03 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/checks.rs @@ -5,9 +5,11 @@ use crate::config_presence::models::MacroPath; use crate::config_values::main::config_entry::config_entry; use crate::config_values::main::set_config_entry::set_config_entry; use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::errors::ConfigError; use crate::model_config_build::constants::{HOOK_KEYS, SORTED_HOOK_KEYS, TAGS_KEY}; -use crate::model_config_build::models::HeaderEntryFacts; +use crate::model_config_build::models::{ConfigBuildHalt, HeaderEntryFacts}; +use crate::model_config_build::types::{ConfigBuildHost, HostHalt, HostNode}; use crate::types::{AuthoredNode, NodeKind}; /// The `CompileInputError` `model '' `. @@ -16,7 +18,7 @@ pub(crate) fn model_error(model_name: &str, text: &str) -> ConfigError { } /// `_validate_model_header_tags`. -pub(crate) fn check_header_tags( +pub(crate) fn check_header_tags( header: &[(ConfigNode, ConfigNode)], model_name: &str, ) -> Result<(), ConfigError> { @@ -33,30 +35,47 @@ pub(crate) fn check_header_tags( } } +/// Whether every item of a hook list is a typed hook entry, checked in order. +fn all_hook_entries( + host: &B, + value: &HostNode, +) -> Result> { + for item in (&value).items() { + let typed: bool = match item { + ConfigNode::Host(_) => host.typed_hook_entry(item).map_err(ConfigBuildHalt::Host)?, + _ => false, + }; + if !typed { + return Ok(false); + } + } + Ok(true) +} + /// `validate_model_hook_config`: every hook list holds typed hook entries only. -pub(crate) fn check_hooks( - values: &[(ConfigNode, ConfigNode)], +pub(crate) fn check_hooks( + host: &B, + values: &[(HostNode, HostNode)], model_name: &str, -) -> Result<(), ConfigError> { +) -> Result<(), HostHalt> { + let config = ConfigBuildHalt::Config; for key in SORTED_HOOK_KEYS { let Some(value) = config_entry(values, key) else { continue; }; if !matches!(value.kind(), NodeKind::List | NodeKind::Tuple) { - return Err(model_error( + return Err(config(model_error( model_name, &format!("{key} must be a list of typed hook entries"), - )); + ))); } - let typed = - |item: &ConfigNode| matches!(item, ConfigNode::Host(host) if host.facts.hook_entry); - if !value.items().iter().all(typed) { - return Err(model_error( + if !all_hook_entries(host, value)? { + return Err(config(model_error( model_name, &format!( "{key} entries must use typed inline_sql(...), sql(...), or python(...) hook syntax" ), - )); + ))); } } Ok(()) @@ -78,7 +97,7 @@ pub(crate) fn own_facts<'a>( } /// The exact text of a string key the header facts are found by. -pub(crate) fn key_text(key: &ConfigNode) -> Option { +pub(crate) fn key_text(key: &ConfigNode) -> Option { match key { ConfigNode::Str(text) => Some(text.clone()), ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.clone(), @@ -87,7 +106,7 @@ pub(crate) fn key_text(key: &ConfigNode) -> Option { } /// The first config field outside hooks holding a macro call, as Python's walk reports it. -pub(crate) fn check_no_config_macros( +pub(crate) fn check_no_config_macros( values: &[(ConfigNode, ConfigNode)], facts: Option<&[(String, HeaderEntryFacts)]>, own: &[String], diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs index 7bdb39265e..fac426ef08 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/context.rs @@ -3,6 +3,7 @@ use crate::config_values::main::config_entry::config_entry; use crate::config_values::models::ConfigNode; use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; use crate::model_config_build::constants::{ ALIAS_KEY, DATABASE_KEY, DESTINATION_DATABASE_CONTEXT, DESTINATION_QUALIFIED_CONTEXT, DESTINATION_SCHEMA_CONTEXT, DESTINATION_TABLE_CONTEXT, MODEL_ALIAS_CONTEXT, @@ -20,7 +21,7 @@ fn named(name: &str, value: ConfigNode) -> (ConfigNode, ConfigNode) } /// The value of `key` when it is a string, as `isinstance(raw, str)` keeps it. -pub(crate) fn string_entry( +pub(crate) fn string_entry( values: &[(ConfigNode, ConfigNode)], key: &str, ) -> Option> { @@ -30,19 +31,19 @@ pub(crate) fn string_entry( } /// `build_run_context_values`: the run's id and target. -pub(crate) fn run_context(inputs: &ConfigBuildInputs) -> Entries { +pub(crate) fn run_context(inputs: &ConfigBuildInputs) -> Entries { vec![ named(RUN_ID_CONTEXT, inputs.run_id.clone()), named(RUN_TARGET_CONTEXT, inputs.target_name.clone()), ] } -fn text_of(value: &ConfigNode) -> String { +fn text_of(value: &ConfigNode) -> String { value.text().unwrap_or_default() } /// `build_model_context_values` for a model's config values. -pub(crate) fn model_context( +pub(crate) fn model_context( inputs: &ConfigBuildInputs, values: &[(ConfigNode, ConfigNode)], model_name: &str, diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs index 32b22852c8..485beb7354 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/expansion.rs @@ -3,6 +3,7 @@ use std::cell::RefCell; use crate::config_values::main::config_entry::config_entry; +use crate::config_values::main::plain_scalar_text::plain_scalar_text; use crate::config_values::models::{ConfigNode, HostChildren}; use crate::config_values::types::ConfigEntries; use crate::errors::ConfigError; @@ -110,10 +111,15 @@ impl TemplateHost for BuildTemplateHost<'_, B> { match value { ConfigNode::Null => Ok(Scalar::Null), ConfigNode::Bool(flag) => Ok(Scalar::Bool(*flag)), - ConfigNode::Host(host) if !host.facts.strict_str => { - Err(self.hold(self.sources.host.text_error(value))) - } - _ => Ok(Scalar::Text(value.python_str())), + _ => match plain_scalar_text(value, false) { + Some(text) => Ok(Scalar::Text(text)), + None => self + .sources + .host + .python_str(value, true) + .map(Scalar::Text) + .map_err(|error| self.hold(error)), + }, } } diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs index 7b223de378..5560f3db39 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/layers.rs @@ -3,16 +3,18 @@ use crate::config_values::main::layered_config::layered_config; use crate::config_values::models::ConfigNode; use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; use crate::model_config_build::_helpers::checks::key_text; use crate::model_config_build::constants::{TAGS_KEY, TEMPLATE_OPEN_TOKEN}; use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt, HostNode}; use crate::types::{AuthoredNode, NodeKind}; /// Config entries. pub(crate) type Entries = ConfigEntries; /// A fresh copy of the project defaults, with its own `tags` list. -pub(crate) fn project_defaults(inputs: &ConfigBuildInputs) -> Entries { +pub(crate) fn project_defaults(inputs: &ConfigBuildInputs) -> Entries { inputs .defaults .iter() @@ -26,7 +28,7 @@ pub(crate) fn project_defaults(inputs: &ConfigBuildInputs) -> Entri } /// The path default `key`'s entries. -pub(crate) fn path_default_values<'a, H: Clone, E>( +pub(crate) fn path_default_values<'a, H: HostObject, E>( inputs: &'a ConfigBuildInputs, key: &str, ) -> Result<&'a Entries, ConfigBuildHalt> { @@ -39,21 +41,23 @@ pub(crate) fn path_default_values<'a, H: Clone, E>( } /// `overlay` over `base`, and the overlay's keys whose merged value is still the overlay's own. -pub(crate) fn layered_own( - base: &Entries, - overlay: &[(ConfigNode, ConfigNode)], -) -> (Entries, Vec) { - let layered = layered_config(base, overlay); +pub(crate) fn layered_own( + host: &B, + base: &HostEntries, + overlay: &[(HostNode, HostNode)], +) -> Result<(HostEntries, Vec), HostHalt> { + let text = |item: &HostNode| host.python_str(item, false); + let layered = layered_config(base, overlay, &text).map_err(ConfigBuildHalt::Host)?; let own: Vec = overlay .iter() .filter_map(|(key, _)| key_text(key)) .filter(|name| !layered.unioned_keys.contains(&name.as_str())) .collect(); - (layered.entries, own) + Ok((layered.entries, own)) } /// Whether expansion keeps `value` as the same object: a string without a template, or a non-container. -pub(crate) fn kept_by_expansion(value: &ConfigNode) -> bool { +pub(crate) fn kept_by_expansion(value: &ConfigNode) -> bool { match value.kind() { NodeKind::Str => !value .with_text(|text| text.contains(TEMPLATE_OPEN_TOKEN)) @@ -64,7 +68,7 @@ pub(crate) fn kept_by_expansion(value: &ConfigNode) -> bool { } /// Whether the entry named `name` in `values` keeps its value through an expansion. -fn kept_entry(values: &Entries, name: &str) -> bool { +fn kept_entry(values: &Entries, name: &str) -> bool { values .iter() .find(|(key, _)| key_text(key).as_deref() == Some(name)) @@ -72,7 +76,7 @@ fn kept_entry(values: &Entries, name: &str) -> bool { } /// The own keys whose values an expansion of `values` keeps. -pub(crate) fn kept_own(values: &Entries, own: Vec) -> Vec { +pub(crate) fn kept_own(values: &Entries, own: Vec) -> Vec { own.into_iter() .filter(|name| kept_entry(values, name)) .collect() diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs index 2089b78cb2..87ea793f80 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/phases.rs @@ -71,10 +71,10 @@ pub(crate) fn layered( .transpose()?; let mut defaults: Entries = project_defaults(model.inputs); if let Some(path_values) = path_values { - defaults = layered_own(&defaults, path_values).0; + defaults = layered_own(model.host, &defaults, path_values)?.0; } - let (mut values, own) = layered_own(&defaults, model.header.entries); - check_hooks(&values, model.name).map_err(ConfigBuildHalt::Config)?; + let (mut values, own) = layered_own(model.host, &defaults, model.header.entries)?; + check_hooks(model.host, &values, model.name)?; let mut hooks: Entries = Vec::new(); for key in HOOK_KEYS { if let Some(position) = values.iter().position(|(name, _)| name.is_text(key)) { diff --git a/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs index d1db8e8219..d2cb4fec06 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/_helpers/storage.rs @@ -4,6 +4,7 @@ use crate::config_values::main::config_entry::config_entry; use crate::config_values::main::same_config_key::same_config_key; use crate::config_values::models::ConfigNode; use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; use crate::model_config_build::_helpers::checks::key_text; use crate::model_config_build::constants::{ FULL_REFRESH_KEY, INCREMENTAL_MATERIALIZATION, MATERIALIZED_KEY, RETENTION_KEY, TABLE_TYPE_KEY, @@ -15,12 +16,12 @@ use crate::types::AuthoredNode; type Entries = ConfigEntries; /// Whether `key` names a storage setting, which the config values leave out. -fn storage_key(key: &ConfigNode) -> bool { +fn storage_key(key: &ConfigNode) -> bool { key.is_text(RETENTION_KEY) || key.is_text(TABLE_TYPE_KEY) } /// Drop storage keys, then `full_refresh` where `build_model_config` drops it. -pub(crate) fn storage_free_values( +pub(crate) fn storage_free_values( resolved: Entries, header: &[(ConfigNode, ConfigNode)], ) -> Entries { @@ -38,7 +39,7 @@ pub(crate) fn storage_free_values( } /// The header's keys sorted by code point; a key that is not a string halts. -pub(crate) fn sorted_header_keys( +pub(crate) fn sorted_header_keys( header: &[(ConfigNode, ConfigNode)], ) -> Result, ConfigBuildHalt> { let mut keys: Vec = Vec::with_capacity(header.len()); diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs index 8a29ef96a7..050a00fbe9 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/main/inherited_config.rs @@ -1,19 +1,19 @@ //! What a path default's models inherit. -use crate::config_values::types::ConfigEntries; use crate::model_config_build::_helpers::layers::{ layered_own, path_default_values, project_defaults, }; -use crate::model_config_build::models::{ConfigBuildHalt, ConfigBuildInputs}; +use crate::model_config_build::models::ConfigBuildInputs; +use crate::model_config_build::types::{ConfigBuildHost, HostEntries, HostHalt}; /// `build_layered_model_values` with an empty header: the defaults under the matched path default. -pub fn inherited_config( - inputs: &ConfigBuildInputs, +pub fn inherited_config( + (host, inputs): (&B, &ConfigBuildInputs), matched_path_default: Option<&str>, -) -> Result, ConfigBuildHalt> { +) -> Result, HostHalt> { let mut layered = project_defaults(inputs); if let Some(key) = matched_path_default { - layered = layered_own(&layered, path_default_values(inputs, key)?).0; + layered = layered_own(host, &layered, path_default_values(inputs, key)?)?.0; } Ok(layered) } diff --git a/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs index ee7d66dfc5..201b9afd8f 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/main/model_context_values.rs @@ -2,12 +2,13 @@ use crate::config_values::models::ConfigNode; use crate::config_values::types::ConfigEntries; +use crate::config_values::types::HostObject; use crate::model_config_build::_helpers::context::model_context; use crate::model_config_build::models::ConfigBuildInputs; /// `build_model_context_values` for a model's config values. #[must_use] -pub fn model_context_values( +pub fn model_context_values( inputs: &ConfigBuildInputs, values: &[(ConfigNode, ConfigNode)], model_name: &str, diff --git a/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs index 06dcd266ac..9cdb666bb2 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/tests/test_types.rs @@ -32,6 +32,14 @@ impl ConfigBuildHost for FixedHost { fn text_error(&self, _value: &Node) -> String { "not text".to_owned() } + + fn python_str(&self, _value: &Node, _exact: bool) -> Result { + Err("no host text".to_owned()) + } + + fn typed_hook_entry(&self, _value: &Node) -> Result { + Ok(false) + } } /// The built values and reads, or why the build stops. diff --git a/crates/sqlbuild-model-config/src/model_config_build/types.rs b/crates/sqlbuild-model-config/src/model_config_build/types.rs index 5444ec214a..2b1e7bf966 100644 --- a/crates/sqlbuild-model-config/src/model_config_build/types.rs +++ b/crates/sqlbuild-model-config/src/model_config_build/types.rs @@ -1,12 +1,13 @@ //! What the model config build asks of its caller. use crate::config_values::models::ConfigNode; +use crate::config_values::types::HostObject; use crate::model_config_build::models::ConfigBuildHalt; /// The process environment, project-var rendering and exact text errors, owned by the caller. pub trait ConfigBuildHost { /// The caller's handle of an opaque value. - type Handle: Clone; + type Handle: HostObject; /// An error the caller raises, held until the build stops. type Error; @@ -20,6 +21,14 @@ pub trait ConfigBuildHost { ) -> Result, Self::Error>; /// The error reading `value` as exact text raises. fn text_error(&self, value: &ConfigNode) -> Self::Error; + /// Python's `str(value)` of a value that is not a plain scalar; `exact` rejects lone surrogates. + fn python_str( + &self, + value: &ConfigNode, + exact: bool, + ) -> Result; + /// Whether `value` is a typed hook entry. + fn typed_hook_entry(&self, value: &ConfigNode) -> Result; } /// A config value of the build host `B`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs index a59dc54979..42e77b7927 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use pyo3::IntoPyObject; -use pyo3::exceptions::{PyException, PyOverflowError, PyUnicodeEncodeError}; +use pyo3::exceptions::{PyOverflowError, PyUnicodeEncodeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, @@ -12,19 +12,13 @@ use pyo3::types::{ use sqlbuild_model_config::config_values::models::{ ConfigNode, HostChildren, HostFacts, HostValue, }; +use sqlbuild_model_config::config_values::types::HostObject; use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::ids::RecordId; -/// The Python object an opaque config value stands for. -pub(crate) type HostHandle = Arc>; -/// A config value converted from Python. -pub(crate) type PyConfigNode = ConfigNode; -/// A converted config mapping's entries. -pub(crate) type PyConfigEntries = Vec<(PyConfigNode, PyConfigNode)>; - /// The typed hook entry classes, which the config build checks hook lists against. const HOOK_ENTRY_CLASSES: [RecordId; 3] = [ RecordId::SqlHookEntry, @@ -32,14 +26,36 @@ const HOOK_ENTRY_CLASSES: [RecordId; 3] = [ RecordId::PythonHookEntry, ]; -fn hook_entry(value: &Bound<'_, PyAny>) -> PyResult { - for class in HOOK_ENTRY_CLASSES { - if value.is_instance(&record_class(value.py(), class)?)? { - return Ok(true); - } +/// The Python object an opaque config value stands for. +#[derive(Clone, Debug)] +pub(crate) struct HostHandle(pub(crate) Arc>); + +impl HostObject for HostHandle { + fn is_date_like(&self) -> bool { + Python::attach(|py| PyNode(self.0.bind(py).clone()).is_date_like()) + } + + /// Python's text of the projected value; a `str()` or `repr()` that raises reads as empty, as `PyNode`. + fn display(node: &ConfigNode, repr: bool) -> String { + Python::attach(|py| { + config_object(py, node).map_or_else( + |_| String::new(), + |value| { + let value: PyNode<'_> = PyNode(value); + if repr { + value.python_repr() + } else { + value.python_str() + } + }, + ) + }) } - Ok(false) } +/// A config value converted from Python. +pub(crate) type PyConfigNode = ConfigNode; +/// A converted config mapping's entries. +pub(crate) type PyConfigEntries = Vec<(PyConfigNode, PyConfigNode)>; /// Whether a Python `str` encodes to UTF-8, which it does unless it holds a lone surrogate. fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { @@ -50,16 +66,7 @@ fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { } } -/// Whether `str(value)` succeeds with UTF-8 text; any error `str()` raises makes it inexact. -fn exact_str(value: &Bound<'_, PyAny>) -> PyResult { - match value.str() { - Ok(text) => utf8_text(&text), - Err(error) if error.is_instance_of::(value.py()) => Ok(false), - Err(error) => Err(error), - } -} - -/// An opaque value with the facts `PyNode` reads of it, and its converted items. +/// An opaque value with the cheap facts `PyNode` reads of it, and its converted items. fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResult { let node: PyNode<'_> = PyNode(value.clone()); let strict_text: bool = match value.downcast::() { @@ -67,18 +74,13 @@ fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResul Err(_) => false, }; Ok(ConfigNode::Host(Box::new(HostValue { - handle: Arc::new(value.clone().unbind()), + handle: HostHandle(Arc::new(value.clone().unbind())), facts: HostFacts { kind: node.kind(), text: node.text(), strict_text, integer: node.integer(), number: node.number(), - python_str: node.python_str(), - strict_str: exact_str(value)?, - python_repr: node.python_repr(), - date_like: node.is_date_like(), - hook_entry: hook_entry(value)?, }, children, }))) @@ -216,7 +218,7 @@ pub(crate) fn config_object<'py>( )? .into_any(), ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), - ConfigNode::Host(host) => host.handle.bind(py).clone(), + ConfigNode::Host(host) => host.handle.0.bind(py).clone(), }) } @@ -236,3 +238,12 @@ pub(crate) fn config_dict<'py>( pub(crate) fn is_text_node(node: &PyConfigNode) -> bool { node.kind() == NodeKind::Str } + +pub(crate) fn hook_entry(value: &Bound<'_, PyAny>) -> PyResult { + for class in HOOK_ENTRY_CLASSES { + if value.is_instance(&record_class(value.py(), class)?)? { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs index 9cd4868913..a81aff8a36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyListMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyStringMethods}; use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_model_config::config_values::main::config_entry::config_entry; use sqlbuild_model_config::config_values::main::layered_config::layered_config; @@ -47,6 +47,12 @@ pub(crate) fn externally_described_models<'py>( .iter() .map(|key| key.extract::()) .collect::>()?; + let text = |item: &PyConfigNode| -> PyResult { + Ok(config_object(py, item)? + .str()? + .to_string_lossy() + .into_owned()) + }; let mut layers: HashMap = HashMap::new(); let mut described: Vec> = Vec::new(); for model_file in native_field(&inputs, "model_files")?.try_iter()? { @@ -62,14 +68,14 @@ pub(crate) fn externally_described_models<'py>( path_defaults.as_any().get_item(&key)?.downcast_into()?; layers.insert(key.clone(), config_entries(&layer)?); } - values = layered_config(&values, &layers[&key]).entries; + values = layered_config(&values, &layers[&key], &text)?.entries; } PathDefaultChoice::Selected(None) => {} PathDefaultChoice::Conflict(error) => return Err(raised_config_error(py, error)), } let header: Bound<'py, PyDict> = native_field(&model_file, "header_values")?.downcast_into()?; - values = layered_config(&values, &config_entries(&header)?).entries; + values = layered_config(&values, &config_entries(&header)?, &text)?.entries; let schema_description: Option = match config_entry(&values, MODEL_SCHEMA_KEY) { Some(name) if is_text_node(name) => model_schemas diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 66ae289768..d4c1015570 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -25,11 +25,13 @@ use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::config_nodes::{ HostHandle, PyConfigEntries, PyConfigNode, config_dict, config_entries, config_node, - config_object, + config_object, hook_entry, }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; +/// The config keys a model's context values read. +const CONTEXT_KEYS: [&str; 3] = ["database", "schema", "alias"]; const BUILD_STOPPED: &str = "a config value read as text without an error"; /// The project-wide inputs every model's config build reads, converted once per compile. @@ -111,6 +113,18 @@ impl ConfigBuildHost for PythonConfigHost<'_> { Err(error) => error, } } + + fn python_str(&self, value: &PyConfigNode, exact: bool) -> PyResult { + let text: Bound<'_, PyString> = config_object(self.environment.py(), value)?.str()?; + if exact { + return Ok(text.to_str()?.to_owned()); + } + Ok(text.to_string_lossy().into_owned()) + } + + fn typed_hook_entry(&self, value: &PyConfigNode) -> PyResult { + hook_entry(&config_object(self.environment.py(), value)?) + } } /// One header entry's parsed facts as the build reads them. @@ -314,7 +328,8 @@ impl NativeModelConfigBuilder { py: Python<'py>, matched_path_default: Option<&str>, ) -> PyResult, Py>> { - match inherited_config::(&self.inputs, matched_path_default) { + let host: PythonConfigHost<'py> = self.host(py); + match inherited_config((&host, &self.inputs), matched_path_default) { Ok(values) => Ok(Ok(config_dict(py, &values)?)), Err(halt) => match halted(py, halt)? { Ok(error) => Ok(Err(native_config_error(py, error)?)), @@ -330,12 +345,14 @@ impl NativeModelConfigBuilder { model_name: &str, include_target_values: bool, ) -> PyResult> { - let context: PyConfigEntries = model_context_values( - &self.inputs, - &config_entries(values)?, - model_name, - include_target_values, - ); + let mut read: PyConfigEntries = Vec::with_capacity(CONTEXT_KEYS.len()); + for key in CONTEXT_KEYS { + if let Some(value) = values.get_item(key)? { + read.push((ConfigNode::Str(key.to_owned()), config_node(&value)?)); + } + } + let context: PyConfigEntries = + model_context_values(&self.inputs, &read, model_name, include_target_values); config_dict(values.py(), &context) } diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index 97e321ecad..b574754880 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -72,3 +72,21 @@ class HeaderColumnPathTestCase: headers: tuple[str, ...] expected_native: int expected_parsed: int + + +@dataclass(frozen=True) +class TemplateTextTestCase: + """A MODEL header and CLI vars whose templated alias reads values as Python's text.""" + + description: str + header: dict[object, object] + cli_vars: dict[str, object] + expected_outcome: tuple[str, str] + + +@dataclass(frozen=True) +class OpaqueReadsTestCase: + """Whether an opaque header value reaches the config, and which of its texts compiling reads.""" + + description: str + expected_reads: tuple[bool, tuple[str, ...]] diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index c627503fe0..697c4439f6 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, fields, replace from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, TypeGuard, cast +from typing import TYPE_CHECKING, ClassVar, TypeGuard, cast import pytest @@ -573,7 +573,9 @@ def __repr__(self) -> str: MARKER: Marker = Marker() -def config_value_inputs(*, project_dir: Path, header: dict[object, object]) -> CompileInputsView: +def config_value_inputs( + *, project_dir: Path, header: dict[object, object], cli_vars: dict[str, object] | None = None +) -> CompileInputsView: """Build compile inputs for the config value project with its one model's header replaced.""" for relative_path, contents in CONFIG_VALUE_PROJECT.items(): @@ -604,6 +606,7 @@ def config_value_inputs(*, project_dir: Path, header: dict[object, object]) -> C ), sql_lexical_syntax=adapter.sql_lexical_syntax, ), + cli_vars=cli_vars, run_id="20261007T000000Z_orders", resolved_connection={}, no_sql_validation=True, @@ -674,3 +677,58 @@ def header_column_paths(*, project_dir: Path, headers: tuple[str, ...]) -> tuple ) work: dict[str, int] = _native.compile_stage_work_info(reset=True) return (work.get("native_header_columns", 0), work.get("parsed_header_columns", 0)) + + +class Exploding: + """An opaque value whose `str()` and `repr()` raise and count each call.""" + + calls: ClassVar[list[str]] = [] + + def __str__(self) -> str: + Exploding.calls.append("str") + raise ValueError("no str") + + def __repr__(self) -> str: + Exploding.calls.append("repr") + raise ValueError("no repr") + + +class ExplodingText(str): + """A string whose `str()` raises, which config unions print.""" + + __slots__ = () + + def __str__(self) -> str: + raise ValueError("no str") + + +def templated_alias( + *, project_dir: Path, header: dict[object, object], cli_vars: dict[str, object] +) -> tuple[str, str]: + """The model's built alias as `("alias", text)`, or the raised error as `(type, message)`.""" + + try: + config: CompileModelConfig = ( + config_value_inputs(project_dir=project_dir, header=header, cli_vars=cli_vars) + .model_inputs[0] + .config + ) + except Exception as error: + return (type(error).__name__, str(error)) + return ("alias", str(config.values.get("alias"))) + + +def opaque_reads(*, project_dir: Path) -> tuple[bool, tuple[str, ...]]: + """Whether an opaque header value reaches the config, and which of its texts compiling read.""" + + Exploding.calls.clear() + marker: Exploding = Exploding() + config: CompileModelConfig = ( + config_value_inputs( + project_dir=project_dir, + header={"description": "x", "alias": "${region}", Severity.WARN: marker}, + ) + .model_inputs[0] + .config + ) + return (config.values[cast("str", Severity.WARN)] is marker, tuple(Exploding.calls)) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py b/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py new file mode 100644 index 0000000000..cb1d0a8772 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_config_value_text.py @@ -0,0 +1,79 @@ +"""Config text read as Python reads it: containers by `str()`, errors raised as Python raises them.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + OpaqueReadsTestCase, + TemplateTextTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( + MARKER, + Exploding, + ExplodingText, + opaque_reads, + templated_alias, +) + +_SAME_ALIAS: str = "${if(eq(given, expected), 'same', 'different')}" + + +@pytest.mark.parametrize( + "test_case", + [ + TemplateTextTestCase( + description="a list holding a format character compares as Python prints it", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": ["\u200b\u00ad"], "expected": "['\\u200b\\xad']"}, + expected_outcome=("alias", "same"), + ), + TemplateTextTestCase( + description="a list holding an opaque object compares by the object's repr", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": [MARKER], "expected": "[Marker()]"}, + expected_outcome=("alias", "same"), + ), + TemplateTextTestCase( + description="a list whose item cannot be printed raises the item's error", + header={"alias": _SAME_ALIAS}, + cli_vars={"given": [Exploding()], "expected": "x"}, + expected_outcome=("ValueError", "no repr"), + ), + TemplateTextTestCase( + description="a tag union over an item that cannot be printed raises its error", + header={"tags": ["daily", ExplodingText("weekly")]}, + cli_vars={}, + expected_outcome=("ValueError", "no str"), + ), + ], + ids=lambda case: case.description, +) +def test_given_non_scalar_values_when_building_config_then_text_and_errors_match_python( + test_case: TemplateTextTestCase, tmp_path: Path +) -> None: + outcome: tuple[str, str] = templated_alias( + project_dir=tmp_path / "orders", header=test_case.header, cli_vars=test_case.cli_vars + ) + + assert outcome == test_case.expected_outcome + + +@pytest.mark.parametrize( + "test_case", + [ + OpaqueReadsTestCase( + description="an opaque value under a str-enum key beside a templated alias", + expected_reads=(True, ("str",)), + ), + ], + ids=lambda case: case.description, +) +def test_given_opaque_value_when_building_config_then_only_the_config_facts_read_its_text( + test_case: OpaqueReadsTestCase, tmp_path: Path +) -> None: + reads: tuple[bool, tuple[str, ...]] = opaque_reads(project_dir=tmp_path / "orders") + + assert reads == test_case.expected_reads From 0948d46d57165355a060c1d9492bf984fc0ef88d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:08:24 +0100 Subject: [PATCH 8/9] perf(native): carry header containers the config build never looks into whole Every header value was converted into config nodes, then cloned through layering and expansion, even large `columns` trees the build only passes through. A header container whose native parse found no template or macro, under a key the build never reads item by item (anything but tags, row-diff, hook and storage keys), is now carried as the header's own object. Dense compiles return to the instruction count they had before the typed build. Because such containers stay the header's own objects, models whose other config values hold templates also reuse their natively parsed columns now; an integration case pins it. --- .../_helpers/model_config/config_nodes.rs | 12 +++++ .../model_config/model_config_builder.rs | 45 ++++++++++++++++++- .../model_config/test_header_column_paths.py | 6 +++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs index 42e77b7927..aa1bbe6cdc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs @@ -174,6 +174,18 @@ fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult) -> PyResult { + host(value, HostChildren::None) +} + +/// Whether `value` is an exact `list`, `tuple` or `dict`. +pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { + value.is_exact_instance_of::() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::() +} + /// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. pub(crate) fn config_node(value: &Bound<'_, PyAny>) -> PyResult { match convert(value, &[])? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index d4c1015570..b634471b40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; +use sqlbuild_model_config::config_presence::models::MacroPath; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::model_config_build::main::build_model_config::build_model_config; @@ -25,7 +26,7 @@ use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::config_nodes::{ HostHandle, PyConfigEntries, PyConfigNode, config_dict, config_entries, config_node, - config_object, hook_entry, + config_object, hook_entry, is_exact_container, opaque_container, }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; @@ -127,6 +128,46 @@ impl ConfigBuildHost for PythonConfigHost<'_> { } } +/// Header keys whose values the build reads item by item, which are always converted. +const INSPECTED_KEYS: [&str; 7] = [ + "tags", + "row_diff_exclude_columns", + "row_diff_tolerances", + "pre_hooks", + "post_hooks", + "time_travel_retention", + "table_type", +]; + +/// Whether `key`'s container may be carried whole: no template or macro in it, items never read. +fn carried_whole(key: &str, facts: Option<&[(String, HeaderEntryFacts)]>) -> bool { + !INSPECTED_KEYS.contains(&key) + && facts + .and_then(|facts| facts.iter().find(|(name, _)| name == key)) + .is_some_and(|(_, facts)| !facts.template && facts.macro_path == MacroPath::Absent) +} + +/// A header's entries as config nodes; containers the build never looks into stay opaque. +fn header_entries( + header: &Bound<'_, PyDict>, + facts: Option<&[(String, HeaderEntryFacts)]>, +) -> PyResult { + let mut entries: PyConfigEntries = Vec::with_capacity(header.len()); + for (key, value) in header.iter() { + let whole: bool = is_exact_container(&value) + && key + .downcast_exact::() + .is_ok_and(|key| key.to_str().is_ok_and(|key| carried_whole(key, facts))); + let node: PyConfigNode = if whole { + opaque_container(&value)? + } else { + config_node(&value)? + }; + entries.push((config_node(&key)?, node)); + } + Ok(entries) +} + /// One header entry's parsed facts as the build reads them. fn entry_facts((key, entry): &(String, EntryFacts)) -> (String, HeaderEntryFacts) { let facts: HeaderEntryFacts = HeaderEntryFacts { @@ -300,9 +341,9 @@ impl NativeModelConfigBuilder { matched_path_default: Option<&str>, model_name: &str, ) -> PyResult> { - let entries: PyConfigEntries = config_entries(header)?; let facts: Option> = facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); + let entries: PyConfigEntries = header_entries(header, facts.as_deref())?; let model_header = ModelHeader { entries: &entries, facts: facts.as_deref(), diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py index d52285d004..a17748b6a7 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_header_column_paths.py @@ -25,6 +25,12 @@ expected_native=3, expected_parsed=0, ), + HeaderColumnPathTestCase( + description="columns in a model whose other config values hold templates", + headers=('alias "orders_${region}", columns (order_id (type INTEGER))',), + expected_native=1, + expected_parsed=0, + ), ], ids=lambda case: case.description, ) From 45e6afa40943a0a693122142628b27a8be53f32a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:04:41 +0100 Subject: [PATCH 9/9] perf(native): convert typed config values key by key on first read The first typed read of a config record converted every value in `config.values` into typed nodes, just to read one scalar key such as `incremental_mode`. A config record now keeps a per-key cache: a value is converted the first time its key is read, through the same `values.get(key)`, and kept for later reads. Checked reads keep their type rules and errors. Lineage's microbatch check compares the mode text as Python's `==` does again, so a string that cannot be encoded reads as not-microbatch instead of raising. A stage counter records each typed conversion, and an integration test pins that keys no planner reader reads are never converted. --- .../_helpers/compile_driver/stage_work.rs | 2 + .../_helpers/model_config/typed_config.rs | 139 +++++++++++------- .../_helpers/planner_strategy/lineage.rs | 10 +- fensu.toml | 2 +- .../sqlbuild/cli/commands/main/_test_types.py | 9 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 23 +++ .../commands/main/test_plan_typed_config.py | 38 +++++ 7 files changed, 166 insertions(+), 57 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 53fbaa31fa..cc89abc6c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -30,6 +30,8 @@ pub(crate) const BACKGROUND_SQL_TEST_PLANS: &str = "background_sql_test_plans"; pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; /// Models whose header columns are parsed again from their config values. pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; +/// Config values converted into typed nodes for a key a reader asked for. +pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads"; /// Compile stage runs since the last reset, by stage. #[pyclass] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs index 997e7fcc5c..593a3ca009 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs @@ -1,21 +1,21 @@ -//! Model config values read as typed config nodes, kept on the config record once converted. +//! Model config values read as typed config nodes, each key converted on first read and kept. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyclass; -use pyo3::types::PyDict; use sqlbuild_model_config::config_values::errors::ConfigTypeError; use sqlbuild_model_config::config_values::main::checked_text::checked_text; -use sqlbuild_model_config::config_values::main::config_entry::config_entry; use sqlbuild_model_config::config_values::models::ConfigNode; +use crate::bindings::_helpers::compile_driver::stage_work::{TYPED_CONFIG_READS, count_stage}; use crate::bindings::_helpers::model_config::config_nodes::{ - HostHandle, PyConfigEntries, PyConfigNode, config_entries, config_node, config_object, + HostHandle, PyConfigNode, config_node, config_object, }; use crate::bindings::_helpers::model_config::config_values::config_value_type_error; use crate::bindings::_helpers::records::fields::{ - cached_value, field as native_field, store_cached, + cached_value, field as native_field, mapping_get, store_cached, }; use crate::bindings::_helpers::records::models::NativeRecord; @@ -23,61 +23,73 @@ const CONFIG_FIELD: &str = "config"; const VALUES_FIELD: &str = "values"; /// The config record's cache slot holding its typed values; no Python attribute reads it. const TYPED_VALUES_SLOT: &str = "__typed_values__"; +const POISONED: &str = "typed config values cache is poisoned"; -/// A config record's values as typed config nodes. +/// The typed nodes of a config record's values read so far, by key. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct TypedConfigValues { - entries: Arc, + read: Mutex)>>, } -/// The entries of a values mapping: a `dict` read directly, any other mapping through `items()`. -fn mapping_entries(values: &Bound<'_, PyAny>) -> PyResult { - if let Ok(mapping) = values.cast::() { - return config_entries(mapping); +/// A config record's values and the typed nodes read of them, each key converted on first read. +pub(crate) type ConfigValues<'py> = (Bound<'py, PyAny>, Bound<'py, TypedConfigValues>); + +/// `values.get(key)` as a typed node, `None` when absent; each key is converted once. +fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { + let (mapping, typed) = values; + let py: Python<'_> = mapping.py(); + let read: &Mutex)>> = &typed.get().read; + let poisoned = || PyRuntimeError::new_err(POISONED); + let found: Option> = read + .lock() + .map_err(|_| poisoned())? + .iter() + .find(|(name, _)| name == key) + .map(|(_, node)| Arc::clone(node)); + if let Some(node) = found { + return Ok(node); } - values - .call_method0("items")? - .try_iter()? - .map(|item| { - let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - Ok((config_node(&key)?, config_node(&value)?)) - }) - .collect() + let node: Arc = Arc::new(config_node(&mapping_get(mapping, key)?)?); + count_stage(py, TYPED_CONFIG_READS, 1)?; + read.lock() + .map_err(|_| poisoned())? + .push((key.to_owned(), Arc::clone(&node))); + Ok(node) } -/// Keep `entries` as the typed values of the config record `config`. -pub(crate) fn keep_typed_values( - config: &Bound<'_, PyAny>, - entries: Arc, -) -> PyResult<()> { - let Ok(record) = config.cast::() else { - return Ok(()); +/// A config record's values, their typed nodes kept on the record so each key converts once. +pub(crate) fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = config.py(); + let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; + let fresh = || { + Bound::new( + py, + TypedConfigValues { + read: Mutex::new(Vec::new()), + }, + ) }; - let typed: Py = Py::new(config.py(), TypedConfigValues { entries })?; - store_cached(record, TYPED_VALUES_SLOT, &typed.into_any()) -} - -/// A config record's values as typed nodes, converted from its `values` once and kept. -pub(crate) fn typed_values(config: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = config.py(); let Ok(record) = config.cast::() else { - return Ok(Arc::new(mapping_entries(&native_field( - config, - VALUES_FIELD, - )?)?)); + let typed: Bound<'py, TypedConfigValues> = fresh()?; + return Ok((values, typed)); }; - if let Some(typed) = cached_value(record, TYPED_VALUES_SLOT)? { - let typed: Bound<'_, TypedConfigValues> = typed.into_bound(py).cast_into()?; - return Ok(Arc::clone(&typed.get().entries)); - } - let entries: Arc = - Arc::new(mapping_entries(&native_field(config, VALUES_FIELD)?)?); - keep_typed_values(config, Arc::clone(&entries))?; - Ok(entries) + let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { + Some(typed) => typed.into_bound(py).cast_into()?, + None => { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + store_cached( + record, + TYPED_VALUES_SLOT, + &typed.clone().into_any().unbind(), + )?; + typed + } + }; + Ok((values, typed)) } -/// A model's config values as typed nodes. -pub(crate) fn model_values(model: &Bound<'_, PyAny>) -> PyResult> { +/// A model's config values as typed nodes, read key by key. +pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { typed_values(&native_field(model, CONFIG_FIELD)?) } @@ -108,10 +120,11 @@ pub(crate) fn node_text(py: Python<'_>, node: &PyConfigNode) -> PyResult /// `key`'s string value, or `None` when absent or `None`. pub(crate) fn config_text( py: Python<'_>, - entries: &[(PyConfigNode, PyConfigNode)], + values: &ConfigValues<'_>, key: &str, ) -> PyResult> { - checked_text(key, config_entry(entries, key)) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) .map_err(|error| type_error(py, &error))? .map(|value| node_text(py, value)) .transpose() @@ -120,11 +133,31 @@ pub(crate) fn config_text( /// `key`'s string value as its Python object, or `None` when absent or `None`. pub(crate) fn config_text_object<'py>( py: Python<'py>, - entries: &[(PyConfigNode, PyConfigNode)], + values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - checked_text(key, config_entry(entries, key)) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) .map_err(|error| type_error(py, &error))? .map(|value| config_object(py, value)) .transpose() } + +/// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. +pub(crate) fn config_text_is( + py: Python<'_>, + values: &ConfigValues<'_>, + key: &str, + expected: &str, +) -> PyResult { + let node: Arc = node(values, key)?; + let text: Option<&PyConfigNode> = + checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; + Ok(match text { + Some(ConfigNode::Str(text)) => text == expected, + Some(ConfigNode::Host(host)) if host.facts.strict_text => { + host.facts.text.as_deref() == Some(expected) + } + _ => false, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 860c4a660e..e25f1f94c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; +use crate::bindings::_helpers::model_config::typed_config::{config_text_is, model_values}; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -49,8 +49,12 @@ fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult> /// Whether a model's `incremental_mode` is microbatch. fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { - let mode: Option = config_text(model.py(), &model_values(model)?, "incremental_mode")?; - Ok(mode.as_deref() == Some(MICROBATCH_MODE)) + config_text_is( + model.py(), + &model_values(model)?, + "incremental_mode", + MICROBATCH_MODE, + ) } /// A model's watermark input names. diff --git a/fensu.toml b/fensu.toml index 893f0d0bf7..e9e6f6c632 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1949,7 +1949,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 2cbd009334..9d10264b4c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1199,3 +1199,12 @@ class CustomHostOutcomeTestCase: rule_body: str expected_exit_code: int expected_messages: tuple[str, ...] + + +@dataclass(frozen=True) +class TypedConfigReadsTestCase: + """Config keys planning never reads, added to every model of a planned project.""" + + description: str + unread_keys: str + expected_same_reads: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index df194d7a58..74bc48c9e9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -985,3 +985,26 @@ def diagnostics_by_model(diagnostics: list[dict[str, object]]) -> dict[str, str] name: str = cast(str, item["resource_name"]) by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") return by_model + + +def typed_config_reads(*, project_dir: Path, extra_header: str) -> int: + """How many config values planning converted into typed nodes for three cursor models.""" + + project_dir.mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "main"\n', + encoding="utf-8", + ) + models_dir: Path = project_dir / "models" + models_dir.mkdir() + for index in range(3): + (models_dir / f"orders_{index}.sql").write_text( + f'MODEL (description "Orders.", materialized table, tags ["daily"]{extra_header});\n\n' + f"SELECT {index} AS order_id, 1 AS loaded_at\n", + encoding="utf-8", + ) + _ = native_module.compile_stage_work_info(reset=True) + exit_code: int = main(["--project-dir", str(project_dir), "plan", "--json", "--no-cache"]) + assert exit_code == 0 + return native_module.compile_stage_work_info(reset=True).get("typed_config_reads", 0) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py b/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py new file mode 100644 index 0000000000..b657e430c8 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_plan_typed_config.py @@ -0,0 +1,38 @@ +"""Planning converts only the config values its readers ask for.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + TypedConfigReadsTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import typed_config_reads + + +@pytest.mark.parametrize( + "test_case", + [ + TypedConfigReadsTestCase( + description="row-diff settings no planner reader reads", + unread_keys=( + ', row_diff_exclude_columns ["loaded_at"], row_diff_sample_seed 7, ' + "row_diff_tolerances (default 1)" + ), + expected_same_reads=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_config_keys_planning_never_reads_when_planning_then_they_are_not_converted( + test_case: TypedConfigReadsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + plain: int = typed_config_reads(project_dir=tmp_path / "plain", extra_header="") + extended: int = typed_config_reads( + project_dir=tmp_path / "extended", extra_header=test_case.unread_keys + ) + _ = capsys.readouterr() + + assert (plain > 0, extended == plain) == (True, test_case.expected_same_reads)