diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs index 6ef36ec6bf..d49ae9c8e6 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs @@ -2,10 +2,8 @@ use sqlbuild_core::text::main::is_word_char::is_word_char; -use crate::compiled_project::constants::{ - CURRENT_DIRECTORY, NEWLINE, OPT_OUT_KEYS, PATH_SEPARATOR, -}; -use crate::compiled_project::models::{LegacySchemaFile, TextPosition}; +use crate::compiled_project::constants::{NEWLINE, OPT_OUT_KEYS}; +use crate::compiled_project::models::TextPosition; /// `_text_location`: the first whole-word opt-out key at or after character `start`. pub(crate) fn opt_out_position(text: &str, start: usize) -> TextPosition { @@ -53,43 +51,3 @@ fn starts_key(characters: &[char], index: usize) -> bool { } false } - -/// The error for a model a legacy schema file under its directory still declares. -pub(crate) fn legacy_schema_error( - model: (&str, &str), - schema_files: &[LegacySchemaFile], -) -> Option { - let (name, relative_path) = model; - let model_parts: Vec<&str> = parts(relative_path); - let mut matches: Vec<&str> = Vec::new(); - for file in schema_files { - let directory: Vec<&str> = parts(&file.relative_path); - if !model_parts.starts_with(&directory[..directory.len().saturating_sub(1)]) { - continue; - } - for entry in &file.model_names { - if entry == name { - matches.push(&file.relative_path); - } - } - } - match matches.as_slice() { - [] => None, - [file] => Some(format!( - "Schema file {file} declares model '{name}', but model metadata must live in \ - {relative_path} MODEL(...). Move description, columns, and audits into the model \ - header." - )), - files => Some(format!( - "Model file {relative_path} matched multiple schema.yml declarations: {}", - files.join(", ") - )), - } -} - -/// A relative path's parts, as `PurePosixPath.parts` gives them (`.` has none). -fn parts(path: &str) -> Vec<&str> { - path.split(PATH_SEPARATOR) - .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) - .collect() -} diff --git a/crates/sqlbuild-analysis/src/compiled_project/constants.rs b/crates/sqlbuild-analysis/src/compiled_project/constants.rs index 83408513be..aac375e399 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/constants.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/constants.rs @@ -19,6 +19,4 @@ pub(crate) const INVALID_CONSTANT_REFERENCE_STOP: u8 = 3; pub(crate) const MACRO_TOKEN: char = '@'; /// The keys whose `false` value opts a model out of SQL analysis. pub(crate) const OPT_OUT_KEYS: [&str; 2] = ["sql_analysis", "sql_validation"]; -pub(crate) const PATH_SEPARATOR: char = '/'; -pub(crate) const CURRENT_DIRECTORY: &str = "."; pub(crate) const NEWLINE: char = '\n'; diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs b/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs deleted file mode 100644 index 58f235366d..0000000000 --- a/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::compiled_project::models::LegacySchemaFile; - -/// The error for a model `(name, relative path)` that a legacy schema file still declares. -pub fn legacy_schema_error( - model: (&str, &str), - schema_files: &[LegacySchemaFile], -) -> Option { - crate::compiled_project::_helpers::sources::legacy_schema_error(model, schema_files) -} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs index a6c274ad63..c9a5f9143b 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs @@ -4,7 +4,6 @@ pub mod calls_macros; pub mod declaration_error_message; pub mod enum_member_literal; pub mod header_metadata_keys; -pub mod legacy_schema_error; pub mod located_scan_error; pub mod merge_call_site_references; pub mod model_facts; diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index a2f5abe0cc..c99bb198be 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -226,13 +226,6 @@ pub struct TextPosition { pub column: usize, } -/// A legacy schema file that declares model entries, by its relative path. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct LegacySchemaFile { - pub relative_path: String, - pub model_names: Vec, -} - /// The authored model a reference-scan fault is located in. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ScanFaultOrigin<'a> { diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs index f597e14154..1cbd5c6831 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs @@ -38,20 +38,6 @@ pub(super) fn analysing_settings( } } -pub(super) fn legacy_schema_files( - files: &[(&str, &[&str])], -) -> Vec { - files - .iter() - .map( - |(path, names)| crate::compiled_project::models::LegacySchemaFile { - relative_path: (*path).to_owned(), - model_names: names.iter().map(|name| (*name).to_owned()).collect(), - }, - ) - .collect() -} - pub(super) fn generic_syntax() -> sqlbuild_sqltext::sql_scan::models::LexicalSyntax { sqlbuild_sqltext::sql_scan::models::LexicalSyntax { line_comment_prefixes: vec!["--".to_owned()], diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs index cbd543952e..0e229e7307 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs @@ -1,9 +1,7 @@ -use crate::compiled_project::main::legacy_schema_error::legacy_schema_error; use crate::compiled_project::main::opt_out_position::opt_out_position; use crate::compiled_project::main::text_position::text_position; use crate::compiled_project::models::TextPosition; -use crate::compiled_project::tests::helpers::legacy_schema_files; -use crate::compiled_project::tests::test_types::{LegacySchemaTestCase, OptOutPositionTestCase}; +use crate::compiled_project::tests::test_types::OptOutPositionTestCase; #[test] fn given_text_when_locating_opt_out_then_first_whole_word_key_after_start_is_found() { @@ -38,50 +36,6 @@ fn given_text_when_locating_opt_out_then_first_whole_word_key_after_start_is_fou } } -#[test] -fn given_legacy_schema_files_when_matching_model_then_attachment_error_is_returned() { - let test_cases = [ - LegacySchemaTestCase { - description: "schema file in another directory", - schema_files: &[("models/staging/schema.yml", &["orders"])], - expected_error: None, - }, - LegacySchemaTestCase { - description: "root schema file declares the model", - schema_files: &[("schema.yml", &["orders"])], - expected_error: Some( - "Schema file schema.yml declares model 'orders', but model metadata must live in \ - models/marts/orders.sql MODEL(...). Move description, columns, and audits into \ - the model header.", - ), - }, - LegacySchemaTestCase { - description: "two schema files declare the model", - schema_files: &[ - ("models/schema.yml", &["orders"]), - ("models/marts/schema.yml", &["orders", "customers"]), - ], - expected_error: Some( - "Model file models/marts/orders.sql matched multiple schema.yml declarations: \ - models/schema.yml, models/marts/schema.yml", - ), - }, - ]; - - for test_case in test_cases { - assert_eq!( - legacy_schema_error( - ("orders", "models/marts/orders.sql"), - &legacy_schema_files(test_case.schema_files) - ) - .as_deref(), - test_case.expected_error, - "{}", - test_case.description - ); - } -} - #[test] fn given_character_offset_when_positioning_then_line_and_column_count_characters() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs index 5f9946a4aa..8c6e4484b8 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs @@ -51,13 +51,6 @@ pub(super) struct OptOutPositionTestCase { pub(super) expected_position: crate::compiled_project::models::TextPosition, } -pub(super) struct LegacySchemaTestCase { - pub(super) description: &'static str, - /// `(schema file, model names)` per legacy schema file. - pub(super) schema_files: &'static [(&'static str, &'static [&'static str])], - pub(super) expected_error: Option<&'static str>, -} - pub(super) struct LocatedScanErrorTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 42d1a928f3..b474f08964 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -14,6 +14,7 @@ pub mod graph; pub mod lineage; pub mod query_analysis; pub mod relation_names; +pub mod required_columns; pub mod resource_diagnostics; pub mod semantic_checks; pub mod semantic_usage; diff --git a/crates/sqlbuild-analysis/src/required_columns/_helpers/mod.rs b/crates/sqlbuild-analysis/src/required_columns/_helpers/mod.rs new file mode 100644 index 0000000000..113bd236a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod payload; +pub(crate) mod queries; +pub(crate) mod relations; +pub(crate) mod tree; diff --git a/crates/sqlbuild-analysis/src/required_columns/_helpers/payload.rs b/crates/sqlbuild-analysis/src/required_columns/_helpers/payload.rs new file mode 100644 index 0000000000..464d72d2ff --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/_helpers/payload.rs @@ -0,0 +1,66 @@ +//! Expression payloads as the Polyglot wheel's `args` exposed them, and walks over them. + +use polyglot_sql::{Expression, ast_json}; +use serde_json::{Map, Value}; + +use crate::required_columns::_helpers::tree::payload; +use crate::required_columns::constants::{ + KIND_COLUMN, KIND_SELECT, KIND_SUBQUERY, SET_OPERATION_KINDS, +}; + +/// `expression.args.get(key)`, unless the wheel exposed it as expression objects. +pub(crate) fn arg(expression: &Expression, key: &str) -> Option { + plain_value(payload(expression).remove(key)) +} + +/// `value`, unless the wheel exposed it as an expression or a list of expressions. +pub(crate) fn plain_value(value: Option) -> Option { + let value: Value = value?; + ast_json::expression_from_value(value.clone()).err()?; + ast_json::expressions_from_value(value.clone()).err()?; + Some(value) +} + +/// `_payload_name`: a payload mapping's string `name`, else empty. +pub(crate) fn payload_name(payload: Option<&Value>) -> String { + payload + .and_then(Value::as_object) + .and_then(|map| map.get("name")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned() +} + +/// `_column_refs_in_payload`: each column outside nested queries, in the wheel's sorted key order. +pub(crate) fn column_refs_in_payload(payload: Option<&Value>) -> Vec<(String, String)> { + match payload { + Some(Value::Array(items)) => items + .iter() + .flat_map(|item| column_refs_in_payload(Some(item))) + .collect(), + Some(Value::Object(map)) => object_column_refs(map), + _ => Vec::new(), + } +} + +fn object_column_refs(map: &Map) -> Vec<(String, String)> { + let nested_query: bool = map.contains_key(KIND_SUBQUERY) + || map.contains_key(KIND_SELECT) + || SET_OPERATION_KINDS + .iter() + .any(|kind| map.contains_key(*kind)); + if nested_query { + return Vec::new(); + } + if let Some(Value::Object(column)) = map.get(KIND_COLUMN) { + return vec![( + payload_name(column.get("name")), + payload_name(column.get("table")), + )]; + } + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + keys.into_iter() + .flat_map(|key| column_refs_in_payload(map.get(key))) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/required_columns/_helpers/queries.rs b/crates/sqlbuild-analysis/src/required_columns/_helpers/queries.rs new file mode 100644 index 0000000000..03ce96d702 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/_helpers/queries.rs @@ -0,0 +1,544 @@ +//! The column-read walk over one query and the queries nested inside it. + +use std::rc::Rc; + +use polyglot_sql::Expression; +use serde_json::{Map, Value}; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::required_columns::_helpers::payload::{ + arg, column_refs_in_payload, payload_name, plain_value, +}; +use crate::required_columns::_helpers::relations::{ + RelationMap, RelationReads, either, unique_reads, unique_resources, +}; +use crate::required_columns::_helpers::tree::{ + alias_or_name, children, is_star, kind, payload, set_operation, top_level_ctes, + unwrap_annotations, +}; +use crate::required_columns::constants::{ + CLAUSE_NAMES, KIND_COLUMN, KIND_SELECT, KIND_SUBQUERY, KIND_TABLE, + OUTPUT_ALIAS_FALLBACK_CLAUSES, OUTPUT_ALIAS_FIRST_CLAUSES, SET_OPERATION_KINDS, +}; +use crate::required_columns::models::ColumnRead; + +/// Each `JOIN ... USING` column's reads, by casefolded name, in first-joined order. +type SharedReads = Vec<(String, Vec)>; +/// Named output columns and the reads each stands for. +type Slots = Vec<(String, Vec)>; + +/// The relations one select's column names resolve against. +struct Scope<'a> { + local: &'a RelationMap, + outer: &'a RelationMap, + shared: &'a SharedReads, +} + +fn shared_get<'a>(shared: &'a SharedReads, name: &str) -> Option<&'a Vec> { + shared + .iter() + .find(|(key, _)| key == name) + .map(|(_, reads)| reads) +} + +fn is_set_operation(kind_name: &str) -> bool { + SET_OPERATION_KINDS.contains(&kind_name) +} + +/// `_resolve_query_reads`: a query's output relation and the external columns it reads. +pub(crate) fn query_reads( + query: &Expression, + inherited: &RelationMap, + outer: &RelationMap, +) -> (RelationReads, Vec) { + let query: &Expression = unwrap_annotations(query); + let mut relations: RelationMap = inherited.clone(); + let mut required: Vec = Vec::new(); + for (cte_name, body, has_column_aliases) in top_level_ctes(query) { + let (mut cte_reads, cte_required) = query_reads(body, &relations, outer); + required.extend(cte_required); + if has_column_aliases { + cte_reads = cte_column_aliases(query, cte_name, &cte_reads); + } + relations.insert(python_casefold(cte_name), Rc::new(cte_reads)); + } + let query_kind: &str = kind(query); + if is_set_operation(query_kind) { + let (reads, set_required) = set_operation_reads(query, &relations, outer); + required.extend(set_required); + return (reads, unique_reads(required)); + } + if query_kind != KIND_SELECT { + return (RelationReads::default(), unique_reads(required)); + } + let mut local: RelationMap = select_tables(query, &relations); + let mut nested_required: Vec = Vec::new(); + for (nested, alias) in direct_nested_queries(query) { + let (nested_reads, reads) = query_reads(nested, &relations, &outer.merged(&local)); + nested_required.extend(reads); + if !alias.is_empty() { + local.insert(python_casefold(alias), Rc::new(nested_reads)); + } + } + let shared: SharedReads = join_using_column_reads(query, &local); + let scope: Scope<'_> = Scope { + local: &local, + outer, + shared: &shared, + }; + let (slots, passthrough, projection_required) = projection_slots(query, &scope); + required.extend(clause_reads(query, &slots, &scope)); + for (_, reads) in &shared { + required.extend(reads.iter().cloned()); + } + for (_, slot) in &slots { + required.extend(slot.iter().cloned()); + } + required.extend(projection_required); + required.extend(nested_required); + let relation: RelationReads = RelationReads { + slots, + passthrough: unique_resources(passthrough), + }; + (relation, unique_reads(required)) +} + +/// Each direct table the select reads that resolves to a known relation, by name and alias. +fn select_tables(query: &Expression, relations: &RelationMap) -> RelationMap { + let mut local: RelationMap = RelationMap::default(); + for table in children(query) { + if kind(table) != KIND_TABLE { + continue; + } + let name: String = python_casefold(table.get_name()); + let Some(relation) = relations.get(&name) else { + continue; + }; + let relation: Rc = Rc::clone(relation); + local.insert(name, Rc::clone(&relation)); + let alias: &str = alias_or_name(table); + if !alias.is_empty() { + local.insert(python_casefold(alias), relation); + } + } + local +} + +/// The select's output slots, passthrough resources and projection reads. +fn projection_slots( + query: &Expression, + scope: &Scope<'_>, +) -> (Slots, Vec<(String, String)>, Vec) { + let mut slots: Slots = Vec::new(); + let mut passthrough: Vec<(String, String)> = Vec::new(); + let mut projection_required: Vec = Vec::new(); + for raw in query.get_expressions() { + let projection: &Expression = unwrap_annotations(raw); + if is_star(projection) { + for relation in star_relations(projection, scope.local) { + slots.extend(relation.slots.iter().cloned()); + passthrough.extend(relation.passthrough.iter().cloned()); + } + slots.extend(star_modifier_slots(projection, scope)); + continue; + } + let output_name: &str = projection.get_output_name(); + let leaves: Vec = expression_reads(projection, scope); + projection_required.extend(leaves.iter().cloned()); + if !output_name.is_empty() { + slots.push((output_name.to_owned(), leaves)); + } + } + (slots, passthrough, projection_required) +} + +/// `_resolve_set_operation_reads`. +fn set_operation_reads( + operation: &Expression, + inherited: &RelationMap, + outer: &RelationMap, +) -> (RelationReads, Vec) { + let Some((left, right, by_name)) = set_operation(operation) else { + return (RelationReads::default(), Vec::new()); + }; + let (left, mut required) = query_reads(left, inherited, outer); + let (right, right_required) = query_reads(right, inherited, outer); + required.extend(right_required); + let slots: Slots = if by_name { + by_name_slots(&left.slots, &right.slots) + } else if left.slots.len() == right.slots.len() { + left.slots + .iter() + .zip(&right.slots) + .map(|((name, left_leaves), (_, right_leaves))| { + (name.clone(), combined(left_leaves, right_leaves)) + }) + .collect() + } else { + Vec::new() + }; + let mut passthrough: Vec<(String, String)> = left.passthrough.clone(); + passthrough.extend(right.passthrough.iter().cloned()); + let relation: RelationReads = RelationReads { + slots, + passthrough: unique_resources(passthrough), + }; + (relation, unique_reads(required)) +} + +/// A `BY NAME` set operation's slots: each left column with the right column of its name. +fn by_name_slots(left: &Slots, right: &Slots) -> Slots { + let mut right_by_name: Vec<(String, &Vec)> = Vec::new(); + for (name, leaves) in right { + let key: String = python_casefold(name); + match right_by_name + .iter_mut() + .find(|(existing, _)| *existing == key) + { + Some(entry) => entry.1 = leaves, + None => right_by_name.push((key, leaves)), + } + } + let mut slots: Slots = Vec::with_capacity(left.len()); + for (name, leaves) in left { + let key: String = python_casefold(name); + let empty: Vec = Vec::new(); + let right_leaves: &Vec = right_by_name + .iter() + .find(|(existing, _)| *existing == key) + .map_or(&empty, |(_, leaves)| leaves); + slots.push((name.clone(), combined(leaves, right_leaves))); + } + slots +} + +fn combined(left: &[ColumnRead], right: &[ColumnRead]) -> Vec { + let mut reads: Vec = left.to_vec(); + reads.extend(right.iter().cloned()); + unique_reads(reads) +} + +/// `_resolve_expression_reads`. +fn expression_reads(expression: &Expression, scope: &Scope<'_>) -> Vec { + let mut resolved: Vec = Vec::new(); + for (column_name, table_name) in direct_column_refs(expression) { + if column_name.is_empty() { + continue; + } + if !table_name.is_empty() { + if let Some(relation) = either(scope.local, scope.outer, &table_name) { + resolved.extend(relation.resolve(&column_name)); + } + continue; + } + resolved.extend(unqualified_column(&column_name, scope).1); + } + unique_reads(resolved) +} + +/// `_resolve_clause_reads`. +fn clause_reads(select: &Expression, output_slots: &Slots, scope: &Scope<'_>) -> Vec { + let mut resolved: Vec = Vec::new(); + let mut select_args: Map = payload(select); + for clause_name in CLAUSE_NAMES { + let payload: Option = plain_value(select_args.remove(clause_name)); + for (column_name, table_name) in column_refs_in_payload(payload.as_ref()) { + if column_name.is_empty() { + continue; + } + if !table_name.is_empty() { + if let Some(relation) = either(scope.local, scope.outer, &table_name) { + resolved.extend(relation.resolve(&column_name)); + } + continue; + } + let output_match: Option<&Vec> = output_slot(&column_name, output_slots); + if let Some(leaves) = output_match + && OUTPUT_ALIAS_FIRST_CLAUSES.contains(&clause_name) + { + resolved.extend(leaves.iter().cloned()); + continue; + } + let (input_present, input_reads) = unqualified_column(&column_name, scope); + if input_present { + resolved.extend(input_reads); + continue; + } + if let Some(leaves) = output_match + && OUTPUT_ALIAS_FALLBACK_CLAUSES.contains(&clause_name) + { + resolved.extend(leaves.iter().cloned()); + } + } + } + unique_reads(resolved) +} + +/// `_output_slot_reads`: the last output slot named `column_name`. +fn output_slot<'a>(column_name: &str, output_slots: &'a Slots) -> Option<&'a Vec> { + let normalized: String = python_casefold(column_name); + output_slots + .iter() + .rev() + .find(|(name, _)| python_casefold(name) == normalized) + .map(|(_, leaves)| leaves) +} + +/// `_star_modifier_slots`: slots a star's `REPLACE` and `RENAME` modifiers add. +fn star_modifier_slots(projection: &Expression, scope: &Scope<'_>) -> Slots { + let mut slots: Slots = Vec::new(); + let expressions: Vec<&Expression> = children(projection); + if let Some(Value::Array(replacements)) = arg(projection, "replace") + && replacements.len() == expressions.len() + { + for (replacement, expression) in replacements.iter().zip(expressions) { + let Some(replacement) = replacement.as_object() else { + continue; + }; + let output_name: String = payload_name(replacement.get("alias")); + if output_name.is_empty() { + continue; + } + slots.push((output_name, expression_reads(expression, scope))); + } + } + if let Some(Value::Array(renames)) = arg(projection, "rename") { + for rename in &renames { + let Some([input, output]) = rename.as_array().map(Vec::as_slice) else { + continue; + }; + let input_name: String = payload_name(Some(input)); + let output_name: String = payload_name(Some(output)); + if input_name.is_empty() || output_name.is_empty() { + continue; + } + slots.push((output_name, unqualified_column(&input_name, scope).1)); + } + } + slots +} + +/// `_resolve_unqualified_column_with_presence`. +fn unqualified_column(column_name: &str, scope: &Scope<'_>) -> (bool, Vec) { + if let Some(reads) = shared_get(scope.shared, &python_casefold(column_name)) { + return (true, reads.clone()); + } + let local: Vec> = present_candidates(scope.local, column_name); + match local.len() { + 1 => return (true, local.into_iter().next().unwrap_or_default()), + 0 => {} + _ => return (true, Vec::new()), + } + let mut outer: Vec> = present_candidates(scope.outer, column_name); + if outer.len() == 1 { + return (true, outer.remove(0)); + } + (!outer.is_empty(), Vec::new()) +} + +/// The reads of each distinct relation that has `column_name`. +fn present_candidates(relations: &RelationMap, column_name: &str) -> Vec> { + let mut candidates: Vec> = Vec::new(); + for relation in relations.unique() { + let (present, leaves) = relation.resolve_with_presence(column_name); + if present { + candidates.push(leaves); + } + } + candidates +} + +/// `_join_using_column_reads`. +fn join_using_column_reads(select: &Expression, local: &RelationMap) -> SharedReads { + let Some(Value::Array(joins)) = arg(select, "joins") else { + return Vec::new(); + }; + let sequence: Vec> = joined_relation_sequence(select, local); + let Some(first) = sequence.first() else { + return Vec::new(); + }; + let mut joined: Vec> = vec![Rc::clone(first)]; + let mut joined_shared: SharedReads = Vec::new(); + let mut by_column: SharedReads = Vec::new(); + for (index, join) in (1..).zip(&joins) { + let Some(join) = join.as_object() else { + continue; + }; + if index >= sequence.len() { + break; + } + let right: &Rc = &sequence[index]; + if let Some(Value::Array(columns)) = join.get("using") { + for column in columns { + let name: String = payload_name(Some(column)); + if name.is_empty() { + continue; + } + let key: String = python_casefold(&name); + let combined_reads: Vec = + using_column_reads(&name, &joined, right, shared_get(&joined_shared, &key)); + match joined_shared + .iter_mut() + .find(|(existing, _)| *existing == key) + { + Some(entry) => entry.1 = combined_reads.clone(), + None => joined_shared.push((key.clone(), combined_reads.clone())), + } + match by_column.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1.extend(combined_reads), + None => by_column.push((key, combined_reads)), + } + } + } + joined.push(Rc::clone(right)); + } + by_column + .into_iter() + .map(|(name, reads)| (name, unique_reads(reads))) + .collect() +} + +/// One `USING` column's reads: the joined side's, unless ambiguous, then the right side's. +fn using_column_reads( + name: &str, + joined: &[Rc], + right: &RelationReads, + left_shared: Option<&Vec>, +) -> Vec { + let left: Vec = match left_shared { + Some(reads) => reads.clone(), + None => { + let mut candidates: Vec> = Vec::new(); + for relation in joined { + let (present, leaves) = relation.resolve_with_presence(name); + if present { + candidates.push(leaves); + } + } + if candidates.len() == 1 { + candidates.remove(0) + } else { + Vec::new() + } + } + }; + combined(&left, &right.resolve_with_presence(name).1) +} + +/// The known relations of the select's direct tables and subqueries, in tree order. +fn joined_relation_sequence(select: &Expression, local: &RelationMap) -> Vec> { + let mut sequence: Vec> = Vec::new(); + for child in children(select) { + let name: &str = match kind(child) { + KIND_TABLE => child.get_name(), + KIND_SUBQUERY => alias_or_name(child), + _ => continue, + }; + if let Some(relation) = local.get(&python_casefold(name)) { + sequence.push(Rc::clone(relation)); + } + } + sequence +} + +/// `_direct_column_refs`: `(column, table)` of each column outside nested queries. +fn direct_column_refs(expression: &Expression) -> Vec<(String, String)> { + let expression_kind: &str = kind(expression); + if expression_kind == KIND_COLUMN { + let table: Option = arg(expression, "table"); + let table_name: String = payload_name(table.as_ref()); + return vec![(expression.get_name().to_owned(), table_name)]; + } + if expression_kind == KIND_SUBQUERY + || expression_kind == KIND_SELECT + || is_set_operation(expression_kind) + { + return Vec::new(); + } + children(expression) + .into_iter() + .flat_map(direct_column_refs) + .collect() +} + +/// `_direct_nested_queries`: each query nested in the select outside its tables, and its alias. +fn direct_nested_queries(select: &Expression) -> Vec<(&Expression, &str)> { + children(select) + .into_iter() + .filter(|child| kind(child) != KIND_TABLE) + .flat_map(nested_queries) + .collect() +} + +fn nested_queries(expression: &Expression) -> Vec<(&Expression, &str)> { + let expression_kind: &str = kind(expression); + if expression_kind == KIND_SUBQUERY { + return expression + .get_this() + .map(|body| vec![(body, alias_or_name(expression))]) + .unwrap_or_default(); + } + if expression_kind == KIND_SELECT || is_set_operation(expression_kind) { + return vec![(expression, "")]; + } + children(expression) + .into_iter() + .flat_map(nested_queries) + .collect() +} + +/// `_star_projection_relations`: the star's qualified relation, else every local relation. +fn star_relations(projection: &Expression, local: &RelationMap) -> Vec> { + let table: Option = arg(projection, "table"); + let table_name: String = payload_name(table.as_ref()); + if table_name.is_empty() { + return local.unique(); + } + local + .get(&python_casefold(&table_name)) + .map(|relation| vec![Rc::clone(relation)]) + .unwrap_or_default() +} + +/// `_apply_cte_column_aliases`: a CTE's slots renamed by its column alias list. +fn cte_column_aliases(query: &Expression, cte_name: &str, reads: &RelationReads) -> RelationReads { + let aliases: Vec = cte_alias_names(query, cte_name); + if aliases.len() != reads.slots.len() { + return RelationReads::default(); + } + RelationReads { + slots: aliases + .into_iter() + .zip(&reads.slots) + .map(|(alias, (_, leaves))| (alias, leaves.clone())) + .collect(), + passthrough: Vec::new(), + } +} + +/// The column alias list of the first CTE payload named `cte_name`. +fn cte_alias_names(query: &Expression, cte_name: &str) -> Vec { + let Some(Value::Object(with)) = arg(query, "with") else { + return Vec::new(); + }; + let Some(Value::Array(ctes)) = with.get("ctes") else { + return Vec::new(); + }; + for cte in ctes.iter().filter_map(Value::as_object) { + let Some(Value::Object(alias)) = cte.get("alias") else { + continue; + }; + if alias.get("name").and_then(Value::as_str) != Some(cte_name) { + continue; + } + let Some(Value::Array(columns)) = cte.get("columns") else { + return Vec::new(); + }; + return columns + .iter() + .filter_map(|column| column.get("name").and_then(Value::as_str)) + .map(str::to_owned) + .collect(); + } + Vec::new() +} diff --git a/crates/sqlbuild-analysis/src/required_columns/_helpers/relations.rs b/crates/sqlbuild-analysis/src/required_columns/_helpers/relations.rs new file mode 100644 index 0000000000..e3a5825aad --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/_helpers/relations.rs @@ -0,0 +1,117 @@ +//! The external columns each local relation's output columns stand for. + +use std::rc::Rc; + +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::required_columns::models::ColumnRead; + +/// `_RelationReads`: named output slots, and resources whose every column passes through. +#[derive(Clone, Debug, Default)] +pub(crate) struct RelationReads { + pub(crate) slots: Vec<(String, Vec)>, + pub(crate) passthrough: Vec<(String, String)>, +} + +impl RelationReads { + /// Whether the relation has `column_name`, and the reads it stands for. + pub(crate) fn resolve_with_presence(&self, column_name: &str) -> (bool, Vec) { + let normalized: String = python_casefold(column_name); + if let Some((_, leaves)) = self + .slots + .iter() + .rev() + .find(|(name, _)| python_casefold(name) == normalized) + { + return (true, leaves.clone()); + } + let leaves: Vec = self + .passthrough + .iter() + .map(|(resource_type, resource_name)| ColumnRead { + resource_type: resource_type.clone(), + resource_name: resource_name.clone(), + column_name: column_name.to_owned(), + }) + .collect(); + (!self.passthrough.is_empty(), leaves) + } + + pub(crate) fn resolve(&self, column_name: &str) -> Vec { + self.resolve_with_presence(column_name).1 + } +} + +/// A Python dict of relations by casefolded name: insertion-ordered, replaced in place. +#[derive(Clone, Debug, Default)] +pub(crate) struct RelationMap { + entries: Vec<(String, Rc)>, +} + +impl RelationMap { + pub(crate) fn get(&self, name: &str) -> Option<&Rc> { + self.entries + .iter() + .find(|(key, _)| key == name) + .map(|(_, relation)| relation) + } + + pub(crate) fn insert(&mut self, name: String, relation: Rc) { + match self.entries.iter_mut().find(|(key, _)| *key == name) { + Some(entry) => entry.1 = relation, + None => self.entries.push((name, relation)), + } + } + + /// `{**self, **other}`. + pub(crate) fn merged(&self, other: &Self) -> Self { + let mut merged: Self = self.clone(); + for (name, relation) in &other.entries { + merged.insert(name.clone(), Rc::clone(relation)); + } + merged + } + + /// `_unique_relations`: each distinct relation once, in first-seen order. + pub(crate) fn unique(&self) -> Vec> { + let mut unique: Vec> = Vec::new(); + for (_, relation) in &self.entries { + if !unique.iter().any(|seen| Rc::ptr_eq(seen, relation)) { + unique.push(Rc::clone(relation)); + } + } + unique + } +} + +/// `relation.get(name) or other.get(name)` with `name` casefolded. +pub(crate) fn either<'a>( + local: &'a RelationMap, + outer: &'a RelationMap, + name: &str, +) -> Option<&'a Rc> { + let key: String = python_casefold(name); + local.get(&key).or_else(|| outer.get(&key)) +} + +/// `_deduplicated_sources`. +pub(crate) fn unique_reads(reads: Vec) -> Vec { + let mut unique: Vec = Vec::with_capacity(reads.len()); + for read in reads { + if !unique.contains(&read) { + unique.push(read); + } + } + unique +} + +/// `_deduplicated_resources`. +pub(crate) fn unique_resources(resources: Vec<(String, String)>) -> Vec<(String, String)> { + let mut unique: Vec<(String, String)> = Vec::with_capacity(resources.len()); + for resource in resources { + if !unique.contains(&resource) { + unique.push(resource); + } + } + unique +} diff --git a/crates/sqlbuild-analysis/src/required_columns/_helpers/tree.rs b/crates/sqlbuild-analysis/src/required_columns/_helpers/tree.rs new file mode 100644 index 0000000000..a308c01a0d --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/_helpers/tree.rs @@ -0,0 +1,91 @@ +//! The accessors Python's walk read from the Polyglot wheel, over the crate's own tree. + +use polyglot_sql::{Expression, ExpressionWalk}; +use serde_json::{Map, Value}; + +use crate::required_columns::constants::{KIND_ANNOTATED, STAR}; + +/// The wheel's `kind`. +pub(crate) fn kind(expression: &Expression) -> &'static str { + expression.variant_name() +} + +/// The wheel's `children()`. +pub(crate) fn children(expression: &Expression) -> Vec<&Expression> { + expression.children() +} + +/// The wheel's `alias_or_name`. +pub(crate) fn alias_or_name(expression: &Expression) -> &str { + let alias: &str = expression.get_alias(); + if alias.is_empty() { + expression.get_name() + } else { + alias + } +} + +/// The wheel's `args`: the serialized payload of the expression's variant. +pub(crate) fn payload(expression: &Expression) -> Map { + match serde_json::to_value(expression) { + Ok(Value::Object(map)) => match map.into_iter().next() { + Some((_, Value::Object(payload))) => payload, + _ => Map::new(), + }, + _ => Map::new(), + } +} + +/// The wheel's `is_star`. +pub(crate) fn is_star(expression: &Expression) -> bool { + match expression { + Expression::Star(_) => true, + Expression::Column(column) => column.name.name == STAR, + _ => false, + } +} + +/// `_unwrap_polyglot_annotations`. +pub(crate) fn unwrap_annotations(expression: &Expression) -> &Expression { + let mut current: &Expression = expression; + while kind(current) == KIND_ANNOTATED { + let Some(inner) = current.get_this() else { + break; + }; + current = inner; + } + current +} + +/// The wheel's `with_ctes()`: each top-level CTE's name, body and whether it aliases columns. +pub(crate) fn top_level_ctes(root: &Expression) -> Vec<(&str, &Expression, bool)> { + let with = match root { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(operation) => operation.with.as_ref(), + Expression::Intersect(operation) => operation.with.as_ref(), + Expression::Except(operation) => operation.with.as_ref(), + _ => None, + }; + with.map_or_else(Vec::new, |with| { + with.ctes + .iter() + .map(|cte| (cte.alias.name.as_str(), &cte.this, !cte.columns.is_empty())) + .collect() + }) +} + +/// A set operation's `(left, right, by_name)`. +pub(crate) fn set_operation(expression: &Expression) -> Option<(&Expression, &Expression, bool)> { + match expression { + Expression::Union(operation) => { + Some((&operation.left, &operation.right, operation.by_name)) + } + Expression::Intersect(operation) => { + Some((&operation.left, &operation.right, operation.by_name)) + } + Expression::Except(operation) => { + Some((&operation.left, &operation.right, operation.by_name)) + } + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/required_columns/constants.rs b/crates/sqlbuild-analysis/src/required_columns/constants.rs new file mode 100644 index 0000000000..92f19c845f --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/constants.rs @@ -0,0 +1,34 @@ +//! Polyglot expression kinds and clause names the column-read walk recognises. + +pub(crate) const KIND_ANNOTATED: &str = "annotated"; +pub(crate) const KIND_COLUMN: &str = "column"; +pub(crate) const KIND_SELECT: &str = "select"; +pub(crate) const KIND_SUBQUERY: &str = "subquery"; +pub(crate) const KIND_TABLE: &str = "table"; +/// The set-operation kinds, whose branches combine column by column. +pub(crate) const SET_OPERATION_KINDS: [&str; 3] = ["union", "intersect", "except"]; +/// Select clauses whose column references count as reads, in Python's order. +pub(crate) const CLAUSE_NAMES: [&str; 12] = [ + "cluster_by", + "connect", + "distinct_on", + "distribute_by", + "group_by", + "having", + "joins", + "order_by", + "qualify", + "sort_by", + "where_clause", + "windows", +]; +/// Clauses where an output alias wins over an input column of the same name. +pub(crate) const OUTPUT_ALIAS_FIRST_CLAUSES: [&str; 2] = ["order_by", "qualify"]; +/// Clauses where an output alias is used only when no input column matches. +pub(crate) const OUTPUT_ALIAS_FALLBACK_CLAUSES: [&str; 4] = + ["cluster_by", "group_by", "having", "sort_by"]; +/// The prefix of the relation stub a table-function call is analysed as. +pub(crate) const TABLE_FUNCTION_STUB_PREFIX: &str = "__sqlbuild_table_function_"; +/// The column name a star projection parses as. +pub(crate) const STAR: &str = "*"; +pub(crate) const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-analysis/src/required_columns/errors.rs b/crates/sqlbuild-analysis/src/required_columns/errors.rs new file mode 100644 index 0000000000..c34d39b633 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/errors.rs @@ -0,0 +1,10 @@ +//! Why a model's column reads could not be resolved at all. + +/// A failure Python's resolution raised rather than treating as unparseable SQL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RequiredColumnsError { + /// Polyglot does not know the dialect; Python's `ValueError`. + UnknownDialect(String), + /// Analysis normalization rejected the SQL; Python's `ValueError`. + Normalization(String), +} diff --git a/crates/sqlbuild-analysis/src/required_columns/main/mod.rs b/crates/sqlbuild-analysis/src/required_columns/main/mod.rs new file mode 100644 index 0000000000..dc5d4d4108 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/main/mod.rs @@ -0,0 +1 @@ +pub mod resolve_required_columns; diff --git a/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs new file mode 100644 index 0000000000..f09b0f5f66 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs @@ -0,0 +1,66 @@ +use std::collections::HashMap; +use std::rc::Rc; + +use polyglot_sql::{Dialect, Expression, ParseOptions}; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::required_columns::_helpers::queries::query_reads; +use crate::required_columns::_helpers::relations::{RelationMap, RelationReads, unique_reads}; +use crate::required_columns::constants::{GENERIC_DIALECT, TABLE_FUNCTION_STUB_PREFIX}; +use crate::required_columns::errors::RequiredColumnsError; +use crate::required_columns::models::{ColumnRead, ReferenceInput}; +use crate::semantic_validation::main::normalize::normalize_analysis_sql; +use crate::semantic_validation::models::NormalizationInput; + +/// The external columns a query reads through its CTEs and subqueries; unparseable SQL reads none. +pub fn resolve_required_columns( + query_sql: &str, + references: &[ReferenceInput], + dialect: Option<&str>, +) -> Result, RequiredColumnsError> { + let dialect_name: &str = dialect.unwrap_or(GENERIC_DIALECT); + let sql: String = normalize_analysis_sql(NormalizationInput { + sql: query_sql.to_owned(), + dialect: dialect_name.to_owned(), + stubs: HashMap::new(), + placeholders: HashMap::new(), + }) + .map_err(RequiredColumnsError::Normalization)?; + let parser: Dialect = Dialect::get_by_name(dialect_name) + .ok_or_else(|| RequiredColumnsError::UnknownDialect(dialect_name.to_owned()))?; + let Ok(mut statements) = parser.parse_with_options(&sql, &ParseOptions::default()) else { + return Ok(Vec::new()); + }; + if statements.len() != 1 { + return Ok(Vec::new()); + } + let parsed: Expression = statements.remove(0); + let mut relations: RelationMap = RelationMap::default(); + for reference in references { + let Some((resource_type, analysis_name)) = lineage_resource(reference) else { + continue; + }; + let relation: RelationReads = RelationReads { + slots: Vec::new(), + passthrough: vec![(resource_type.to_owned(), reference.name.clone())], + }; + relations.insert(python_casefold(&analysis_name), Rc::new(relation)); + } + let (_, required) = query_reads(&parsed, &relations, &RelationMap::default()); + Ok(unique_reads(required)) +} + +/// A reference's lineage resource type and the relation name its query analyses as. +fn lineage_resource(reference: &ReferenceInput) -> Option<(&'static str, String)> { + let resource_type: &'static str = match reference.kind.as_str() { + "ref" => "model", + "source" => "source", + "seed" => "seed", + "table_fn" => { + let name: String = format!("{TABLE_FUNCTION_STUB_PREFIX}{}", reference.name); + return Some(("table_fn", name)); + } + _ => return None, + }; + Some((resource_type, reference.name.clone())) +} diff --git a/crates/sqlbuild-analysis/src/required_columns/mod.rs b/crates/sqlbuild-analysis/src/required_columns/mod.rs new file mode 100644 index 0000000000..d38cea8561 --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-analysis/src/required_columns/models.rs b/crates/sqlbuild-analysis/src/required_columns/models.rs new file mode 100644 index 0000000000..2f05ef5d5e --- /dev/null +++ b/crates/sqlbuild-analysis/src/required_columns/models.rs @@ -0,0 +1,17 @@ +//! A model's SQL references and the external columns its query reads. + +/// One SQL reference of a model: its reference kind value and referenced name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReferenceInput { + /// `ref`, `source`, `seed`, `table_fn`, or another kind that has no lineage. + pub kind: String, + pub name: String, +} + +/// One external column a query reads: the resource type value, resource and column. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ColumnRead { + pub resource_type: String, + pub resource_name: String, + pub column_name: String, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 20e74c10ef..62d4a083e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod normalization_results; mod positions; pub(crate) mod queries; pub(crate) mod relation_names; +mod required_columns; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index a8bcdaeb43..b398ef704f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -105,6 +105,7 @@ fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> P pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::analysis::relation_names::register(module)?; + crate::bindings::_helpers::analysis::required_columns::register(module)?; crate::bindings::_helpers::analysis::explicit_references::register(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs new file mode 100644 index 0000000000..0deadf4ef8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs @@ -0,0 +1,43 @@ +//! The external columns a model's query reads. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::required_columns::errors::RequiredColumnsError; +use sqlbuild_analysis::required_columns::main::resolve_required_columns::resolve_required_columns as resolve; +use sqlbuild_analysis::required_columns::models::{ColumnRead, ReferenceInput}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +/// Each `(resource type, resource, column)` the query reads, given its `(kind, name)` references. +#[pyfunction] +#[pyo3(signature = (query_sql, references, dialect))] +fn resolve_required_columns( + py: Python<'_>, + query_sql: &str, + references: Vec<(String, String)>, + dialect: Option<&str>, +) -> PyResult> { + let references: Vec = references + .into_iter() + .map(|(kind, name)| ReferenceInput { kind, name }) + .collect(); + let reads: Vec = py + .compiler_detach(|| Ok(resolve(query_sql, &references, dialect))) + .map_err(value_error)? + .map_err(|error| match error { + RequiredColumnsError::UnknownDialect(name) => { + PyValueError::new_err(format!("Unknown dialect: {name}")) + } + RequiredColumnsError::Normalization(message) => value_error(message), + })?; + Ok(reads + .into_iter() + .map(|read| (read.resource_type, read.resource_name, read.column_name)) + .collect()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_required_columns, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 49237456cb..a208ee6dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_hooks::{ ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, }; use crate::bindings::_helpers::compiled_project::model_loop_host::{ - FinishedModel, InterpolationRow, ModelLoopInput, ModelLoopRow, ModelPaths, RecordClasses, + DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, ModelPaths, RecordClasses, RecordInputs, ReferenceRow, finished_record, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ @@ -62,7 +62,11 @@ use crate::bindings::_helpers::render::declaration_expansion::{ expand_declarations, }; use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::bindings::_helpers::sqltext::static_sql::{ + InterpolationRow as SqlInterpolationRow, InterpolationSources, interpolation_batch, +}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -70,11 +74,10 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{FromPyObject, pyclass, pymethods}; use sqlbuild_analysis::compiled_project::main::analysis_gate::analysis_gate; use sqlbuild_analysis::compiled_project::main::calls_macros::calls_macros; -use sqlbuild_analysis::compiled_project::main::legacy_schema_error::legacy_schema_error; use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; use sqlbuild_analysis::compiled_project::models::{ - AnalysisGate, LegacySchemaFile, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, + AnalysisGate, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, }; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; @@ -102,14 +105,15 @@ const VALUES_ARGUMENT: &str = "values"; const CODE_ARGUMENT: &str = "code"; const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; const ANALYSIS_CONFLICT_CODE: &str = "P003"; +const SQL_HOOKS_KEY: &str = "sql_hooks"; +const HOOK_FILES_KEY: &str = "hook_files"; +const DEFINITIONS_KEY: &str = "definitions"; +const COMPILE_ERROR_KEY: &str = "compile_error"; /// The native validator, header contract classes, and the loop's Python classes. type LoopChecks = (Py, Py, Py); -/// `(analysis switches, native syntax, (project config path, legacy schema files))`. -type LoopSettingsInput = ( - (bool, bool, bool, bool), - (String, Vec<(String, Vec)>), -); +/// `(analysis switches, project config path)`. +type LoopSettingsInput = ((bool, bool, bool, bool), String); /// The Python classes and error factory the loop builds usages, hooks and errors with. #[derive(FromPyObject)] @@ -173,7 +177,6 @@ pub(crate) struct NativeModelLoop { pub(crate) reserved_markers: Vec, pub(crate) settings: ModelLoopSettings, pub(crate) project_config_path: String, - legacy_schema_files: Vec, contract_classes: Py, loop_classes: Py, } @@ -212,8 +215,9 @@ impl NativeModelLoop { let (validator, contract_classes, loop_classes) = checks; let ( (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis), - (project_config_path, legacy_schema_files), + project_config_path, ) = settings; + Python::attach(|py| index_sql_hook_definitions(loop_classes.bind(py)))?; Ok(Self { validator, syntax: syntax.into(), @@ -225,13 +229,6 @@ impl NativeModelLoop { require_sql_analysis, }, project_config_path, - legacy_schema_files: legacy_schema_files - .into_iter() - .map(|(relative_path, model_names)| LegacySchemaFile { - relative_path, - model_names, - }) - .collect(), contract_classes, loop_classes, }) @@ -242,13 +239,11 @@ impl NativeModelLoop { slf: &Bound<'py, Self>, project: Bound<'py, NativeCompiledProject>, host: Bound<'py, PyAny>, - models: Vec, + model_files: Vec>, ) -> PyResult>> { let py: Python<'py> = slf.py(); - let models: Vec = models - .into_iter() - .map(|input| ModelLoopRow::new(py, input)) - .collect::>()?; + let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; + let models: Vec = prepared_rows(py, &classes, model_files)?; project .borrow_mut() .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); @@ -256,6 +251,77 @@ impl NativeModelLoop { } } +/// Replace the SQL hook classes' discovered `hook_files` with their `definitions` by name. +fn index_sql_hook_definitions(loop_classes: &Bound<'_, PyDict>) -> PyResult<()> { + let py: Python<'_> = loop_classes.py(); + let hooks: Bound<'_, PyDict> = loop_classes + .as_any() + .get_item(SQL_HOOKS_KEY)? + .downcast_into()?; + let definitions: Bound<'_, PyDict> = PyDict::new(py); + for hook_file in hooks.as_any().get_item(HOOK_FILES_KEY)?.try_iter()? { + let hook_file: Bound<'_, PyAny> = hook_file?; + let name: Bound<'_, PyAny> = hook_file.getattr("name")?; + if let Some(existing) = definitions.get_item(&name)? { + let message: String = format!( + "Duplicate SQL hook name '{}' found in {} and {}", + name.str()?, + existing.getattr("relative_path")?.str()?, + hook_file.getattr("relative_path")?.str()? + ); + let error: Bound<'_, PyAny> = loop_classes + .as_any() + .get_item(COMPILE_ERROR_KEY)? + .call1((message,))?; + return Err(PyErr::from_value(error)); + } + definitions.set_item(name, hook_file)?; + } + hooks.set_item(DEFINITIONS_KEY, definitions) +} + +/// Each model's `@@`-interpolated query and declaration scan, all interpolated before any runs. +fn prepared_rows<'py>( + py: Python<'py>, + classes: &LoopClasses<'py>, + model_files: Vec>, +) -> PyResult> { + let hooks = &classes.sql_hooks; + let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); + for model_file in &model_files { + let record: &NativeModelFile = model_file.get(); + let (file_path, _) = record.paths(py); + let (query_sql, _) = record.texts(py); + sqls.push(( + query_sql.to_str()?.to_owned(), + file_path.str()?.to_str()?.to_owned(), + )); + } + let interpolated: Vec = interpolation_batch( + InterpolationSources( + hooks.effective_vars.clone(), + hooks.environment.clone(), + None, + hooks.render_project_var.clone(), + ), + &sqls, + )?; + let mut rows: Vec = Vec::with_capacity(model_files.len()); + for ((model_file, (authored, _)), (rendered, spans, reads, error)) in + model_files.into_iter().zip(sqls).zip(interpolated) + { + let sql: String = rendered.unwrap_or(authored); + let scan: Option = error.is_none().then(|| declaration_scan(&sql)); + let reads: Vec<(String, String)> = reads + .into_iter() + .map(|(kind, name)| (kind.to_owned(), name)) + .collect(); + let interpolation: InterpolationRow = (sql, reads, error, spans); + rows.push(ModelLoopRow::new(py, (model_file, scan, interpolation))?); + } + Ok(rows) +} + impl NativeModelLoop { fn run_models<'py>( &self, @@ -645,10 +711,6 @@ impl NativeModelLoop { schema: schema.as_ref(), }; let header: AttachedHeader<'py> = header_schema(py, context, &source)?; - if let Some(message) = legacy_schema_error((name, relative_path), &self.legacy_schema_files) - { - return Err(compile_error(context.classes, &message, None)); - } let mut named_usages: Vec> = hook_usages; named_usages.extend(schema_usages); named_usages.extend(python_hooks); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 39757e7695..c540e99aa9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -73,7 +73,7 @@ pub(crate) fn project_facts( } /// `read` applied to each item of the iterable attribute `name` of `owner`. -fn each( +pub(crate) fn each( owner: &Bound<'_, PyAny>, name: &str, read: impl Fn(&Bound<'_, PyAny>) -> PyResult, @@ -94,7 +94,7 @@ fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult, name: &str) -> PyResult { +pub(crate) fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { owner.getattr(name)?.getattr("stem")?.extract() } @@ -142,7 +142,7 @@ fn seed_file_fact(seed: &Bound<'_, PyAny>) -> PyResult { }) } -fn column_name(column: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn column_name(column: &Bound<'_, PyAny>) -> PyResult { column.getattr("name")?.extract() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs new file mode 100644 index 0000000000..ee11b901b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -0,0 +1,76 @@ +//! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; +use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; + +use crate::bindings::_helpers::model_config::model_config_builder::{ + layered_values, raised_config_error, +}; +use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; + +const DESCRIPTION_KEY: &str = "description"; +const MODEL_SCHEMA_KEY: &str = "model_schema"; + +/// Whether `value` is a string with non-whitespace text. +fn non_empty_text(value: Option>) -> PyResult { + let Some(value) = value.filter(|value| value.is_instance_of::()) else { + return Ok(false); + }; + value.call_method0("strip")?.is_truthy() +} + +/// The file path of each model whose layered config or model schema gives it a description. +#[pyfunction] +fn externally_described_models<'py>( + inputs: Bound<'py, PyAny>, + model_schemas: Bound<'py, PyDict>, +) -> PyResult>> { + let py = inputs.py(); + let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let defaults: Bound<'py, PyDict> = + project_defaults_mapping(&project_config.getattr("defaults")?)?; + let path_defaults: Bound<'py, PyDict> = + project_config.getattr("path_defaults")?.downcast_into()?; + let path_keys: Vec = path_defaults + .keys() + .iter() + .map(|key| key.extract::()) + .collect::>()?; + let mut described: Vec> = Vec::new(); + for model_file in inputs.getattr("model_files")?.try_iter()? { + let model_file: Bound<'py, PyAny> = model_file?; + let relative_path: String = model_file.getattr("relative_path")?.str()?.extract()?; + let mut values: Bound<'py, PyDict> = defaults.copy()?; + match select_path_default(&relative_path, &path_keys) { + PathDefaultChoice::Selected(Some(key)) => { + let layer: Bound<'py, PyDict> = + path_defaults.as_any().get_item(&key)?.downcast_into()?; + values = layered_values(py, &values, &layer)?; + } + PathDefaultChoice::Selected(None) => {} + PathDefaultChoice::Conflict(error) => return Err(raised_config_error(py, error)), + } + let header: Bound<'py, PyDict> = model_file.getattr("header_values")?.downcast_into()?; + values = layered_values(py, &values, &header)?; + let schema_description: Option> = + match values.get_item(MODEL_SCHEMA_KEY)? { + Some(name) if name.is_instance_of::() => model_schemas + .get_item(name)? + .map(|schema| schema.getattr(DESCRIPTION_KEY)) + .transpose()?, + _ => None, + }; + if non_empty_text(values.get_item(DESCRIPTION_KEY)?)? || non_empty_text(schema_description)? + { + described.push(model_file.getattr("file_path")?); + } + } + Ok(described) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(externally_described_models, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index cf2db8c188..ef4f1e6b7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -3,8 +3,10 @@ pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; pub(crate) mod deferred_targets; +pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod header_metadata; pub(crate) mod model_config_builder; pub(crate) mod model_validator; +pub(crate) mod project_defaults; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 4782fb01d2..4e219ebeab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; +use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; /// Hook keys in the sorted order `validate_model_hook_config` checks them. @@ -35,6 +36,8 @@ const MODEL_CONFIG_LABEL: &str = "model config"; const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; const TAGS_KEY: &str = "tags"; +const NATIVE_CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; +const NATIVE_CONFIG_ERROR_FUNCTION: &str = "native_config_error"; const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; @@ -125,12 +128,13 @@ type TargetInputs<'py> = ( impl NativeModelConfigBuilder { #[new] fn new( - layers: (Bound<'_, PyDict>, Bound<'_, PyDict>, Bound<'_, PyTuple>), + layers: (Bound<'_, PyAny>, Bound<'_, PyDict>, Bound<'_, PyTuple>), sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), target: TargetInputs<'_>, ) -> PyResult { let (run, target_namespace) = target; let (defaults, path_defaults, hook_entry_types) = layers; + let defaults: Bound<'_, PyDict> = project_defaults_mapping(&defaults)?; let path_keys = path_defaults .keys() .iter() @@ -570,6 +574,33 @@ impl<'py> Expansion<'_, 'py> { } } +/// `merged_layer`, with a config error raised as the Python stage raises it. +pub(crate) fn layered_values<'py>( + py: Python<'py>, + base: &Bound<'py, PyDict>, + overlay: &Bound<'py, PyDict>, +) -> PyResult> { + merged_layer(py, base, overlay).map_err(|halt| match halt { + Halt::Error(error) => error, + Halt::Config(error) => raised_config_error(py, error), + }) +} + +/// The Python exception `native_config_error` makes of a native config error. +pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { + let raised = || -> PyResult { + let error: Py = native_config_error(py, error)?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("error", error)?; + let exception: Bound<'_, PyAny> = py + .import(NATIVE_CONFIG_ERROR_MODULE)? + .getattr(NATIVE_CONFIG_ERROR_FUNCTION)? + .call((), Some(&arguments))?; + Ok(PyErr::from_value(exception)) + }; + raised().unwrap_or_else(|error| error) +} + /// Merge `overlay` over a copy of `base` as `_merged_with_tag_union` does. fn merged_layer<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index b00a49fe69..fab7173244 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyclass, pymethods}; use sqlbuild_attachments::functions::models::KnownResources; @@ -13,9 +13,15 @@ use sqlbuild_model_config::model_validation::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::discovery::project_facts::{column_name, each, stem}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const RETURNS_KEY: &str = "returns"; +/// The only key of a table function's `returns` mapping. +const TABLE_RETURNS_KEY: &str = "table"; + /// Model, seed, source, function and table function names. type ResourceNames = ( HashSet, @@ -47,8 +53,65 @@ impl NativeModelValidator { } } +/// Model, seed, source, function and table function names discovered in `inputs`. +fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { + let model_stem = |file: &Bound<'_, PyAny>| stem(file, FILE_PATH_ATTRIBUTE); + let models: HashSet = each(inputs, "model_files", model_stem)? + .into_iter() + .collect(); + let mut seeds: HashSet = HashSet::new(); + for schema in inputs.getattr("schema_files")?.try_iter()? { + seeds.extend(each(&schema?, "seed_entries", column_name)?); + } + let mut sources: HashSet = HashSet::new(); + for file in inputs.getattr("source_files")?.try_iter()? { + sources.extend(each(&file?, "source_entries", column_name)?); + } + let mut functions: HashSet = HashSet::new(); + let mut table_functions: HashSet = HashSet::new(); + for file in inputs.getattr("sql_function_files")?.try_iter()? { + let file: Bound<'_, PyAny> = file?; + let name: String = model_stem(&file)?; + if returns_table(&file)? { + table_functions.insert(name.clone()); + } + functions.insert(name); + } + functions.extend(each(inputs, "python_function_files", model_stem)?); + Ok((models, seeds, sources, functions, table_functions)) +} + +/// Whether a SQL function's header `returns` is a mapping whose only key is `table`. +fn returns_table(file: &Bound<'_, PyAny>) -> PyResult { + let headers: Bound<'_, PyAny> = file.getattr("header_values")?; + let returns: Bound<'_, PyAny> = headers.call_method1("get", (RETURNS_KEY,))?; + let Ok(returns) = returns.downcast::() else { + return Ok(false); + }; + let keys: Vec> = returns.keys().into_iter().collect(); + Ok(keys.len() == 1 && keys[0].eq(TABLE_RETURNS_KEY)?) +} + #[pymethods] impl NativeModelValidator { + /// The validator for the resources and materializations `inputs` discovered. + #[staticmethod] + fn from_inputs(inputs: Bound<'_, PyAny>, microbatch_concurrency: bool) -> PyResult { + let names: ResourceNames = discovered_names(&inputs)?; + let custom_materializations: HashSet = + each(&inputs, "materialization_files", |file| { + file.getattr("name")?.extract() + })? + .into_iter() + .collect(); + Self::new(names, custom_materializations, microbatch_concurrency) + } + + /// The discovered model names references resolve against. + fn model_names(&self) -> HashSet { + self.project.models.clone() + } + #[new] fn new( names: ResourceNames, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/project_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/project_defaults.rs new file mode 100644 index 0000000000..2d17e95531 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/project_defaults.rs @@ -0,0 +1,69 @@ +//! The sparse mapping of a project's `[defaults]` that every model's config layers over. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; + +/// Each default in `project_defaults_to_mapping`'s order, and whether a falsy value is unset. +const DEFAULT_KEYS: [(&str, bool); 30] = [ + ("materialized", false), + ("database", false), + ("schema", false), + ("contract", false), + ("incremental_strategy", false), + ("incremental_mode", false), + ("microbatch_strategy", false), + ("cursor_watermark_mode", false), + ("merge_exclude_columns", true), + ("full_refresh", false), + ("append_cursor_inclusive", false), + ("cursor_start", false), + ("cursor_end", false), + ("cursor_start_max_ahead", false), + ("cursor_start_max_action", false), + ("cursor_future_max_distance", false), + ("cursor_future_action", false), + ("lookback", false), + ("batch_size", false), + ("batch_concurrency", false), + ("max_microbatches", false), + ("unaccounted_partition_policy", false), + ("replay_on_change", false), + ("row_diff_exclude_columns", true), + ("row_diff_tolerances", true), + ("row_diff_sample_rows", false), + ("row_diff_sample_seed", false), + ("tags", true), + ("pre_hooks", false), + ("post_hooks", false), +]; +const TAGS_KEY: &str = "tags"; + +/// `project_defaults_to_mapping`: each set default of `defaults`, with `tags` as a new list. +pub(crate) fn project_defaults_mapping<'py>( + defaults: &Bound<'py, PyAny>, +) -> PyResult> { + let values: Bound<'py, PyDict> = PyDict::new(defaults.py()); + for (key, falsy_unset) in DEFAULT_KEYS { + let value: Bound<'py, PyAny> = defaults.getattr(key)?; + let unset: bool = if falsy_unset { + !value.is_truthy()? + } else { + value.is_none() + }; + if unset { + continue; + } + if key == TAGS_KEY { + values.set_item( + key, + PyList::new( + defaults.py(), + value.try_iter()?.collect::>>()?, + )?, + )?; + } else { + values.set_item(key, value)?; + } + } + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index b6ea38d056..afe3c15955 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_templates, deferred_targets, model_config_builder, - model_validator, + column_migrations, config_errors, config_templates, deferred_targets, described_models, + model_config_builder, model_validator, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,6 +12,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { config_errors::register(module)?; config_templates::register(module)?; deferred_targets::register(module)?; + described_models::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs new file mode 100644 index 0000000000..62cadc33e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -0,0 +1,217 @@ +//! The public declaration indexes: enums, constants and resolved reusable model schemas. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +/// `CompileInputError(message)`. +fn compile_input_error(py: Python<'_>, message: String) -> PyErr { + match py + .import(COMPILE_ERRORS_MODULE) + .and_then(|module| module.getattr(COMPILE_INPUT_ERROR)) + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(owner.getattr(name)?.str()?.to_string()) +} + +/// Add each declaration of `files` to `index`, rejecting a name declared twice. +fn index_declarations<'py>( + files: &Bound<'py, PyAny>, + kind: &str, + index: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = files.py(); + for file in files.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let name: Bound<'py, PyAny> = declaration.getattr("name")?; + if let Some(existing) = index.get_item(&name)? { + return Err(compile_input_error( + py, + format!( + "Duplicate public {kind} '{}' in {} and {}", + name.str()?, + text(&existing, "relative_path")?, + text(&declaration, "relative_path")? + ), + )); + } + index.set_item(name, declaration)?; + } + } + Ok(()) +} + +/// Every public enum and constant by name, collision-checked. +#[pyfunction] +fn public_declaration_indexes<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = inputs.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + let constants: Bound<'py, PyDict> = PyDict::new(py); + index_declarations(&inputs.getattr("enum_files")?, "enum", &enums)?; + index_declarations(&inputs.getattr("constant_files")?, "constant", &constants)?; + Ok((enums, constants)) +} + +/// Resolving `extends` chains across the authored public model schemas. +struct SchemaResolution<'py> { + py: Python<'py>, + authored: Vec<(String, Bound<'py, PyAny>)>, + by_name: HashMap, + resolved: HashMap>, + /// Names in the order Python's `resolved` dict gains them. + order: Vec, + replace: Bound<'py, PyAny>, +} + +impl<'py> SchemaResolution<'py> { + /// Python's `_resolve_model_schema_declaration`. + fn resolve(&mut self, name: &str, resolving: &[String]) -> PyResult> { + if let Some(existing) = self.resolved.get(name) { + return Ok(existing.clone()); + } + if let Some(start) = resolving.iter().position(|item| item == name) { + let mut cycle: Vec<&str> = resolving[start..].iter().map(String::as_str).collect(); + cycle.push(name); + return Err(compile_input_error( + self.py, + format!("Model schema inheritance cycle: {}", cycle.join(" -> ")), + )); + } + let declaration: Bound<'py, PyAny> = self.authored[self.by_name[name]].1.clone(); + let extends: Option = declaration.getattr("extends")?.extract()?; + let mut inherited: Vec> = Vec::new(); + if let Some(parent) = extends { + if !self.by_name.contains_key(&parent) { + return Err(compile_input_error( + self.py, + format!( + "Schema '{name}' in {} extends unknown schema '{parent}'", + text(&declaration, "relative_path")? + ), + )); + } + let mut chain: Vec = resolving.to_vec(); + chain.push(name.to_owned()); + let parent: Bound<'py, PyAny> = self.resolve(&parent, &chain)?; + for column in parent.getattr("columns")?.try_iter()? { + inherited.push(column?); + } + } + self.reject_overrides(name, &declaration, &inherited)?; + let mut columns: Vec> = inherited; + for column in declaration.getattr("columns")?.try_iter()? { + columns.push(column?); + } + let fields: Bound<'py, PyDict> = PyDict::new(self.py); + fields.set_item("columns", PyTuple::new(self.py, columns)?)?; + let resolved: Bound<'py, PyAny> = self.replace.call((declaration,), Some(&fields))?; + self.resolved.insert(name.to_owned(), resolved.clone()); + self.order.push(name.to_owned()); + Ok(resolved) + } + + /// Python's `_validate_no_inherited_column_overrides`. + fn reject_overrides( + &self, + name: &str, + declaration: &Bound<'py, PyAny>, + inherited: &[Bound<'py, PyAny>], + ) -> PyResult<()> { + let mut by_name: HashMap> = HashMap::new(); + for column in inherited { + by_name.insert(lowered_name(column)?, column); + } + for local in declaration.getattr("columns")?.try_iter()? { + let local: Bound<'py, PyAny> = local?; + let Some(column) = by_name.get(&lowered_name(&local)?) else { + continue; + }; + let location: Bound<'py, PyAny> = column.getattr("location")?; + let origin: String = if location.is_none() { + "an ancestor schema".to_owned() + } else { + format!("{}:{}", text(&location, "path")?, text(&location, "line")?) + }; + return Err(compile_input_error( + self.py, + format!( + "Schema '{name}' in {} redeclares inherited column '{}' from {origin}; \ + column overrides are not supported", + text(declaration, "relative_path")?, + text(&local, "name")? + ), + )); + } + Ok(()) + } +} + +/// `column.name.lower()`. +fn lowered_name(column: &Bound<'_, PyAny>) -> PyResult { + column.getattr("name")?.call_method0("lower")?.extract() +} + +/// Every public model schema by name, with its inherited columns first, collision-checked. +#[pyfunction] +fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + let mut by_name: HashMap = HashMap::new(); + for file in inputs.getattr("model_schema_files")?.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let name: String = declaration.getattr("name")?.extract()?; + if let Some(index) = by_name.get(&name) { + return Err(compile_input_error( + py, + format!( + "Duplicate public schema '{name}' in {} and {}", + text(&authored[*index].1, "relative_path")?, + text(&declaration, "relative_path")? + ), + )); + } + by_name.insert(name.clone(), authored.len()); + authored.push((name, declaration)); + } + } + let names: Vec = authored.iter().map(|(name, _)| name.clone()).collect(); + let mut resolution: SchemaResolution<'py> = SchemaResolution { + py, + authored, + by_name, + resolved: HashMap::new(), + order: Vec::new(), + replace: py.import(DATACLASSES_MODULE)?.getattr("replace")?, + }; + let index: Bound<'py, PyDict> = PyDict::new(py); + for name in &names { + resolution.resolve(name, &[])?; + } + for name in &resolution.order { + index.set_item(name, &resolution.resolved[name])?; + } + Ok(index) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(public_declaration_indexes, module)?)?; + module.add_function(wrap_pyfunction!(public_model_schema_index, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 8b1d78a6c8..1383b1f743 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod declaration_expansion; +pub(crate) mod declaration_indexes; pub(crate) mod macro_arguments; pub(crate) mod macro_call_classes; pub(crate) mod macro_call_memo; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index b57e7ed63e..e627aa1daf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::render::{ - declaration_expansion, macro_arguments, macro_calls, macro_expansion, + declaration_expansion, declaration_indexes, macro_arguments, macro_calls, macro_expansion, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { declaration_expansion::register(module)?; + declaration_indexes::register(module)?; macro_arguments::register(module)?; macro_calls::register(module)?; macro_expansion::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 80534d811b..abb065732b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -14,7 +14,7 @@ const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; /// `(sql or None when unchanged, spans, reads, error message or None)` for one SQL text. -type InterpolationRow = ( +pub(crate) type InterpolationRow = ( Option, Vec<(usize, usize, usize, usize)>, Vec<(&'static str, String)>, @@ -147,13 +147,21 @@ fn key_text(key: &Bound<'_, PyAny>) -> String { fn interpolate_sql_batch<'py>( sqls: Vec<(String, String)>, sources: InterpolationSources<'py>, +) -> PyResult> { + interpolation_batch(sources, &sqls) +} + +/// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. +pub(crate) fn interpolation_batch( + sources: InterpolationSources<'_>, + sqls: &[(String, String)], ) -> PyResult> { let host = PythonInterpolationHost { sources, raised: RefCell::new(None), }; let mut rows = Vec::with_capacity(sqls.len()); - for (sql, file_path) in &sqls { + for (sql, file_path) in sqls { let row = match interpolate_sql(&host, sql, file_path) { Ok(result) => (result.sql, result.spans, read_rows(result.reads), None), Err(failure) => ( diff --git a/python-boundary.toml b/python-boundary.toml index e0fc938819..a823f96233 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 143789 +remaining_lines = 141910 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py index da990d3b90..ae2537eeff 100644 --- a/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py @@ -26,6 +26,7 @@ ) _EXPECTED_STAGING: str = "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)" _TEST_PATH: str = "tests/unit/test_stg_orders.sql" +_SCHEMA_PATH: str = "schemas/orders.sql" def _scenario(*ctes: str, tail: str = "\nSELECT 1\n") -> str: @@ -56,10 +57,44 @@ def _order_label(header: str) -> dict[str, str]: return {_ORDER_LABEL_PATH: f"FUNCTION (\n{header});\n\nUPPER(p_status)\n"} +def _schema(*, name: str, extends: str | None, column: str) -> str: + parent: str = "" if extends is None else f"extends {extends}, " + return f"SCHEMA (name {name}, {parent}columns ({column} (type INTEGER)));\n" + + def attachment_failure_cases() -> tuple[FailureCase, ...]: """Return audit, function, source, SQL test and scenario failures the native stage reports.""" return ( + failure_case( + name="model-schema-extends-unknown", + expected_code="P001", + expected_message=( + "Schema 'order_row' in schemas/orders.sql extends unknown schema 'base_row'" + ), + files={_SCHEMA_PATH: _schema(name="order_row", extends="base_row", column="id")}, + ), + failure_case( + name="model-schema-inheritance-cycle", + expected_code="P001", + expected_message="Model schema inheritance cycle: order_row -> base_row -> order_row", + files={ + _SCHEMA_PATH: _schema(name="order_row", extends="base_row", column="id") + + _schema(name="base_row", extends="order_row", column="amount") + }, + ), + failure_case( + name="model-schema-redeclared-column", + expected_code="P001", + expected_message=( + "Schema 'order_row' in schemas/orders.sql redeclares inherited column 'ID' from " + "schemas/orders.sql:1; column overrides are not supported" + ), + files={ + _SCHEMA_PATH: _schema(name="base_row", extends=None, column="id") + + _schema(name="order_row", extends="base_row", column="ID") + }, + ), failure_case( name="generic-audit-unknown-run-scope", expected_code="P001", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index aa7a9af123..53fabed117 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -54,6 +54,7 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, EnumDeclaration, + ModelSchemaDeclaration, ) from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @@ -67,6 +68,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow from sqlbuild.spec.contracts.models import ( + DefaultsConfig, LocalConfig, ProjectConfig, ScenarioConfig, @@ -732,13 +734,13 @@ class NativeModelLoop: checks: tuple[object, dict[str, object], dict[str, object]], syntax: dict[str, object], reserved_markers: list[str], - settings: tuple[tuple[bool, bool, bool, bool], tuple[str, list[tuple[str, list[str]]]]], + settings: tuple[tuple[bool, bool, bool, bool], str], ) -> None: ... def run( self, project: NativeCompiledProject, host: object, - models: list[tuple[object, ...]], + model_files: list[DiscoveredSqlModelFile], ) -> list[NativeModelInput]: ... def relation_key_row( @@ -879,6 +881,18 @@ def parse_schema_document( def parse_sources_document( loaded: object, path: str, classes: dict[str, object] ) -> tuple[SourceEntry, ...]: ... +def externally_described_models( + inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] +) -> list[Path]: ... +def resolve_required_columns( + query_sql: str, references: list[tuple[str, str]], dialect: str | None +) -> list[tuple[str, str, str]]: ... +def public_declaration_indexes( + inputs: DiscoveredProjectInputs, +) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: ... +def public_model_schema_index( + inputs: DiscoveredProjectInputs, +) -> dict[str, ModelSchemaDeclaration]: ... def discovered_inputs_failure( inputs: DiscoveredProjectInputs, nodes: list[dict[str, object]] ) -> tuple[object, ...] | None: ... @@ -1172,7 +1186,7 @@ def expand_effective_vars( class NativeModelConfigBuilder: def __init__( self, - layers: tuple[dict[str, object], dict[str, dict[str, object]], tuple[type, ...]], + layers: tuple[DefaultsConfig, dict[str, dict[str, object]], tuple[type, ...]], sources: tuple[dict[str, object], object], target: tuple[tuple[str | None, str], tuple[str | None, str | None] | None], ) -> None: ... @@ -1190,6 +1204,11 @@ class NativeModelValidator: custom_materializations: set[str], microbatch_concurrency: bool, ) -> None: ... + @staticmethod + def from_inputs( + inputs: DiscoveredProjectInputs, microbatch_concurrency: bool + ) -> NativeModelValidator: ... + def model_names(self) -> set[str]: ... def validate( self, values: dict[str, object], diff --git a/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py index c87c9f902f..86402dc8de 100644 --- a/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py +++ b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py @@ -4,7 +4,6 @@ from sqlbuild.compiler.planner.types import BackfillAction, OnSchemaChange from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help _ON_SCHEMA_CHANGE_KEY: str = "on_schema_change" @@ -46,12 +45,6 @@ def change_policy_problem_impl(*, key: str, value: object) -> str | None: return f"unknown {key} '{value}'; valid values: {valid}" -def change_policy_header_help_impl(*, key: str) -> str: - """Show a valid MODEL header entry for a change-policy key.""" - - return model_header_help(purpose=f"use a valid {key}", entry=f"{key} {_EXAMPLE_VALUES[key]}") - - def change_policy_toml_help_impl(*, key: str, file_name: str, section: str) -> str: """Show a valid TOML setting line for a change-policy key.""" diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py deleted file mode 100644 index 9d40a3bc96..0000000000 --- a/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public MODEL-header help entrypoint for change-policy values.""" - -from __future__ import annotations - -from sqlbuild.compiler.authored_values._helpers.change_policy import ( - change_policy_header_help_impl, -) - - -def change_policy_header_help(*, key: str) -> str: - """Show a valid MODEL header entry for on_schema_change or replay_on_change.""" - - return change_policy_header_help_impl(key=key) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cte_scopes.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cte_scopes.py deleted file mode 100644 index 40f141cb09..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cte_scopes.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Top-level CTEs and direct tables of Polyglot wheel trees, for resolved column reads.""" - -from __future__ import annotations - -from typing import Any, cast - -from sqlbuild.compiler.sql_analysis.constants import ( - POLYGLOT_KIND_ANNOTATED as _POLYGLOT_KIND_ANNOTATED, -) -from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_KIND_TABLE as _POLYGLOT_KIND_TABLE -from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_PAYLOAD_NAME as _POLYGLOT_PAYLOAD_NAME - - -def _polyglot_top_level_ctes(root: Any) -> tuple[tuple[str, Any, bool], ...]: - targeted_ctes: object = getattr(root, "with_ctes", None) - if callable(targeted_ctes): - values: object = targeted_ctes() - if isinstance(values, list): - return tuple( - (str(name), body, bool(has_column_alias)) for name, has_column_alias, body in values - ) - with_payload: object = root.arg("with") - raw_ctes: object = ( - cast(dict[str, object], with_payload).get("ctes") - if isinstance(with_payload, dict) - else None - ) - if not isinstance(raw_ctes, list): - return () - children: tuple[Any, ...] = tuple(root.children()) - if len(children) < len(raw_ctes): - return () - cte_bodies: tuple[Any, ...] = children[-len(raw_ctes) :] - ctes: list[tuple[str, Any, bool]] = [] - for raw_cte, body in zip(raw_ctes, cte_bodies, strict=True): - if not isinstance(raw_cte, dict): - return () - cte_payload: dict[str, object] = cast(dict[str, object], raw_cte) - cte_name: str = _polyglot_name_payload_value(cte_payload.get("alias")) - if not cte_name: - return () - columns: object = cte_payload.get("columns") - ctes.append((cte_name, body, isinstance(columns, list) and bool(columns))) - return tuple(ctes) - - -def _polyglot_direct_select_tables(select: Any) -> tuple[Any, ...]: - return tuple( - child - for child in select.children() - if str(getattr(child, "kind", "")) == _POLYGLOT_KIND_TABLE - ) - - -def _unwrap_polyglot_annotations(expression: Any) -> Any: - while str(getattr(expression, "kind", "")) == _POLYGLOT_KIND_ANNOTATED: - expression = expression.this - return expression - - -def _polyglot_name_payload_value(payload: object) -> str: - if isinstance(payload, str): - return payload - if not isinstance(payload, dict): - return "" - name: object = cast(dict[str, object], payload).get(_POLYGLOT_PAYLOAD_NAME) - return name if isinstance(name, str) else "" diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py b/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py index 55f5f5ee44..7f929085c3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py @@ -3,30 +3,6 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql - - -def _lineage_resource_type(reference: CompileSqlReference) -> CompiledResourceType | None: - if reference.ref_kind == SqlReferenceKind.REF: - return CompiledResourceType.MODEL - if reference.ref_kind == SqlReferenceKind.SOURCE: - return CompiledResourceType.SOURCE - if reference.ref_kind == SqlReferenceKind.SEED: - return CompiledResourceType.SEED - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION: - return CompiledResourceType.TABLE_FN - return None - - -def _analysis_reference_name(reference: CompileSqlReference) -> str: - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION: - return table_function_analysis_name(reference.ref_name) - return reference.ref_name def table_function_analysis_name(function_name: str) -> str: @@ -41,14 +17,3 @@ def substitute_placeholder_defaults(*, query_sql: str, placeholders: dict[str, s if not placeholders: return query_sql return _native.substitute_placeholder_defaults(query_sql, list(placeholders.items())) - - -def _replace_refs_with_stubs( - *, - query_sql: str, - dialect: str | None = None, - relation_stubs: dict[str, str] | None = None, -) -> str: - """Replace SQLBuild marker calls with parseable SQL stubs.""" - - return normalize_analysis_sql(sql=query_sql, dialect=dialect, stubs=relation_stubs) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/required_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/required_columns.py deleted file mode 100644 index d137543a4b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/required_columns.py +++ /dev/null @@ -1,712 +0,0 @@ -"""Conservative external column reads resolved through local query relations.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, cast - -from sqlbuild.compiler.compile._helpers.analysis.cte_scopes import ( - _polyglot_direct_select_tables, - _polyglot_top_level_ctes, - _unwrap_polyglot_annotations, -) -from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( - _analysis_reference_name, - _lineage_resource_type, - _replace_refs_with_stubs, -) -from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.sql_analysis.constants import ( - POLYGLOT_KIND_COLUMN, - POLYGLOT_KIND_SELECT, - POLYGLOT_KIND_TABLE, - POLYGLOT_SET_OPERATION_KINDS, -) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql - -_POLYGLOT_KIND_SUBQUERY: str = "subquery" - - -@dataclass(frozen=True) -class _RelationReads: - """External leaves represented by one local relation's output columns.""" - - slots: tuple[tuple[str, tuple[CompiledLineageSourceFact, ...]], ...] = () - passthrough_resources: tuple[tuple[CompiledResourceType, str], ...] = () - - def resolve(self, column_name: str) -> tuple[CompiledLineageSourceFact, ...]: - _, leaves = self.resolve_with_presence(column_name) - return leaves - - def resolve_with_presence( - self, column_name: str - ) -> tuple[bool, tuple[CompiledLineageSourceFact, ...]]: - normalized_name: str = column_name.casefold() - matching: tuple[CompiledLineageSourceFact, ...] | None = next( - (leaves for name, leaves in reversed(self.slots) if name.casefold() == normalized_name), - None, - ) - if matching is not None: - return True, matching - leaves: tuple[CompiledLineageSourceFact, ...] = tuple( - CompiledLineageSourceFact( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name in self.passthrough_resources - ) - return bool(self.passthrough_resources), leaves - - -def resolve_required_external_columns( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - """Resolve model column reads to external resources without expanding whole-query lineage.""" - - polyglot_module: Any = import_polyglot_sql() - try: - parsed: Any = polyglot_module.parse_one( - _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect), - dialect=dialect or "generic", - ) - except polyglot_module.PolyglotError: - return () - relations: dict[str, _RelationReads] = {} - for reference in references: - resource_type: CompiledResourceType | None = _lineage_resource_type(reference) - if resource_type is None: - continue - relations[_analysis_reference_name(reference).casefold()] = _RelationReads( - passthrough_resources=((resource_type, reference.ref_name),) - ) - _, required = _resolve_query_reads( - query=parsed, - inherited_relations=relations, - outer_relations={}, - ) - return _deduplicated_sources(required) - - -def _resolve_query_reads( - *, - query: Any, - inherited_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], -) -> tuple[_RelationReads, tuple[CompiledLineageSourceFact, ...]]: - query = _unwrap_polyglot_annotations(query) - relations: dict[str, _RelationReads] = dict(inherited_relations) - required: list[CompiledLineageSourceFact] = [] - for cte_name, cte_body, has_column_aliases in _polyglot_top_level_ctes(query): - cte_reads: _RelationReads - cte_required: tuple[CompiledLineageSourceFact, ...] - cte_reads, cte_required = _resolve_query_reads( - query=cte_body, - inherited_relations=relations, - outer_relations=outer_relations, - ) - required.extend(cte_required) - if has_column_aliases: - cte_reads = _apply_cte_column_aliases(query=query, cte_name=cte_name, reads=cte_reads) - relations[cte_name.casefold()] = cte_reads - - kind: str = str(getattr(query, "kind", "")) - if kind in POLYGLOT_SET_OPERATION_KINDS: - set_reads, set_required = _resolve_set_operation_reads( - operation=query, - inherited_relations=relations, - outer_relations=outer_relations, - ) - required.extend(set_required) - return set_reads, _deduplicated_sources(required) - if kind != POLYGLOT_KIND_SELECT: - return _RelationReads(), _deduplicated_sources(required) - - local_relations: dict[str, _RelationReads] = {} - for table in _polyglot_direct_select_tables(query): - table_name: str = str(getattr(table, "name", "") or "") - relation: _RelationReads | None = relations.get(table_name.casefold()) - if relation is None: - continue - local_relations[table_name.casefold()] = relation - alias_or_name: str = str(getattr(table, "alias_or_name", "") or "") - if alias_or_name: - local_relations[alias_or_name.casefold()] = relation - - nested_required: list[CompiledLineageSourceFact] = [] - for nested_query, alias in _direct_nested_queries(query): - nested_reads, reads = _resolve_query_reads( - query=nested_query, - inherited_relations=relations, - outer_relations={**outer_relations, **local_relations}, - ) - nested_required.extend(reads) - if alias: - local_relations[alias.casefold()] = nested_reads - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]] = ( - _join_using_column_reads(select=query, local_relations=local_relations) - ) - - slots: list[tuple[str, tuple[CompiledLineageSourceFact, ...]]] = [] - projection_required: list[CompiledLineageSourceFact] = [] - passthrough_resources: list[tuple[CompiledResourceType, str]] = [] - for raw_projection in getattr(query, "expressions", ()): - projection: Any = _unwrap_polyglot_annotations(raw_projection) - if bool(getattr(projection, "is_star", False)): - star_relations: tuple[_RelationReads, ...] = _star_projection_relations( - projection=projection, - local_relations=local_relations, - ) - for relation in star_relations: - slots.extend(relation.slots) - passthrough_resources.extend(relation.passthrough_resources) - slots.extend( - _star_modifier_slots( - projection=projection, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - ) - continue - output_name: str = str(getattr(projection, "output_name", "") or "") - leaves: tuple[CompiledLineageSourceFact, ...] = _resolve_expression_reads( - expression=projection, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - projection_required.extend(leaves) - if output_name: - slots.append((output_name, leaves)) - - required.extend( - _resolve_clause_reads( - select=query, - output_slots=tuple(slots), - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - ) - for leaves in shared_column_reads.values(): - required.extend(leaves) - for _, slot in slots: - required.extend(slot) - required.extend(projection_required) - required.extend(nested_required) - return ( - _RelationReads( - slots=tuple(slots), - passthrough_resources=_deduplicated_resources(passthrough_resources), - ), - _deduplicated_sources(required), - ) - - -def _resolve_set_operation_reads( - *, - operation: Any, - inherited_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], -) -> tuple[_RelationReads, tuple[CompiledLineageSourceFact, ...]]: - branches: list[_RelationReads] = [] - required: list[CompiledLineageSourceFact] = [] - for side in ("left", "right"): - branch: Any | None = operation.arg(side) - if branch is None: - return _RelationReads(), _deduplicated_sources(required) - branch_reads, branch_required = _resolve_query_reads( - query=branch, - inherited_relations=inherited_relations, - outer_relations=outer_relations, - ) - branches.append(branch_reads) - required.extend(branch_required) - left, right = branches - if bool(operation.arg("by_name")): - right_by_name: dict[str, tuple[CompiledLineageSourceFact, ...]] = { - name.casefold(): leaves for name, leaves in right.slots - } - slots: tuple[tuple[str, tuple[CompiledLineageSourceFact, ...]], ...] = tuple( - ( - name, - _deduplicated_sources([*leaves, *right_by_name.get(name.casefold(), ())]), - ) - for name, leaves in left.slots - ) - elif len(left.slots) == len(right.slots): - slots = tuple( - (left_name, _deduplicated_sources([*left_leaves, *right_leaves])) - for (left_name, left_leaves), (_, right_leaves) in zip( - left.slots, - right.slots, - strict=True, - ) - ) - else: - slots = () - passthrough_resources: tuple[tuple[CompiledResourceType, str], ...] = _deduplicated_resources( - [*left.passthrough_resources, *right.passthrough_resources] - ) - return ( - _RelationReads(slots=slots, passthrough_resources=passthrough_resources), - _deduplicated_sources(required), - ) - - -def _resolve_expression_reads( - *, - expression: Any, - local_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]], -) -> tuple[CompiledLineageSourceFact, ...]: - resolved: list[CompiledLineageSourceFact] = [] - for column_name, table_name in _direct_column_refs(expression): - if not column_name: - continue - if table_name: - relation: _RelationReads | None = local_relations.get( - table_name.casefold() - ) or outer_relations.get(table_name.casefold()) - if relation is not None: - resolved.extend(relation.resolve(column_name)) - continue - resolved.extend( - _resolve_unqualified_column( - column_name=column_name, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - ) - return _deduplicated_sources(resolved) - - -def _resolve_clause_reads( - *, - select: Any, - output_slots: tuple[tuple[str, tuple[CompiledLineageSourceFact, ...]], ...], - local_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]], -) -> tuple[CompiledLineageSourceFact, ...]: - output_alias_first_clauses: frozenset[str] = frozenset({"order_by", "qualify"}) - output_alias_fallback_clauses: frozenset[str] = frozenset( - {"cluster_by", "group_by", "having", "sort_by"} - ) - clause_names: tuple[str, ...] = ( - "cluster_by", - "connect", - "distinct_on", - "distribute_by", - "group_by", - "having", - "joins", - "order_by", - "qualify", - "sort_by", - "where_clause", - "windows", - ) - resolved: list[CompiledLineageSourceFact] = [] - args: dict[str, object] = getattr(select, "args", {}) - for clause_name in clause_names: - refs: tuple[tuple[str, str], ...] = _column_refs_in_payload(args.get(clause_name)) - for column_name, table_name in refs: - if not column_name: - continue - if table_name: - relation: _RelationReads | None = local_relations.get( - table_name.casefold() - ) or outer_relations.get(table_name.casefold()) - if relation is not None: - resolved.extend(relation.resolve(column_name)) - continue - output_match: tuple[bool, tuple[CompiledLineageSourceFact, ...]] = _output_slot_reads( - column_name=column_name, output_slots=output_slots - ) - if clause_name in output_alias_first_clauses and output_match[0]: - resolved.extend(output_match[1]) - continue - input_present, input_reads = _resolve_unqualified_column_with_presence( - column_name=column_name, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - if input_present: - resolved.extend(input_reads) - continue - if clause_name in output_alias_fallback_clauses and output_match[0]: - resolved.extend(output_match[1]) - return _deduplicated_sources(resolved) - - -def _output_slot_reads( - *, - column_name: str, - output_slots: tuple[tuple[str, tuple[CompiledLineageSourceFact, ...]], ...], -) -> tuple[bool, tuple[CompiledLineageSourceFact, ...]]: - normalized_name: str = column_name.casefold() - for output_name, leaves in reversed(output_slots): - if output_name.casefold() == normalized_name: - return True, leaves - return False, () - - -def _column_refs_in_payload(payload: object) -> tuple[tuple[str, str], ...]: - if isinstance(payload, list): - refs: list[tuple[str, str]] = [] - for value in payload: - refs.extend(_column_refs_in_payload(value)) - return tuple(refs) - if not isinstance(payload, dict): - return () - payload_dict: dict[str, object] = cast(dict[str, object], payload) - if any( - query_kind in payload_dict - for query_kind in { - _POLYGLOT_KIND_SUBQUERY, - POLYGLOT_KIND_SELECT, - *POLYGLOT_SET_OPERATION_KINDS, - } - ): - return () - column_payload: object = payload_dict.get(POLYGLOT_KIND_COLUMN) - if isinstance(column_payload, dict): - column_dict: dict[str, object] = cast(dict[str, object], column_payload) - column_name: str = _payload_name(column_dict.get("name")) - table_name: str = _payload_name(column_dict.get("table")) - return ((column_name, table_name),) - refs = [] - for value in payload_dict.values(): - refs.extend(_column_refs_in_payload(value)) - return tuple(refs) - - -def _star_modifier_slots( - *, - projection: Any, - local_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]], -) -> tuple[tuple[str, tuple[CompiledLineageSourceFact, ...]], ...]: - slots: list[tuple[str, tuple[CompiledLineageSourceFact, ...]]] = [] - replacements: object = getattr(projection, "args", {}).get("replace") - replacement_expressions: tuple[Any, ...] = tuple(projection.children()) - if isinstance(replacements, list) and len(replacements) == len(replacement_expressions): - for replacement, expression in zip( - replacements, - replacement_expressions, - strict=True, - ): - if not isinstance(replacement, dict): - continue - replacement_dict: dict[str, object] = cast(dict[str, object], replacement) - output_name: str = _payload_name(replacement_dict.get("alias")) - if not output_name: - continue - slots.append( - ( - output_name, - _resolve_expression_reads( - expression=expression, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ), - ) - ) - renames: object = getattr(projection, "args", {}).get("rename") - if isinstance(renames, list): - for rename in renames: - if not isinstance(rename, list): - continue - try: - input_payload, output_payload = rename - except ValueError: - continue - input_name: str = _payload_name(input_payload) - output_name = _payload_name(output_payload) - if not input_name or not output_name: - continue - slots.append( - ( - output_name, - _resolve_unqualified_column( - column_name=input_name, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ), - ) - ) - return tuple(slots) - - -def _resolve_unqualified_column( - *, - column_name: str, - local_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]], -) -> tuple[CompiledLineageSourceFact, ...]: - _, leaves = _resolve_unqualified_column_with_presence( - column_name=column_name, - local_relations=local_relations, - outer_relations=outer_relations, - shared_column_reads=shared_column_reads, - ) - return leaves - - -def _resolve_unqualified_column_with_presence( - *, - column_name: str, - local_relations: dict[str, _RelationReads], - outer_relations: dict[str, _RelationReads], - shared_column_reads: dict[str, tuple[CompiledLineageSourceFact, ...]], -) -> tuple[bool, tuple[CompiledLineageSourceFact, ...]]: - shared_reads: tuple[CompiledLineageSourceFact, ...] | None = shared_column_reads.get( - column_name.casefold() - ) - if shared_reads is not None: - return True, shared_reads - unique_relations: tuple[_RelationReads, ...] = _unique_relations(local_relations) - candidates: list[tuple[CompiledLineageSourceFact, ...]] = [] - for relation in unique_relations: - present, leaves = relation.resolve_with_presence(column_name) - if present: - candidates.append(leaves) - if len(candidates) == 1: - return True, candidates[0] - if candidates: - return True, () - outer_candidates: list[tuple[CompiledLineageSourceFact, ...]] = [] - for relation in _unique_relations(outer_relations): - present, leaves = relation.resolve_with_presence(column_name) - if present: - outer_candidates.append(leaves) - if len(outer_candidates) == 1: - return True, outer_candidates[0] - return bool(outer_candidates), () - - -def _join_using_column_reads( - *, - select: Any, - local_relations: dict[str, _RelationReads], -) -> dict[str, tuple[CompiledLineageSourceFact, ...]]: - joins: object = getattr(select, "args", {}).get("joins") - if not isinstance(joins, list): - return {} - relation_sequence: list[_RelationReads] = [] - for child in select.children(): - kind: str = str(getattr(child, "kind", "")) - relation: _RelationReads | None = None - if kind == POLYGLOT_KIND_TABLE: - table_name: str = str(getattr(child, "name", "") or "") - relation = local_relations.get(table_name.casefold()) - elif kind == _POLYGLOT_KIND_SUBQUERY: - alias: str = str(getattr(child, "alias_or_name", "") or "") - relation = local_relations.get(alias.casefold()) - if relation is not None: - relation_sequence.append(relation) - if not relation_sequence: - return {} - joined_relations: list[_RelationReads] = [relation_sequence[0]] - joined_shared_reads: dict[str, tuple[CompiledLineageSourceFact, ...]] = {} - reads_by_column: dict[str, list[CompiledLineageSourceFact]] = {} - for index, join in enumerate(joins, start=1): - if not isinstance(join, dict): - continue - join_dict: dict[str, object] = cast(dict[str, object], join) - if index >= len(relation_sequence): - break - right_relation: _RelationReads = relation_sequence[index] - using_columns: object = join_dict.get("using") - if isinstance(using_columns, list): - for column in using_columns: - name: str = _payload_name(column) - if not name: - continue - normalized_name: str = name.casefold() - column_reads: list[CompiledLineageSourceFact] = reads_by_column.setdefault( - normalized_name, [] - ) - left_reads: tuple[CompiledLineageSourceFact, ...] | None = joined_shared_reads.get( - normalized_name - ) - if left_reads is None: - left_candidates: list[tuple[CompiledLineageSourceFact, ...]] = [] - for relation in joined_relations: - present, leaves = relation.resolve_with_presence(name) - if present: - left_candidates.append(leaves) - left_reads = left_candidates[0] if len(left_candidates) == 1 else () - _, right_reads = right_relation.resolve_with_presence(name) - combined_reads: tuple[CompiledLineageSourceFact, ...] = _deduplicated_sources( - [*left_reads, *right_reads] - ) - joined_shared_reads[normalized_name] = combined_reads - column_reads.extend(combined_reads) - joined_relations.append(right_relation) - return { - name: _deduplicated_sources(column_reads) for name, column_reads in reads_by_column.items() - } - - -def _payload_name(payload: object) -> str: - if not isinstance(payload, dict): - return "" - payload_dict: dict[str, object] = cast(dict[str, object], payload) - name: object = payload_dict.get("name") - return name if isinstance(name, str) else "" - - -def _direct_column_refs(expression: Any) -> tuple[tuple[str, str], ...]: - kind: str = str(getattr(expression, "kind", "")) - if kind == POLYGLOT_KIND_COLUMN: - column_name: str = str(getattr(expression, "name", "") or "") - table_name: str = "" - table_payload: object = getattr(expression, "args", {}).get("table") - if isinstance(table_payload, dict): - raw_name: object = table_payload.get("name") - if isinstance(raw_name, str): - table_name = raw_name - return ((column_name, table_name),) - if kind == _POLYGLOT_KIND_SUBQUERY or kind == POLYGLOT_KIND_SELECT: - return () - if kind in POLYGLOT_SET_OPERATION_KINDS: - return () - refs: list[tuple[str, str]] = [] - for child in expression.children(): - refs.extend(_direct_column_refs(child)) - return tuple(refs) - - -def _direct_nested_queries(select: Any) -> tuple[tuple[Any, str], ...]: - nested: list[tuple[Any, str]] = [] - for child in select.children(): - if str(getattr(child, "kind", "")) == POLYGLOT_KIND_TABLE: - continue - nested.extend(_nested_queries_in_expression(child)) - return tuple(nested) - - -def _nested_queries_in_expression(expression: Any) -> tuple[tuple[Any, str], ...]: - kind: str = str(getattr(expression, "kind", "")) - if kind == _POLYGLOT_KIND_SUBQUERY: - body: Any | None = getattr(expression, "this", None) - return ( - ((body, str(getattr(expression, "alias_or_name", "") or "")),) - if body is not None - else () - ) - if kind == POLYGLOT_KIND_SELECT or kind in POLYGLOT_SET_OPERATION_KINDS: - return ((expression, ""),) - nested: list[tuple[Any, str]] = [] - for child in expression.children(): - nested.extend(_nested_queries_in_expression(child)) - return tuple(nested) - - -def _star_projection_relations( - *, - projection: Any, - local_relations: dict[str, _RelationReads], -) -> tuple[_RelationReads, ...]: - table_payload: object = getattr(projection, "args", {}).get("table") - table_name: str = "" - if isinstance(table_payload, dict): - raw_name: object = table_payload.get("name") - if isinstance(raw_name, str): - table_name = raw_name - if table_name: - relation: _RelationReads | None = local_relations.get(table_name.casefold()) - return (relation,) if relation is not None else () - return _unique_relations(local_relations) - - -def _apply_cte_column_aliases( - *, - query: Any, - cte_name: str, - reads: _RelationReads, -) -> _RelationReads: - aliases: tuple[str, ...] = () - for raw_cte in _raw_cte_payloads(query): - alias_payload: object = raw_cte.get("alias") - if not isinstance(alias_payload, dict): - continue - alias_dict: dict[str, object] = cast(dict[str, object], alias_payload) - if alias_dict.get("name") != cte_name: - continue - raw_columns: object = raw_cte.get("columns") - if isinstance(raw_columns, list): - aliases = tuple( - str(cast(dict[str, object], column).get("name")) - for column in raw_columns - if isinstance(column, dict) - and isinstance(cast(dict[str, object], column).get("name"), str) - ) - break - if len(aliases) != len(reads.slots): - return _RelationReads() - return _RelationReads( - slots=tuple( - (alias, leaves) for alias, (_, leaves) in zip(aliases, reads.slots, strict=True) - ) - ) - - -def _raw_cte_payloads(query: Any) -> tuple[dict[str, object], ...]: - with_payload: object = getattr(query, "args", {}).get("with") - if not isinstance(with_payload, dict): - return () - raw_ctes: object = with_payload.get("ctes") - if not isinstance(raw_ctes, list): - return () - return tuple(raw_cte for raw_cte in raw_ctes if isinstance(raw_cte, dict)) - - -def _unique_relations(relations: dict[str, _RelationReads]) -> tuple[_RelationReads, ...]: - unique: list[_RelationReads] = [] - seen: set[int] = set() - for relation in relations.values(): - identity: int = id(relation) - if identity in seen: - continue - seen.add(identity) - unique.append(relation) - return tuple(unique) - - -def _deduplicated_resources( - resources: list[tuple[CompiledResourceType, str]], -) -> tuple[tuple[CompiledResourceType, str], ...]: - return tuple(dict.fromkeys(resources)) - - -def _deduplicated_sources( - sources: list[CompiledLineageSourceFact] | tuple[CompiledLineageSourceFact, ...], -) -> tuple[CompiledLineageSourceFact, ...]: - unique: list[CompiledLineageSourceFact] = [] - seen: set[tuple[CompiledResourceType, str, str]] = set() - for source in sources: - key: tuple[CompiledResourceType, str, str] = ( - CompiledResourceType(source.resource_type), - source.resource_name, - source.column_name, - ) - if key in seen: - continue - seen.add(key) - unique.append(source) - return tuple(unique) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 63b904d861..ed20062ef3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -320,15 +320,3 @@ def referenced_model_names( } ) ) - - -def upstream_signatures( - *, model_input: ModelInput, signatures: dict[str, str], available_names: frozenset[str] -) -> dict[str, str]: - """Return the output signatures of the analyzed models this model references.""" - - return { - name: signatures[name] - for name in referenced_model_names(model_input=model_input, available_names=available_names) - if name in signatures - } diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index e262ecad97..665ff09619 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -10,16 +10,7 @@ from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - legacy_schema_rows, native_model_config_session, - validate_declared_schema_models_are_attached, -) -from sqlbuild.compiler.compile._helpers.attachment.references import ( - build_known_function_names, - build_known_ref_names, - build_known_seed_names, - build_known_source_names, - build_known_table_function_names, ) from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( @@ -31,9 +22,6 @@ from sqlbuild.compiler.compile._helpers.render.macros import ( macro_expansion_classes, ) -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - interpolate_sql_batch, -) from sqlbuild.compiler.compile._helpers.render.templating import ( expand_effective_vars, expand_template_data, @@ -46,20 +34,15 @@ DeclarationResolutionContext, LoadedMacro, ModelInputBuildContext, - ModelResourceNames, ModelValidatorContext, NativeModelConfigInputs, - SqlInterpolation, ) from sqlbuild.compiler.compile.types import ModelInput from sqlbuild.compiler.compiled_project.main._model_loop_classes import model_loop_classes -from sqlbuild.compiler.compiled_project.types import SpanRow from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredProjectInputs, - DiscoveredSchemaFile, - DiscoveredSqlHookFile, DiscoveredSqlModelFile, EnumDeclaration, NamedSqlHookEntry, @@ -67,10 +50,6 @@ ) from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes -from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( - scan_native_declaration_references, -) -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, @@ -113,12 +92,6 @@ class _ModelValidationContext: defer_model_sql_validation: bool external_sql_reference_resolver: ExternalSqlReferenceResolver | None sql_lexical_syntax: SqlLexicalSyntax - known_model_names: set[str] - known_seed_names: set[str] - known_source_names: set[str] - known_function_names: set[str] - known_table_function_names: set[str] - custom_materialization_names: frozenset[str] validators: ModelValidatorContext @@ -127,9 +100,6 @@ class _ModelInputLoop: discovered_inputs: DiscoveredProjectInputs context: ModelInputBuildContext validation_context: _ModelValidationContext - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] - legacy_schema_files: tuple[DiscoveredSchemaFile, ...] - native_declaration_references: dict[Path, NativeDeclarationScan] @dataclass @@ -137,7 +107,6 @@ class _ModelLoopHost: """The attachment steps the native model loop calls back into, one model index at a time.""" loop: _ModelInputLoop - interpolations: dict[Path, SqlInterpolation] def constant_text(self, *, declaration: ConstantDeclaration, file_path: Path) -> str: return render_constant_declaration( @@ -159,16 +128,12 @@ def build_model_inputs( ) -> tuple[ModelInput, ...]: """Attach schema metadata to discovered model files.""" - legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = tuple( - schema_file for schema_file in discovered_inputs.schema_files if schema_file.model_entries - ) return _build_model_inputs( discovered_inputs=discovered_inputs, context=context, no_sql_validation=no_sql_validation, defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, - legacy_schema_files=legacy_schema_files, native_project=native_project, ) @@ -180,29 +145,17 @@ def _build_model_inputs( no_sql_validation: bool, defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - legacy_schema_files: tuple[DiscoveredSchemaFile, ...], native_project: _native.NativeCompiledProject, ) -> tuple[ModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings - known_model_names: set[str] = build_known_ref_names(discovered_inputs) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - known_function_names: set[str] = build_known_function_names(discovered_inputs) - known_table_function_names: set[str] = build_known_table_function_names(discovered_inputs) - if external_sql_reference_resolver is not None: - external_sql_reference_resolver.validate_model_names(known_model_names=known_model_names) - custom_materialization_names: frozenset[str] = frozenset( - mf.name for mf in discovered_inputs.materialization_files - ) - names: ModelResourceNames = ModelResourceNames( - models=known_model_names, - seeds=known_seed_names, - sources=known_source_names, - functions=known_function_names, - table_functions=known_table_function_names, - custom_materializations=custom_materialization_names, + validator: _native.NativeModelValidator = _native.NativeModelValidator.from_inputs( + discovered_inputs, effective_settings.microbatch_concurrency ) + if external_sql_reference_resolver is not None: + external_sql_reference_resolver.validate_model_names( + known_model_names=validator.model_names() + ) validation_context: _ModelValidationContext = _ModelValidationContext( effective_settings=effective_settings, no_sql_validation=no_sql_validation, @@ -210,14 +163,7 @@ def _build_model_inputs( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, sql_lexical_syntax=context.sql_lexical_syntax, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - custom_materialization_names=custom_materialization_names, validators=ModelValidatorContext( - names=names, settings=effective_settings, external_sql_reference_resolver=external_sql_reference_resolver, native_config=native_model_config_session( @@ -229,64 +175,20 @@ def _build_model_inputs( run_id=context.run_id, microbatch_concurrency=effective_settings.microbatch_concurrency, ), - names=names, + validator=validator, ), ), ) - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( - discovered_inputs.sql_hook_files - ) - render_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - interpolations: dict[Path, SqlInterpolation] = dict( - zip( - (model_file.file_path for model_file in render_files), - interpolate_sql_batch( - sqls=tuple( - (model_file.query_sql, model_file.file_path) for model_file in render_files - ), - effective_vars=effective_vars, - ), - strict=True, - ) - ) - prepared_files: tuple[DiscoveredSqlModelFile, ...] = tuple( - model_file - for model_file in render_files - if interpolations[model_file.file_path].error is None - ) loop: _ModelInputLoop = _ModelInputLoop( discovered_inputs=discovered_inputs, context=context, validation_context=validation_context, - sql_hook_definitions=sql_hook_definitions, - legacy_schema_files=legacy_schema_files, - native_declaration_references=dict( - zip( - (model_file.file_path for model_file in prepared_files), - scan_native_declaration_references( - sqls=tuple( - interpolations[model_file.file_path].sql for model_file in prepared_files - ) - ), - strict=True, - ) - ), - ) - model_inputs: list[ModelInput] = _run_native_model_loop( - loop=loop, interpolations=interpolations, native_project=native_project - ) - validate_declared_schema_models_are_attached( - model_inputs=tuple(model_inputs), - schema_files=discovered_inputs.schema_files, ) - return tuple(model_inputs) + return tuple(_run_native_model_loop(loop=loop, native_project=native_project)) def _run_native_model_loop( - *, - loop: _ModelInputLoop, - interpolations: dict[Path, SqlInterpolation], - native_project: _native.NativeCompiledProject, + *, loop: _ModelInputLoop, native_project: _native.NativeCompiledProject ) -> list[ModelInput]: """Run the model loop natively; models it cannot finish yet call back into the attachment.""" @@ -320,7 +222,7 @@ def _run_native_model_loop( "sql_hooks": { "named_sql_hook_entry": NamedSqlHookEntry, "python_hook_entry": PythonHookEntry, - "definitions": loop.sql_hook_definitions, + "hook_files": loop.discovered_inputs.sql_hook_files, "effective_vars": loop.context.effective_vars, "environment": UnicodeEnvironment(), "render_project_var": render_project_var_text, @@ -345,42 +247,15 @@ def _run_native_model_loop( loop.validation_context.defer_model_sql_validation, settings.require_sql_analysis, ), - ( - str(loop.validation_context.project_config_path), - legacy_schema_rows(loop.legacy_schema_files), - ), + str(loop.validation_context.project_config_path), ), ) - host: _ModelLoopHost = _ModelLoopHost(loop=loop, interpolations=interpolations) inputs: list[_native.NativeModelInput] = native_loop.run( - native_project, - host, - [ - ( - model_file, - loop.native_declaration_references.get(model_file.file_path), - _interpolation_row(interpolations[model_file.file_path]), - ) - for model_file in files - ], + native_project, _ModelLoopHost(loop=loop), list(files) ) return list(inputs) -def _interpolation_row( - interpolation: SqlInterpolation, -) -> tuple[str, list[tuple[str, str]], str | None, list[SpanRow]]: - return ( - interpolation.sql, - list(interpolation.reads), - interpolation.error, - [ - (span.source_start, span.source_end, span.output_start, span.output_end) - for span in interpolation.spans - ], - ) - - def build_effective_connection( *, project_config: ProjectConfig, @@ -434,19 +309,3 @@ def build_effective_vars( return expand_effective_vars( _native.effective_config_vars((project_config, local_config), target_config, cli_vars) ) - - -def _index_sql_hook_definitions( - hook_files: tuple[DiscoveredSqlHookFile, ...], -) -> dict[str, DiscoveredSqlHookFile]: - definitions: dict[str, DiscoveredSqlHookFile] = {} - hook_file: DiscoveredSqlHookFile - for hook_file in hook_files: - existing: DiscoveredSqlHookFile | None = definitions.get(hook_file.name) - if existing is not None: - raise CompileInputError( - f"Duplicate SQL hook name '{hook_file.name}' found in " - f"{existing.relative_path} and {hook_file.relative_path}" - ) - definitions[hook_file.name] = hook_file - return definitions diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 251aed7bb2..f29b2e7697 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -11,35 +11,24 @@ record_template_reads, ) from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import ( - MODEL_AUDIT_OVERRIDE_KEYS, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileModelConfig, - ModelResourceNames, ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, ) -from sqlbuild.compiler.compile.types import ModelInput from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, DiscoveredSqlModelFile, - ModelSchemaDeclaration, NamedSqlHookEntry, PythonHookEntry, SqlHookEntry, ) from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.path_defaults.main._select import select_path_default from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.spec.contracts.models import ( - DefaultsConfig, MaterializationDefaultsConfig, ProjectConfig, - SchemaModelEntry, TargetConfig, ) @@ -62,267 +51,8 @@ _MODEL_SCHEMA_KEY: str = "model_schema" -def build_layered_model_values( - *, - defaults: DefaultsConfig, - path_defaults: dict[str, dict[str, object]], - matched_path_default: str | None, - model_header_values: dict[str, object], -) -> dict[str, object]: - """Layer project defaults, path defaults, and MODEL header values.""" - - values: dict[str, object] = project_defaults_to_mapping(defaults) - if matched_path_default is not None: - values = _merged_with_tag_union(base=values, overlay=path_defaults[matched_path_default]) - return _merged_with_tag_union(base=values, overlay=model_header_values) - - -def _merged_with_tag_union( - *, base: dict[str, object], overlay: dict[str, object] -) -> dict[str, object]: - """Merge overlay into a copy of base, preserving special config merge semantics.""" - - overlay_tags: object | None = overlay.get("tags") - base_tags: object | None = base.get("tags") - overlay_row_diff_exclude_columns: object | None = overlay.get("row_diff_exclude_columns") - base_row_diff_exclude_columns: object | None = base.get("row_diff_exclude_columns") - overlay_row_diff_tolerances: object | None = overlay.get("row_diff_tolerances") - base_row_diff_tolerances: object | None = base.get("row_diff_tolerances") - result: dict[str, object] = dict(base) - result.update(overlay) - if overlay_tags is not None and base_tags is not None: - merged: list[str] = list(_as_string_list(base_tags)) - tag: str - for tag in _as_string_list(overlay_tags): - if tag not in merged: - merged.append(tag) - result["tags"] = merged - if overlay_row_diff_exclude_columns is not None and base_row_diff_exclude_columns is not None: - result["row_diff_exclude_columns"] = tuple( - _merge_string_sequence( - base=base_row_diff_exclude_columns, - overlay=overlay_row_diff_exclude_columns, - ) - ) - if overlay_row_diff_tolerances is not None and base_row_diff_tolerances is not None: - result["row_diff_tolerances"] = _merge_row_diff_tolerances_mapping( - base=base_row_diff_tolerances, - overlay=overlay_row_diff_tolerances, - ) - return result - - -def _merge_string_sequence(*, base: object, overlay: object) -> list[str]: - """Merge string sequence-like values while preserving first occurrence order.""" - - merged: list[str] = list(_as_string_list(base)) - value: str - for value in _as_string_list(overlay): - if value not in merged: - merged.append(value) - return merged - - -def _merge_row_diff_tolerances_mapping(*, base: object, overlay: object) -> object: - """Deep merge row diff tolerance mappings by section and rule key.""" - - if not isinstance(base, dict) or not isinstance(overlay, dict): - return overlay - - base_mapping: dict[str, object] = cast(dict[str, object], base) - overlay_mapping: dict[str, object] = cast(dict[str, object], overlay) - merged: dict[str, object] = dict(base_mapping) - section: str - for section in ("by_type", "by_column"): - base_section: object | None = base_mapping.get(section) - overlay_section: object | None = overlay_mapping.get(section) - if overlay_section is None: - continue - if isinstance(base_section, dict) and isinstance(overlay_section, dict): - merged[section] = {**base_section, **overlay_section} - else: - merged[section] = overlay_section - key: object - value: object - for key, value in overlay_mapping.items(): - if isinstance(key, str) and key not in MODEL_AUDIT_OVERRIDE_KEYS: - merged[key] = value - return merged - - -def _as_string_list(value: object) -> list[str]: - """Coerce a tags value to a list of strings.""" - - if isinstance(value, list): - return [str(item) for item in value] - if isinstance(value, tuple): - return [str(item) for item in value] - return [] - - -def has_resolved_model_description( - *, - model_file: DiscoveredSqlModelFile, - defaults: DefaultsConfig, - path_defaults: dict[str, dict[str, object]], - model_schemas: dict[str, ModelSchemaDeclaration], -) -> bool: - """Return whether layered config or the model schema gives the model a description.""" - - values: dict[str, object] = build_layered_model_values( - defaults=defaults, - path_defaults=path_defaults, - matched_path_default=find_matching_path_default( - model_file=model_file, path_defaults=path_defaults - ), - model_header_values=model_file.header_values, - ) - schema_name: object | None = values.get(_MODEL_SCHEMA_KEY) - schema: ModelSchemaDeclaration | None = ( - model_schemas.get(schema_name) if isinstance(schema_name, str) else None - ) - return _is_non_empty_text(values.get(_MODEL_DESCRIPTION_KEY)) or _is_non_empty_text( - schema.description if schema is not None else None - ) - - -def _is_non_empty_text(value: object | None) -> bool: - return isinstance(value, str) and bool(value.strip()) - - -def find_matching_path_default( - *, - model_file: DiscoveredSqlModelFile, - path_defaults: dict[str, dict[str, object]], -) -> str | None: - """Return the nearest matching path_defaults key for a model file.""" - - return select_path_default( - model_path=str(model_file.relative_path), - path_keys=tuple(path_defaults), - ).selected_key - - -def project_defaults_to_mapping(defaults: DefaultsConfig) -> dict[str, object]: - """Convert project defaults into a sparse mapping for pre-semantic overlay.""" - - values: dict[str, object] = {} - if defaults.materialized is not None: - values["materialized"] = defaults.materialized - if defaults.database is not None: - values["database"] = defaults.database - if defaults.schema is not None: - values["schema"] = defaults.schema - if defaults.contract is not None: - values["contract"] = defaults.contract - if defaults.incremental_strategy is not None: - values["incremental_strategy"] = defaults.incremental_strategy - if defaults.incremental_mode is not None: - values["incremental_mode"] = defaults.incremental_mode - if defaults.microbatch_strategy is not None: - values["microbatch_strategy"] = defaults.microbatch_strategy - if defaults.cursor_watermark_mode is not None: - values["cursor_watermark_mode"] = defaults.cursor_watermark_mode - if defaults.merge_exclude_columns: - values["merge_exclude_columns"] = defaults.merge_exclude_columns - if defaults.full_refresh is not None: - values["full_refresh"] = defaults.full_refresh - if defaults.append_cursor_inclusive is not None: - values["append_cursor_inclusive"] = defaults.append_cursor_inclusive - if defaults.cursor_start is not None: - values["cursor_start"] = defaults.cursor_start - if defaults.cursor_end is not None: - values["cursor_end"] = defaults.cursor_end - if defaults.cursor_start_max_ahead is not None: - values["cursor_start_max_ahead"] = defaults.cursor_start_max_ahead - if defaults.cursor_start_max_action is not None: - values["cursor_start_max_action"] = defaults.cursor_start_max_action - if defaults.cursor_future_max_distance is not None: - values["cursor_future_max_distance"] = defaults.cursor_future_max_distance - if defaults.cursor_future_action is not None: - values["cursor_future_action"] = defaults.cursor_future_action - if defaults.lookback is not None: - values["lookback"] = defaults.lookback - if defaults.batch_size is not None: - values["batch_size"] = defaults.batch_size - if defaults.batch_concurrency is not None: - values["batch_concurrency"] = defaults.batch_concurrency - if defaults.max_microbatches is not None: - values["max_microbatches"] = defaults.max_microbatches - if defaults.unaccounted_partition_policy is not None: - values["unaccounted_partition_policy"] = defaults.unaccounted_partition_policy - if defaults.replay_on_change is not None: - values["replay_on_change"] = defaults.replay_on_change - if defaults.row_diff_exclude_columns: - values["row_diff_exclude_columns"] = defaults.row_diff_exclude_columns - if defaults.row_diff_tolerances: - values["row_diff_tolerances"] = defaults.row_diff_tolerances - if defaults.row_diff_sample_rows is not None: - values["row_diff_sample_rows"] = defaults.row_diff_sample_rows - if defaults.row_diff_sample_seed is not None: - values["row_diff_sample_seed"] = defaults.row_diff_sample_seed - if defaults.tags: - values["tags"] = list(defaults.tags) - if defaults.pre_hooks is not None: - values["pre_hooks"] = defaults.pre_hooks - if defaults.post_hooks is not None: - values["post_hooks"] = defaults.post_hooks - return values - - -def legacy_schema_rows( - schema_files: tuple[DiscoveredSchemaFile, ...], -) -> list[tuple[str, list[str]]]: - """Each legacy schema file's relative path and the model names it declares.""" - - rows: list[tuple[str, list[str]]] = [] - for schema_file in schema_files: - rows.append( - (str(schema_file.relative_path), [entry.name for entry in schema_file.model_entries]) - ) - return rows - - -def validate_declared_schema_models_are_attached( - *, - model_inputs: tuple[ModelInput, ...], - schema_files: tuple[DiscoveredSchemaFile, ...], -) -> None: - """Ensure every declared schema.yml model entry attaches within its directory scope.""" - - attached_model_names: set[str | None] = { - model_input.attached_schema_name for model_input in model_inputs - } - schema_file: DiscoveredSchemaFile - for schema_file in schema_files: - schema_entry: SchemaModelEntry - for schema_entry in schema_file.model_entries: - if schema_entry.name not in attached_model_names: - raise CompileInputError( - f"schema.yml declaration for model '{schema_entry.name}' in " - f"{schema_file.relative_path} " - "does not match any discovered model file in that directory scope" - ) - - -def _str_from_dict(*, values: dict[str, object], key: str) -> str | None: - """Extract a string value from a dict.""" - - raw: object | None = values.get(key) - return raw if isinstance(raw, str) else None - - -def _bool_from_dict(*, values: dict[str, object], key: str) -> bool: - raw: object | None = values.get(key) - if raw is None: - return False - if not isinstance(raw, bool): - raise CompileInputError(f"AUDIT() '{key}' must be a boolean") - return raw - - def native_model_config_session( - *, inputs: NativeModelConfigInputs, names: ModelResourceNames + *, inputs: NativeModelConfigInputs, validator: _native.NativeModelValidator ) -> NativeModelConfigSession: """Capture the project inputs every model's native config build and validation read.""" @@ -335,7 +65,7 @@ def native_model_config_session( path_defaults=project_config.path_defaults, builder=_native.NativeModelConfigBuilder( ( - project_defaults_to_mapping(project_config.defaults), + project_config.defaults, project_config.path_defaults, (SqlHookEntry, NamedSqlHookEntry, PythonHookEntry), ), @@ -345,11 +75,7 @@ def native_model_config_session( None if target is None else (target.database, target.schema), ), ), - validator=_native.NativeModelValidator( - (names.models, names.seeds, names.sources, names.functions, names.table_functions), - set(names.custom_materializations), - inputs.microbatch_concurrency, - ), + validator=validator, inherited_storage={ materialized: ( resolve_time_travel_retention( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py deleted file mode 100644 index 4238a34d8c..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import TABLE_FUNCTION_RETURN_KEYS -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, -) - - -def build_known_ref_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: - """Build the set of names valid as __ref() targets.""" - - return { - discovered_model_file.file_path.stem - for discovered_model_file in discovered_inputs.model_files - } - - -def build_known_seed_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: - """Build the set of names valid as __seed() targets.""" - - seed_names: set[str] = set() - for schema_file in discovered_inputs.schema_files: - for seed_entry in schema_file.seed_entries: - seed_names.add(seed_entry.name) - return seed_names - - -def build_known_source_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: - """Build the set of names valid as __source() targets.""" - - source_names: set[str] = set() - for source_file in discovered_inputs.source_files: - for source_entry in source_file.source_entries: - source_names.add(source_entry.name) - return source_names - - -def build_known_function_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: - """Build the set of names valid as __udf() targets.""" - - return { - *(function_file.file_path.stem for function_file in discovered_inputs.sql_function_files), - *( - function_file.file_path.stem - for function_file in discovered_inputs.python_function_files - ), - } - - -def build_known_table_function_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: - """Build the set of discovered SQL functions declared as table functions.""" - - return { - function_file.file_path.stem - for function_file in discovered_inputs.sql_function_files - if _is_table_function_returns(function_file.header_values.get("returns")) - } - - -def _is_table_function_returns(raw_returns: object) -> bool: - return isinstance(raw_returns, dict) and set(raw_returns) == TABLE_FUNCTION_RETURN_KEYS diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py index b78cd9b79c..cc2e0e7731 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py @@ -29,12 +29,6 @@ } -def compiled_resource_type(kind: ResourceKind) -> CompiledResourceType | None: - """Return the compiled resource type diagnostics use for a scope resource kind.""" - - return _RESOURCE_TYPES.get(kind) - - def report_scope_index_errors(*, index: ScopeIndex) -> None: """Report each error-severity scope finding as its own compile diagnostic.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index a8ae343999..8c1bfab4f3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import replace from functools import partial from pathlib import Path from types import MappingProxyType @@ -23,11 +22,9 @@ ConstantDeclaration, DiscoveredConstantFile, DiscoveredEnumFile, - DiscoveredModelSchemaFile, DiscoveredProjectInputs, DiscoveredSqlModelFile, EnumDeclaration, - ModelSchemaDeclaration, ) from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( build_native_declaration_contexts, @@ -46,7 +43,6 @@ ResourceKind, ScopeKind, ) -from sqlbuild.spec.contracts.models import SchemaColumn from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind @@ -54,34 +50,6 @@ _ACCEPTED_VALUES_AUDIT: str = "accepted_values" -def build_public_declaration_indexes( - *, discovered_inputs: DiscoveredProjectInputs -) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: - """Build collision-checked indexes of every public declaration.""" - - enums: dict[str, EnumDeclaration] = {} - constants: dict[str, ConstantDeclaration] = {} - enum_file: DiscoveredEnumFile - for enum_file in discovered_inputs.enum_files: - declaration: EnumDeclaration - for declaration in enum_file.declarations: - enums = _with_declaration( - declarations=enums, - declaration=declaration, - kind="enum", - ) - constant_file: DiscoveredConstantFile - for constant_file in discovered_inputs.constant_files: - constant_declaration: ConstantDeclaration - for constant_declaration in constant_file.declarations: - constants = _with_declaration( - declarations=constants, - declaration=constant_declaration, - kind="constant", - ) - return enums, constants - - def build_declaration_scope_resolver( *, discovered_inputs: DiscoveredProjectInputs, @@ -164,103 +132,6 @@ def build_model_declaration_indexes( ) -def build_public_model_schema_index( - *, discovered_inputs: DiscoveredProjectInputs -) -> dict[str, ModelSchemaDeclaration]: - """Build and resolve the collision-checked public model-schema index.""" - - authored: dict[str, ModelSchemaDeclaration] = {} - schema_file: DiscoveredModelSchemaFile - for schema_file in discovered_inputs.model_schema_files: - declaration: ModelSchemaDeclaration - for declaration in schema_file.declarations: - existing: ModelSchemaDeclaration | None = authored.get(declaration.name) - if existing is not None: - raise CompileInputError( - f"Duplicate public schema '{declaration.name}' in " - f"{existing.relative_path} and {declaration.relative_path}" - ) - authored[declaration.name] = declaration - - resolved: dict[str, ModelSchemaDeclaration] = {} - for schema_name in authored: - _declaration: ModelSchemaDeclaration - _declaration, resolved = _resolve_model_schema_declaration( - name=schema_name, - authored=authored, - resolved=resolved, - resolving=(), - ) - return resolved - - -def _resolve_model_schema_declaration( - *, - name: str, - authored: dict[str, ModelSchemaDeclaration], - resolved: dict[str, ModelSchemaDeclaration], - resolving: tuple[str, ...], -) -> tuple[ModelSchemaDeclaration, dict[str, ModelSchemaDeclaration]]: - existing_resolved: ModelSchemaDeclaration | None = resolved.get(name) - if existing_resolved is not None: - return existing_resolved, resolved - if name in resolving: - cycle_start: int = resolving.index(name) - cycle: str = " -> ".join((*resolving[cycle_start:], name)) - raise CompileInputError(f"Model schema inheritance cycle: {cycle}") - declaration: ModelSchemaDeclaration = authored[name] - inherited_columns: tuple[SchemaColumn, ...] = () - if declaration.extends is not None: - parent: ModelSchemaDeclaration | None = authored.get(declaration.extends) - if parent is None: - raise CompileInputError( - f"Schema '{name}' in {declaration.relative_path} extends unknown schema " - f"'{declaration.extends}'" - ) - resolved_parent: ModelSchemaDeclaration - resolved_parent, resolved = _resolve_model_schema_declaration( - name=parent.name, - authored=authored, - resolved=resolved, - resolving=(*resolving, name), - ) - inherited_columns = resolved_parent.columns - _validate_no_inherited_column_overrides( - declaration=declaration, - inherited_columns=inherited_columns, - ) - resolved_declaration: ModelSchemaDeclaration = replace( - declaration, - columns=(*inherited_columns, *declaration.columns), - ) - return resolved_declaration, resolved | {name: resolved_declaration} - - -def _validate_no_inherited_column_overrides( - *, - declaration: ModelSchemaDeclaration, - inherited_columns: tuple[SchemaColumn, ...], -) -> None: - inherited_by_name: dict[str, SchemaColumn] = { - column.name.lower(): column for column in inherited_columns - } - local_column: SchemaColumn - for local_column in declaration.columns: - inherited: SchemaColumn | None = inherited_by_name.get(local_column.name.lower()) - if inherited is None: - continue - inherited_origin: str = ( - f"{inherited.location.path}:{inherited.location.line}" - if inherited.location is not None - else "an ancestor schema" - ) - raise CompileInputError( - f"Schema '{declaration.name}' in {declaration.relative_path} redeclares inherited " - f"column '{local_column.name}' from {inherited_origin}; column overrides are not " - "supported" - ) - - def expand_scanned_declaration_references( # noqa: PLR0913 *, sql: str, @@ -302,18 +173,6 @@ def expand_scanned_declaration_references( # noqa: PLR0913 ) -def _with_declaration[T: EnumDeclaration | ConstantDeclaration]( - *, declarations: dict[str, T], declaration: T, kind: str -) -> dict[str, T]: - existing: T | None = declarations.get(declaration.name) - if existing is not None: - raise CompileInputError( - f"Duplicate public {kind} '{declaration.name}' in {existing.relative_path} and " - f"{declaration.relative_path}" - ) - return declarations | {declaration.name: declaration} - - def render_constant_declaration( *, declaration: ConstantDeclaration, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index c2cbcf5503..e462b2ca33 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -11,21 +11,18 @@ expand_scanned_declaration_references, ) from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros_result, expand_sql_macros_with_spans, ) from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, DeclarationExpansionResult, DeclarationResolutionContext, DeclarationScopeResolver, ExpansionSpan, LoadedMacro, MacroContext, - MacroExpansionResult, SqlInterpolation, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer @@ -39,53 +36,6 @@ ] -def expand_authored_sql_result( # noqa: PLR0913 - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - context_values: Mapping[str, str | None] | None = None, - declarations: DeclarationResolutionContext | None = None, - declaration_resolver: DeclarationScopeResolver | None = None, -) -> AuthoredSqlExpansionResult: - """Apply all expansion passes and retain facts emitted by those passes.""" - - interpolated_sql: str = substitute_sql_vars( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - declaration_context: DeclarationResolutionContext = ( - declarations or DeclarationResolutionContext() - ) - declaration_result: DeclarationExpansionResult = expand_scanned_declaration_references( - sql=interpolated_sql, - references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], - file_path=file_path, - declarations=declaration_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) - macro_result: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_result.sql, - file_path=file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - consumer=declaration_context.consumer, - ) - return AuthoredSqlExpansionResult( - sql=macro_result.sql, - usages=tuple(dict.fromkeys((*declaration_result.usages, *macro_result.usages))), - argument_references=macro_result.argument_references, - ) - - def expand_authored_sql_with_spans( *, sql: str, @@ -138,24 +88,6 @@ def expand_authored_sql_with_spans( return macro_expanded_sql, (interpolation_spans, declaration_spans, macro_spans) -def substitute_sql_vars( - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None = None, -) -> str: - """Replace @@name, @@ENV:NAME, and allowed @@CTX:name references in SQL text.""" - - return applied_interpolation( - interpolate_sql_batch( - sqls=((sql, file_path),), - effective_vars=effective_vars, - context_values=context_values, - )[0] - ) - - def substitute_sql_vars_with_spans( *, sql: str, diff --git a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py index 5a42738641..b0d21a08e3 100644 --- a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py +++ b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py @@ -4,12 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - has_resolved_model_description, -) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_public_model_schema_index, -) +import sqlbuild._native as _native from sqlbuild.compiler.discovery.main._model_description_inputs import ( discover_model_description_inputs, ) @@ -26,16 +21,7 @@ def externally_described_models(*, project_dir: Path) -> frozenset[Path]: discovered_inputs: DiscoveredProjectInputs = discover_model_description_inputs( project_dir=project_dir ) - model_schemas: dict[str, ModelSchemaDeclaration] = build_public_model_schema_index( - discovered_inputs=discovered_inputs - ) - return frozenset( - model_file.file_path - for model_file in discovered_inputs.model_files - if has_resolved_model_description( - model_file=model_file, - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - model_schemas=model_schemas, - ) + model_schemas: dict[str, ModelSchemaDeclaration] = _native.public_model_schema_index( + discovered_inputs ) + return frozenset(_native.externally_described_models(discovered_inputs, model_schemas)) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 58e3475c16..9e29853df1 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -69,8 +69,6 @@ OMITTED_CEREMONIAL_SELECT_SQL: str = f"\nSELECT {SQL_CEREMONIAL_SELECT_VALUE}" -TABLE_FUNCTION_RETURN_KEYS: frozenset[str] = frozenset({"table"}) - MACRO_TOKEN: str = "@" DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 3581348c65..5a1d9961e1 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -115,11 +115,3 @@ def __init__( self.test_index = test_index self.token = token self.token_offset = token_offset - - -class CompactAnalysisInputError(CompileInputError): - """Raised when compact SQL-analysis batch inputs are inconsistent.""" - - -class AnalysisCacheEntryError(ValueError): - """Raised when a persisted model analysis cache entry is invalid.""" diff --git a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py b/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py index 24148071f8..bfae6b5bae 100644 --- a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py +++ b/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py @@ -1,8 +1,6 @@ """Public compile-time resolved column-read analysis operation.""" -from sqlbuild.compiler.compile._helpers.analysis.required_columns import ( - resolve_required_external_columns, -) +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( CompiledLineageSourceFact, CompileSqlReference, @@ -17,8 +15,15 @@ def analyze_resolved_column_reads( ) -> tuple[CompiledLineageSourceFact, ...]: """Return CTE-aware column reads resolved to external resources.""" - return resolve_required_external_columns( - query_sql=query_sql, - references=references, - dialect=dialect, + reference_rows: list[tuple[str, str]] = [ + (str(reference.ref_kind), reference.ref_name) for reference in references + ] + reads: list[tuple[str, str, str]] = _native.resolve_required_columns( + query_sql, reference_rows, dialect + ) + return tuple( + CompiledLineageSourceFact( + resource_type=resource_type, resource_name=resource_name, column_name=column_name + ) + for resource_type, resource_name, column_name in reads ) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 4505b74dd0..d5e691c979 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -21,10 +21,6 @@ target_context_with_macro_call_store, ) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_public_declaration_indexes, - build_public_model_schema_index, -) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -222,11 +218,9 @@ def _build_models_with_declarations( ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] - public_enums, public_constants = build_public_declaration_indexes( - discovered_inputs=discovered_inputs - ) - public_model_schemas: dict[str, ModelSchemaDeclaration] = build_public_model_schema_index( - discovered_inputs=discovered_inputs + public_enums, public_constants = _native.public_declaration_indexes(discovered_inputs) + public_model_schemas: dict[str, ModelSchemaDeclaration] = _native.public_model_schema_index( + discovered_inputs ) declarations: DeclarationResolutionContext = DeclarationResolutionContext( enums=public_enums, diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index 06e375ed70..ccb8a873a8 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -4,6 +4,7 @@ from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, @@ -11,7 +12,6 @@ ) from sqlbuild.compiler.compile._helpers.render.declarations import ( build_model_declaration_indexes, - build_public_declaration_indexes, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.models import ( @@ -65,9 +65,7 @@ def build_sql_expansion_context( loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros enums: dict[str, EnumDeclaration] constants: dict[str, ConstantDeclaration] - enums, constants = build_public_declaration_indexes( - discovered_inputs=effective_discovered_inputs - ) + enums, constants = _native.public_declaration_indexes(effective_discovered_inputs) local_declarations: dict[Path, DeclarationResolutionContext] = {} for model_file in effective_discovered_inputs.model_files: local_enums: dict[str, EnumDeclaration] diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 997f0f6de3..e8218a86c1 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, AuditInput, - CompactBatchResponseCallback, CompiledAuditView, CompiledFunctionView, CompiledModelView, @@ -193,16 +192,6 @@ class DynamicColumnContractProof: bare_dynamic_pivot: bool = False -@dataclass(frozen=True) -class CteFactResolvers: - """Expression resolvers used for conservative CTE fact recovery.""" - - expression_type: Callable[..., Any] - nullability: Callable[..., Any] - shallow_nullability: Callable[..., Any] - alias_nullability: Callable[..., Any] - - @dataclass(frozen=True) class NonNullFilterContext: """Resolved relations and columns constrained by non-null filters.""" @@ -311,10 +300,6 @@ class DeclarationScopeResolver: default_factory=dict, compare=False, repr=False ) - def cache_context(self, *, key: tuple[str, str], context: DeclarationResolutionContext) -> None: - """Retain a process-local context for equivalent lexical consumers.""" - self.contexts_by_directory[key] = context - @dataclass(frozen=True) class DeclarationScopeBuild: @@ -399,15 +384,6 @@ class DeclarationExpansionResult: usages: tuple[UsageRecord, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class AuthoredSqlExpansionResult: - """Fully expanded authored SQL with all resolved declaration usages.""" - - sql: str - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class DeclarationResolutionContext: """Declarations visible during one authored SQL expansion.""" @@ -1511,62 +1487,6 @@ def __post_init__(self) -> None: ) -@dataclass(frozen=True) -class CompactProjectedFacts: - """Decoded compact projection and lineage rows shared by projected analyses.""" - - columns: tuple[InferredColumn, ...] - lineage_rows: tuple[ - tuple[int, int, int, tuple[tuple[int, int, int], ...]], - ..., - ] - - -@dataclass(frozen=True) -class CompactBatchExecutionOptions: - """Optional binding work and cache publication for one native batch.""" - - binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None = None - on_response: CompactBatchResponseCallback | None = None - shareable_prekeys: frozenset[str] | None = None - - -@dataclass(frozen=True) -class CompactBatchPreparation: - """Deduplicated native request data and ordered cleaned SQL inputs.""" - - cleaned_sql: tuple[str, ...] - queries: tuple[dict[str, object], ...] - templates: tuple[dict[str, object], ...] - projections: tuple[dict[str, object], ...] - binding_catalog: Any | None = field(default=None, repr=False, compare=False) - shared_query_indexes: frozenset[int] = frozenset() - - -@dataclass(frozen=True) -class CompactBatchInputs: - """Ordered per-model inputs of one compact native analysis batch.""" - - query_sqls: tuple[str, ...] - references: tuple[tuple[CompileSqlReference, ...], ...] - placeholders: tuple[dict[str, str] | None, ...] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - column_types_by_table: dict[str, dict[str, str]] - inference_profile: Any - recover_cte_facts: tuple[bool, ...] - rich_type_inference: bool - binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None - - -@dataclass(frozen=True) -class CompactBatchContext: - """Binding catalog, cleaned SQL, and binding references prepared for one batch.""" - - binding_catalog: Any | None - normalized_sqls: list[str] - binding_queries: tuple[PreparedBindingQuery | None, ...] - - @dataclass(frozen=True) class SharedBindingQuery: """Relation-stubbed binding query shared by models with identical input shapes.""" @@ -1576,101 +1496,6 @@ class SharedBindingQuery: key: tuple[object, ...] -@dataclass(frozen=True) -class PreparedBindingQuery: - """Native binding references for one model and its optional shared form.""" - - references: list[tuple[str, bool]] - overrides: dict[str, Mapping[str, str]] - shared: SharedBindingQuery | None = None - - -@dataclass(frozen=True) -class CompactProjectionCaches: - """Shared identity caches used while decoding compact native facts.""" - - columns: dict[tuple[int, int | None, int], InferredColumn] | None = None - facts: dict[int, CompactProjectedFacts] | None = None - decoded_facts: ( - dict[ - int, - tuple[InferredColumn, tuple[int, int, int, tuple[tuple[int, int, int], ...]]], - ] - | None - ) = None - - def column(self, key: tuple[int, int | None, int]) -> InferredColumn | None: - return self.columns.get(key) if self.columns is not None else None - - def remember_column(self, *, key: tuple[int, int | None, int], column: InferredColumn) -> None: - if self.columns is not None: - self.columns[key] = column - - def fact(self, template_index: int | None) -> CompactProjectedFacts | None: - if self.facts is None or template_index is None: - return None - return self.facts.get(template_index) - - def remember_fact(self, *, template_index: int | None, fact: CompactProjectedFacts) -> None: - if self.facts is not None and template_index is not None: - self.facts[template_index] = fact - - def decoded_fact( - self, fact_index: int - ) -> tuple[InferredColumn, tuple[int, int, int, tuple[tuple[int, int, int], ...]]] | None: - return self.decoded_facts.get(fact_index) if self.decoded_facts is not None else None - - def remember_decoded_fact( - self, - *, - fact_index: int, - fact: tuple[InferredColumn, tuple[int, int, int, tuple[tuple[int, int, int], ...]]], - ) -> None: - if self.decoded_facts is not None: - self.decoded_facts[fact_index] = fact - - -@dataclass(frozen=True) -class ProjectedAnalysisRequest: - """Inputs needed to project one native analysis into the public result contract.""" - - analysis: dict[str, Any] | None - binding_diagnostics: tuple[SqlBindingDiagnostic, ...] - binding_validated: bool - compact_rows: list[object] | None = None - compact_fact_rows: list[object] | None = None - string_pool: tuple[str, ...] = () - caches: CompactProjectionCaches = field(default_factory=CompactProjectionCaches) - template_index: int | None = None - resource_name_indexes: dict[int, int] = field(default_factory=dict) - resource_names: dict[int, str] = field(default_factory=dict) - - -@dataclass(frozen=True) -class NativeCompactAnalysis: - """One native compact result prepared for Python contract projection.""" - - cleaned_sql: str - analysis: dict[str, Any] | None - projected: bool = False - binding_diagnostics: tuple[SqlBindingDiagnostic, ...] | None = None - compact_rows: list[object] | None = None - compact_fact_rows: list[object] | None = None - string_pool: tuple[str, ...] = () - compact_column_cache: dict[tuple[int, int | None, int], InferredColumn] | None = None - compact_template_index: int | None = None - compact_fact_cache: dict[int, CompactProjectedFacts] | None = None - compact_decoded_fact_cache: ( - dict[ - int, - tuple[InferredColumn, tuple[int, int, int, tuple[tuple[int, int, int], ...]]], - ] - | None - ) = None - resource_name_indexes: dict[int, int] = field(default_factory=dict) - resource_names: dict[int, str] = field(default_factory=dict) - - @dataclass(frozen=True) class NativeModelConfigSession: """The native config builder, validator, and inherited storage policies for one compile.""" @@ -1693,18 +1518,6 @@ class NativeModelConfigInputs: microbatch_concurrency: bool -@dataclass(frozen=True) -class ModelResourceNames: - """Discovered resource names the native reference check reads.""" - - models: set[str] - seeds: set[str] - sources: set[str] - functions: set[str] - table_functions: set[str] - custom_materializations: frozenset[str] - - @dataclass(frozen=True) class ModelValidationRequest: """One model's effective config and the facts its validators read.""" @@ -1720,7 +1533,6 @@ class ModelValidationRequest: class ModelValidatorContext: """The project facts the model validators read, and the native config session.""" - names: ModelResourceNames settings: SettingsConfig external_sql_reference_resolver: ExternalSqlReferenceResolver | None native_config: NativeModelConfigSession diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 0b9da83ea9..44f5090117 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -15,7 +15,6 @@ from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( - CompactBatchPreparation, CompiledLineageColumnFact, CompiledObjectKey, CompiledRelationLocation, @@ -597,12 +596,6 @@ def render_typed_array(self, *, value: SqlValue) -> str: ... def render_typed_object(self, *, value: SqlValue) -> str: ... -class CompactBatchResponseCallback(Protocol): - """Keyword-only callback for publishing one native compact response.""" - - def __call__(self, *, preparation: CompactBatchPreparation, response: object) -> None: ... - - class CompileInputsReadyCallback(Protocol): """Invocation-local observer for the completed pre-semantic input snapshot.""" diff --git a/src/sqlbuild/compiler/lineage/exceptions.py b/src/sqlbuild/compiler/lineage/exceptions.py index 20dee9b906..5913f3a8a7 100644 --- a/src/sqlbuild/compiler/lineage/exceptions.py +++ b/src/sqlbuild/compiler/lineage/exceptions.py @@ -1,9 +1,5 @@ """Column-lineage failures.""" -class LineageAnalysisError(RuntimeError): - """Raised when the native lineage boundary returns an invalid payload.""" - - class UnknownLineageDialectError(ValueError): """Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index ac7b1c0f54..5213e39156 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -50,12 +50,7 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME -from sqlbuild.compiler.planner._helpers.graph.buildability import ( - check_buildability, - missing_upstream_message, -) from sqlbuild.compiler.planner._helpers.graph.core import ( - build_execution_edge_origins, build_execution_upstream_deps, ) from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( @@ -78,9 +73,7 @@ CursorInputEvidence, CursorOverrides, CursorSnapshotScope, - MissingUpstream, ModelCursorSnapshot, - PlannerScope, WarehouseFingerprints, WarehouseSnapshot, ) @@ -204,49 +197,6 @@ class _StateTableSchemas: migrations: frozenset[str] -def build_warehouse_snapshot( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - full_refresh: bool = False, - on_progress: Callable[[str], None] | None = None, - deferred_locations: dict[str, CompiledRelationLocation] | None = None, - deferred_relations: dict[str, RelationInfo] | None = None, -) -> WarehouseSnapshot: - """Gather warehouse state and validate selected upstream availability.""" - - snapshot: WarehouseSnapshot = gather_warehouse_snapshot( - project=project, - adapter=adapter, - connection=connection, - execute=adapter.execute, - selected_keys=scope.selected_keys, - full_refresh_model_names=effectively_full_refreshed_model_names( - project=project, - cli_full_refresh=full_refresh, - ), - on_progress=on_progress, - deferred_locations=deferred_locations, - ) - missing: tuple[MissingUpstream, ...] = check_buildability( - selected_keys=scope.selected_keys, - upstream_deps=scope.upstream_deps, - snapshot=snapshot, - deferred_relations=deferred_relations, - satisfied_keys=frozenset(seed.key for seed in project.seeds if seed.external), - ) - if missing: - raise PlannerInputError( - missing_upstream_message( - missing=missing, edge_origins=build_execution_edge_origins(project) - ), - code="S301", - ) - return snapshot - - def gather_warehouse_snapshot( *, project: CompiledProjectView, diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index e9ff44177c..be247a0320 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -30,35 +30,6 @@ POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 BINDING_UNKNOWN_TABLE_INTERNAL_CODE: str = "B000" -POLYGLOT_KIND_ANNOTATED: str = "annotated" -POLYGLOT_KIND_COLUMN: str = "column" -POLYGLOT_KIND_EXCEPT: str = "except" -POLYGLOT_KIND_INTERSECT: str = "intersect" -POLYGLOT_KIND_SELECT: str = "select" -POLYGLOT_KIND_TABLE: str = "table" -POLYGLOT_KIND_UNION: str = "union" -POLYGLOT_SET_OPERATION_KINDS: frozenset[str] = frozenset( - {POLYGLOT_KIND_UNION, POLYGLOT_KIND_INTERSECT, POLYGLOT_KIND_EXCEPT} -) -POLYGLOT_PAYLOAD_COLUMN: str = "column" -POLYGLOT_PAYLOAD_NAME: str = "name" -POLYGLOT_ANALYSIS_IS_STAR: str = "isStar" -POLYGLOT_ANALYSIS_NAME: str = "name" -POLYGLOT_ANALYSIS_NULLABILITY: str = "nullability" -POLYGLOT_ANALYSIS_NULLABILITY_NON_NULL: str = "non_null" -POLYGLOT_ANALYSIS_NULLABILITY_NULLABLE: str = "nullable" -POLYGLOT_ANALYSIS_PROJECTIONS: str = "projections" -POLYGLOT_ANALYSIS_SOURCE_ALIAS: str = "sourceAlias" -POLYGLOT_ANALYSIS_SOURCE_CONFIDENCE: str = "confidence" -POLYGLOT_ANALYSIS_SOURCE_NAME: str = "sourceName" -POLYGLOT_ANALYSIS_TABLE: str = "table" -POLYGLOT_ANALYSIS_TRANSFORM_KIND: str = "transformKind" -POLYGLOT_ANALYSIS_STAR_PROJECTIONS: str = "starProjections" -POLYGLOT_ANALYSIS_UPSTREAM: str = "upstream" -POLYGLOT_ANALYSIS_TRANSFORM_AGGREGATION: str = "aggregation" -POLYGLOT_ANALYSIS_TRANSFORM_CAST: str = "cast" -POLYGLOT_ANALYSIS_TRANSFORM_CONSTANT: str = "constant" -POLYGLOT_ANALYSIS_TRANSFORM_DIRECT: str = "direct" TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) SQL_QUOTED_IDENTIFIER_DELIMITER: str = '"' CASE_SENSITIVE_BINDING_DIALECTS: frozenset[str] = frozenset({"snowflake", "postgresql"}) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_decode_schema_validation.py b/src/sqlbuild/compiler/sql_analysis/main/_decode_schema_validation.py deleted file mode 100644 index d72928817c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_decode_schema_validation.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Decode binding evidence from native compilation.""" - -from sqlbuild.compiler.sql_analysis._helpers.schema_validation import _binding_result -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult - - -def decode_schema_validation( - *, sql: str, dialect: str | None, response: object -) -> SqlBindingResult: - """Decode binding evidence produced by either native compilation entry point.""" - return _binding_result(sql=sql, dialect=dialect, response=response) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py b/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py deleted file mode 100644 index 649dbd11d2..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Top-level SQL code position scanning entrypoint.""" - -from collections.abc import Iterator - -from sqlbuild.compiler.sql_analysis._helpers.scanning import iter_code_positions_impl -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def iter_code_positions( - *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None -) -> Iterator[tuple[int, int]]: - """Yield each code offset outside comments and quotes with its parenthesis depth.""" - - return iter_code_positions_impl(sql=sql, context=context, syntax=syntax) diff --git a/src/sqlbuild/compiler/sql_analysis/types.py b/src/sqlbuild/compiler/sql_analysis/types.py index f8fa8d5ac4..266c536fef 100644 --- a/src/sqlbuild/compiler/sql_analysis/types.py +++ b/src/sqlbuild/compiler/sql_analysis/types.py @@ -70,13 +70,3 @@ class NativeValidationModule(Protocol): def validate_sql_with_schema_json(self, request_json: str) -> str: ... def validate_sql_with_schemas_json(self, request_json: str) -> str: ... - - -class NativeQueryAnalysisModule(Protocol): - """Native compact query-analysis boundary.""" - - def analyze_queries_json(self, request_json: str) -> str: ... - - def analyze_project_queries_json(self, request_json: str) -> str: ... - - def analyze_project_queries_compact_json(self, request_json: str) -> str: ... diff --git a/tests/goldens/compiler/failure/model-schema-extends-unknown.json b/tests/goldens/compiler/failure/model-schema-extends-unknown.json new file mode 100644 index 0000000000..4721971d57 --- /dev/null +++ b/tests/goldens/compiler/failure/model-schema-extends-unknown.json @@ -0,0 +1,18 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Schema 'order_row' in schemas/orders.sql extends unknown schema 'base_row'" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/model-schema-inheritance-cycle.json b/tests/goldens/compiler/failure/model-schema-inheritance-cycle.json new file mode 100644 index 0000000000..9347daebcf --- /dev/null +++ b/tests/goldens/compiler/failure/model-schema-inheritance-cycle.json @@ -0,0 +1,18 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Model schema inheritance cycle: order_row -> base_row -> order_row" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/model-schema-redeclared-column.json b/tests/goldens/compiler/failure/model-schema-redeclared-column.json new file mode 100644 index 0000000000..f4d8e65977 --- /dev/null +++ b/tests/goldens/compiler/failure/model-schema-redeclared-column.json @@ -0,0 +1,18 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Schema 'order_row' in schemas/orders.sql redeclares inherited column 'ID' from schemas/orders.sql:1; column overrides are not supported" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index c15b992a00..cab46a6b74 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -14,7 +14,6 @@ from sqlbuild.compiler.compile.models import ( CompileModelConfig, CompileSqlReference, - ModelResourceNames, ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, @@ -24,14 +23,6 @@ from sqlbuild.spec.contracts.models import ProjectConfig _ACCEPTED: str = "accepted" -_NAMES: ModelResourceNames = ModelResourceNames( - models={"orders", "customers"}, - seeds={"regions"}, - sources={"raw.orders"}, - functions={"cents", "order_lines"}, - table_functions={"order_lines"}, - custom_materializations=frozenset({"ledger"}), -) def validation_outcome(*, values: dict[str, object]) -> object: @@ -46,7 +37,17 @@ def validation_outcome(*, values: dict[str, object]) -> object: run_id="20261008T000000Z_orders", microbatch_concurrency=False, ), - names=_NAMES, + validator=_native.NativeModelValidator( + ( + {"orders", "customers"}, + {"regions"}, + {"raw.orders"}, + {"cents", "order_lines"}, + {"order_lines"}, + ), + {"ledger"}, + False, + ), ) request: ModelValidationRequest = ModelValidationRequest( model_file=DiscoveredSqlModelFile( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 8284c495c4..7af83aa85b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -11,7 +11,6 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, - ModelResourceNames, NativeModelConfigInputs, NativeModelConfigSession, ) @@ -115,13 +114,8 @@ def build_config_natively( # noqa: PLR0913 run_id="run_123", microbatch_concurrency=False, ), - names=ModelResourceNames( - models={"orders"}, - seeds=set(), - sources=set(), - functions=set(), - table_functions=set(), - custom_materializations=frozenset(), + validator=_native.NativeModelValidator( + ({"orders"}, set(), set(), set(), set()), set(), False ), ) return build_native_model_config( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py index e8381aa308..91845ccf44 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py @@ -4,9 +4,9 @@ import pytest -from sqlbuild.compiler.compile._helpers.attachment.model_config import find_matching_path_default from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.path_defaults.main._select import select_path_default from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( FindMatchingPathDefaultErrorTestCase, FindMatchingPathDefaultTestCase, @@ -86,10 +86,9 @@ def test_given_windows_style_model_path_when_finding_path_default_then_returns_n query_sql="select 1", ) - path_default: str | None = find_matching_path_default( - model_file=model_file, - path_defaults=test_case.path_defaults, - ) + path_default: str | None = select_path_default( + model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) + ).selected_key assert path_default == test_case.expected_path_default @@ -126,9 +125,8 @@ def test_given_ambiguous_globs_when_finding_path_default_then_raises_structured_ ) with pytest.raises(DiscoveryConflictError, match=test_case.expected_error_fragment) as error: - find_matching_path_default( - model_file=model_file, - path_defaults=test_case.path_defaults, + select_path_default( + model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) ) assert error.value.code == "D007" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index 5c38d3d855..a7a4ad428d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,8 +4,8 @@ import pytest -from sqlbuild.compiler.compile._helpers.analysis.required_columns import ( - resolve_required_external_columns, +from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( + analyze_resolved_column_reads, ) from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference from sqlbuild.compiler.compile.types import CompiledResourceType @@ -222,7 +222,7 @@ def test_given_query_when_resolving_required_columns_then_returns_external_reads( test_case: RequiredExternalColumnsTestCase, ) -> None: - result: tuple[CompiledLineageSourceFact, ...] = resolve_required_external_columns( + result: tuple[CompiledLineageSourceFact, ...] = analyze_resolved_column_reads( query_sql=test_case.query_sql, references=test_case.references, dialect="duckdb", @@ -261,7 +261,7 @@ def test_given_long_union_chain_when_resolving_required_columns_then_finishes_qu ) started: float = time.perf_counter() - result: tuple[CompiledLineageSourceFact, ...] = resolve_required_external_columns( + result: tuple[CompiledLineageSourceFact, ...] = analyze_resolved_column_reads( query_sql=f"WITH combined AS ({branches}) SELECT order_id, amount FROM combined", references=(CompileSqlReference(SqlReferenceKind.SOURCE, "orders"),), dialect="duckdb", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index ef4d614ef6..39898fea7e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -8,7 +8,7 @@ from sqlbuild.compiler.compile._helpers.render.sql_vars import ( interpolate_sql_batch, - substitute_sql_vars, + substitute_sql_vars_with_spans, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import SqlInterpolation @@ -130,7 +130,8 @@ def test_given_sql_and_vars_when_substituting_then_returns_expected( for target_name, environment_value in test_case.environment_variables.items(): monkeypatch.setenv(target_name, environment_value) - result: str = substitute_sql_vars( + result: str + result, _spans = substitute_sql_vars_with_spans( sql=test_case.sql, file_path=_FILE_PATH, effective_vars=test_case.effective_vars, @@ -214,7 +215,7 @@ def test_given_missing_var_when_substituting_then_raises( test_case: SubstituteSqlVarsErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - substitute_sql_vars( + substitute_sql_vars_with_spans( sql=test_case.sql, file_path=_FILE_PATH, effective_vars=test_case.effective_vars,