diff --git a/Cargo.lock b/Cargo.lock index c87be16302..9686b7a315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,6 +665,7 @@ name = "sqlbuild-discovery" version = "0.138.0" dependencies = [ "rayon", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-sqltext", "tempfile", diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs index 7d04aaf9d9..fee4e4b92e 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -7,8 +7,8 @@ use crate::yaml::_helpers::anchors::{ use crate::yaml::_helpers::block_scalars::{column_of, scan_block_scalar}; use crate::yaml::_helpers::resolver::resolve_scalar; use crate::yaml::constants::{ - FLOW_KEY_INDICATOR, MAP_TAG, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG, - PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, + FLOW_KEY_INDICATOR, MAP_TAG, MAX_NATIVE_IMPLICIT_KEY_CHARACTERS, MAX_NESTING_DEPTH, + NON_SPECIFIC_TAG, PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, }; use crate::yaml::models::{ComposedDocument, Node, NodeContent}; use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag}; @@ -142,8 +142,8 @@ impl<'input> Composer<'input> { self.chars.get(index).copied() } - /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. - fn spans_lines(&self, span: Span) -> bool { + /// An implicit key's extent from its first property to its `:` indicator; `None` if explicit. + fn implicit_key_extent(&self, span: Span) -> Option<(usize, usize)> { let key_end = span.end.index().min(self.chars.len()); let indicator = (key_end..self.chars.len()) .find(|index| !matches!(self.chars[*index], ' ' | '\t' | '\n' | '\r')) @@ -166,10 +166,22 @@ impl<'input> Composer<'input> { self.char_at(start + 1), None | Some(' ' | '\t' | '\n' | '\r') ); - !explicit - && self.chars[start.min(end)..end] + (!explicit).then_some((start.min(end), end)) + } + + /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. + fn spans_lines(&self, span: Span) -> bool { + self.implicit_key_extent(span).is_some_and(|(start, end)| { + self.chars[start..end] .iter() .any(|character| matches!(character, '\n' | '\r')) + }) + } + + /// Whether an implicit key nears PyYAML's 1024-character simple-key limit. + fn is_long_implicit_key(&self, span: Span) -> bool { + self.implicit_key_extent(span) + .is_some_and(|(start, end)| end - start > MAX_NATIVE_IMPLICIT_KEY_CHARACTERS) } /// Whether the collection starting at `span` is written in flow style. @@ -327,6 +339,11 @@ impl<'input> Composer<'input> { "a plain scalar in a flow collection cannot start with ':'", )); } + if placement.key && self.is_long_implicit_key(span) { + return Err(unsupported( + "implicit keys near PyYAML's 1024-character limit are left to Python", + )); + } if placement.key && placement.flow && self.spans_lines(span) { return Err(ConfigError::new( ConfigErrorKind::Syntax, diff --git a/crates/sqlbuild-config/src/yaml/constants.rs b/crates/sqlbuild-config/src/yaml/constants.rs index 2a5b16581e..bd4f2ec34a 100644 --- a/crates/sqlbuild-config/src/yaml/constants.rs +++ b/crates/sqlbuild-config/src/yaml/constants.rs @@ -60,3 +60,5 @@ pub(crate) const STRIP_INDICATOR: char = '-'; pub(crate) const SEQUENCE_ENTRY_TOKEN: &str = "-"; pub(crate) const DIRECTIVE_INDICATOR: char = '%'; pub(crate) const PLAIN_FORBIDDEN_FIRST_CHARACTERS: [char; 5] = ['|', '>', '%', '@', '`']; +/// Longer implicit keys are left to Python, which rejects simple keys beyond 1024 characters. +pub(crate) const MAX_NATIVE_IMPLICIT_KEY_CHARACTERS: usize = 1000; diff --git a/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs b/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs new file mode 100644 index 0000000000..29c84702d6 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs @@ -0,0 +1,34 @@ +//! Line helpers behind the `inspect.cleandoc` port. + +const TAB_SIZE: usize = 8; + +/// `str.expandtabs()`: a tab advances to the next multiple of eight columns. +pub(crate) fn expand_tabs(text: &str) -> String { + let mut expanded: String = String::with_capacity(text.len()); + let mut column: usize = 0; + for character in text.chars() { + match character { + '\t' => { + let width = TAB_SIZE - column % TAB_SIZE; + expanded.extend(std::iter::repeat_n(' ', width)); + column += width; + } + '\n' | '\r' => { + expanded.push(character); + column = 0; + } + _ => { + expanded.push(character); + column += 1; + } + } + } + expanded +} + +/// `line[count:]` in code points. +pub(crate) fn skip_chars(line: &str, count: usize) -> &str { + line.char_indices() + .nth(count) + .map_or("", |(index, _)| &line[index..]) +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 54370118d3..579d227f76 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod alnum_ranges; +pub(crate) mod cleandoc; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index c78964b1c4..1c4f886f29 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -4,4 +4,5 @@ pub mod is_python_alnum; pub mod is_python_space; pub mod is_python_word; pub mod python_alnum_unicode_version; +pub mod python_cleandoc; pub mod python_strip; diff --git a/crates/sqlbuild-core/src/text/main/python_cleandoc.rs b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs new file mode 100644 index 0000000000..0619ba36cb --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs @@ -0,0 +1,34 @@ +//! Python's `inspect.cleandoc` (3.12), which SQL test and scenario bodies go through. + +use crate::text::_helpers::cleandoc::{expand_tabs, skip_chars}; +use crate::text::main::is_python_space::is_python_space; + +/// `inspect.cleandoc(text)`: expand tabs, dedent later lines and drop outer empty lines. +pub fn python_cleandoc(text: &str) -> String { + let expanded: String = expand_tabs(text); + let mut lines: Vec<&str> = expanded.split('\n').collect(); + let margin: Option = lines[1..] + .iter() + .filter_map(|line| { + let content = line.trim_start_matches(is_python_space); + (!content.is_empty()).then(|| line.chars().count() - content.chars().count()) + }) + .min(); + lines[0] = lines[0].trim_start_matches(is_python_space); + let mut cleaned: Vec<&str> = Vec::with_capacity(lines.len()); + cleaned.push(lines[0]); + cleaned.extend( + lines[1..] + .iter() + .map(|line| margin.map_or(*line, |margin| skip_chars(line, margin))), + ); + let last: usize = cleaned + .iter() + .rposition(|line| !line.is_empty()) + .map_or(0, |index| index + 1); + let first: usize = cleaned[..last] + .iter() + .position(|line| !line.is_empty()) + .unwrap_or(last); + cleaned[first..last].join("\n") +} diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 44344eb15b..cb2d6a4aed 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs similarity index 98% rename from crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs rename to crates/sqlbuild-discovery/src/_helpers/header_keys.rs index 4623434102..93eefd20f1 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs +++ b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs @@ -7,6 +7,7 @@ use sqlbuild_core::text::main::is_python_word::is_python_word; /// The facts of one header that declares keys outside its supported set. pub(crate) struct UnsupportedKeys<'a> { + pub(crate) kind: FailureKind, pub(crate) statement: &'a str, pub(crate) file_path: &'a str, pub(crate) header: &'a str, @@ -18,6 +19,7 @@ pub(crate) struct UnsupportedKeys<'a> { pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> DiscoveryFailure { let UnsupportedKeys { + kind, statement, file_path, header, @@ -33,7 +35,7 @@ pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> Discovery .filter(|character| **character == '\n') .count(); DiscoveryFailure { - kind: FailureKind::ModelSql, + kind: *kind, message: format!( "{statement} in '{file_path}:{line}' has unsupported keys: {}", keys.join(", ") diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs index d1c99c888e..d7ec3c7a5c 100644 --- a/crates/sqlbuild-discovery/src/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -1,4 +1,6 @@ +pub(crate) mod header_keys; pub(crate) mod pool; pub(crate) mod reading; pub(crate) mod scoped_paths; +pub(crate) mod statement_headers; pub(crate) mod suggestions; diff --git a/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs new file mode 100644 index 0000000000..e19619d021 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs @@ -0,0 +1,66 @@ +//! Parse a statement header and reject its unsupported keys, as Python's discovery does. + +use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// One statement kind's header contract. +pub(crate) struct StatementHeader<'a> { + pub(crate) kind: FailureKind, + /// The name in syntax errors, such as `TEST`. + pub(crate) statement_name: &'a str, + /// The name in unsupported-key errors, such as `TEST()`. + pub(crate) statement: &'a str, + pub(crate) file_path: &'a str, + pub(crate) supported_keys: &'a [String], +} + +/// `parse_header_values` then `reject_unsupported_header_keys`; `header_line` is one-based. +pub(crate) fn parse_statement_header( + contract: &StatementHeader<'_>, + header: &str, + header_line: usize, +) -> Result, DiscoveryFailure> { + let values: Vec<(String, AuthoredValue)> = match parse_one(header) { + (_, _, Some(error)) => return Err(syntax_failure(contract, &error)), + (Some(AuthoredValue::Map(values)), _, None) => values, + _ => { + return Err(syntax_failure( + contract, + "Native MODEL header parser returned neither values nor an error", + )); + } + }; + let unsupported: Vec<&str> = values + .iter() + .map(|(key, _)| key.as_str()) + .filter(|key| !is_supported(contract.supported_keys, key)) + .collect(); + if unsupported.is_empty() { + return Ok(values); + } + Err(unsupported_keys_failure(&UnsupportedKeys { + kind: contract.kind, + statement: contract.statement, + file_path: contract.file_path, + header, + header_line, + keys: &unsupported, + supported_keys: contract.supported_keys, + })) +} + +fn syntax_failure(contract: &StatementHeader<'_>, error: &str) -> DiscoveryFailure { + DiscoveryFailure::new( + contract.kind, + format!( + "{}(...) in '{}' contains invalid SQLBuild header syntax: {error}", + contract.statement_name, contract.file_path + ), + ) +} + +fn is_supported(supported_keys: &[String], key: &str) -> bool { + supported_keys.iter().any(|supported| supported == key) +} diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index b4828a179a..32730b15f6 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -25,5 +25,29 @@ pub const CANONICAL_AUTHORED_ROOTS: [&[&str]; 5] = [ ]; /// The SQL test root, whose `macros/` SQL files are tests rather than declarations. pub const SQL_TESTS_ROOT: &[&str] = &["tests", "unit"]; +/// The SQL scenario root. +pub const SQL_SCENARIOS_ROOT: &str = "tests/scenarios"; /// The directory of macro tests below the SQL test root. pub const MACRO_TESTS_DIRECTORY: &str = "macros"; +/// Inherited declaration directories, which may also be project-wide global roots. +pub const GLOBAL_DECLARATION_DIRECTORIES: [&str; 3] = ["macros", "enums", "constants"]; +/// Folder-local declaration directories. +pub const LOCAL_DECLARATION_DIRECTORIES: [&str; 3] = ["_macros", "_enums", "_constants"]; +/// Directories a declaration group may hold for named declarations. +pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [ + "audits", "_audits", "schemas", "_schemas", "hooks", "_hooks", +]; +/// Project-wide named declaration roles. +pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"]; +/// Root of seed files, walked so the Python snapshot shares its listings. +pub const SEEDS_DIRECTORY: &str = "seeds"; +/// Suffix of macro files. +pub const PYTHON_FILE_SUFFIX: &str = ".py"; +/// The stem of package initialisers, which are not macro files. +pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; +/// The project-wide audit role directory. +pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; +/// The folder-local audit role directory inside a declaration group. +pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; +/// The singular audit directory below an audit role. +pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs new file mode 100644 index 0000000000..7172da607a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs @@ -0,0 +1,82 @@ +//! Python's `_validate_declaration_groups` and `_declaration_groups`. + +use crate::constants::{ + CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GROUPED_NAMED_DECLARATION_DIRECTORIES, + SCOPED_DECLARATION_DIRECTORIES, +}; +use crate::declarations::_helpers::paths::parent; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::DeclarationGroup; +use crate::models::StageDeferral; +use crate::tree::main::children::children; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// Every `_sqlbuild/` directory below a canonical root, root by root, in path order. +fn all_groups(tree: &ProjectTree) -> Result, StageDeferral> { + let mut groups: Vec = Vec::new(); + for root_parts in CANONICAL_AUTHORED_ROOTS { + let root: String = root_parts.join("/"); + if !tree.is_dir(&root) { + continue; + } + for directory in rglob(tree, &root, |entry| { + entry.name == DECLARATION_GROUP_DIRECTORY + })? { + if tree.is_dir(&directory) { + groups.push(DeclarationGroup { + root: root.clone(), + directory, + }); + } + } + } + Ok(groups) +} + +/// Python's `_declaration_groups`: the groups below a concrete owner directory. +pub(crate) fn owned_groups(tree: &ProjectTree) -> Result, StageDeferral> { + Ok(all_groups(tree)? + .into_iter() + .filter(|group| parent(&group.directory) != group.root) + .collect()) +} + +pub(crate) fn validate_declaration_groups(tree: &ProjectTree) -> Result<(), ScanError> { + if tree.exists(DECLARATION_GROUP_DIRECTORY) { + return Err(declaration_failure(format!( + "Grouped declaration root {DECLARATION_GROUP_DIRECTORY}/ must be below a canonical \ + authored root" + ))); + } + for group in all_groups(tree)? { + if parent(&group.directory) == group.root { + return Err(declaration_failure(format!( + "Grouped declaration root {}/ must be below a concrete owner directory; use the \ + project-wide macros/, enums/, constants/, audits/, schemas/, or hooks/ root \ + instead", + group.directory + ))); + } + let unsupported: Vec = children(tree, &group.directory)? + .into_iter() + .filter(|(_, entry)| !is_group_role(entry)) + .map(|(path, _)| path) + .collect(); + if !unsupported.is_empty() { + return Err(declaration_failure(format!( + "Declaration group {}/ contains unsupported entries: {}", + group.directory, + unsupported.join(", ") + ))); + } + } + Ok(()) +} + +fn is_group_role(entry: &TreeEntry) -> bool { + entry.is_dir + && (SCOPED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str()) + || GROUPED_NAMED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str())) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs new file mode 100644 index 0000000000..8f99a20d08 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -0,0 +1,138 @@ +//! Python's `_scan_declaration_file_facts`: macro, enum and constant files with scope facts. + +use crate::constants::{ + CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, + LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, PYTHON_INIT_MODULE_STEM, + SCOPED_DECLARATION_DIRECTORIES, SQL_FILE_SUFFIX, +}; +use crate::declarations::_helpers::declaration_groups::validate_declaration_groups; +use crate::declarations::_helpers::paths::{directory_facts, name, parent, stem}; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::{DeclarationFileFact, DeclarationKind, ScopeKind}; +use crate::tree::main::directories::directories; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; + +/// Where a declaration root sits and who owns it. +struct DeclarationRoot<'a> { + ownership_root: &'a str, + directory: &'a str, + owning_path: Option<&'a str>, + scope_kind: ScopeKind, +} + +/// Every declaration file below every macro, enum and constant root, by relative path. +pub(crate) fn declaration_file_facts( + tree: &ProjectTree, +) -> Result, ScanError> { + validate_declaration_groups(tree)?; + let mut local: Vec<&str> = LOCAL_DECLARATION_DIRECTORIES.to_vec(); + local.sort_unstable(); + if let Some(directory) = local.into_iter().find(|directory| tree.is_dir(directory)) { + return Err(declaration_failure(format!( + "Scoped declaration root {directory}/ must be below a canonical authored root" + ))); + } + let mut facts: Vec = Vec::new(); + let mut global: Vec<&str> = GLOBAL_DECLARATION_DIRECTORIES.to_vec(); + global.sort_unstable(); + for directory in global { + if tree.is_dir(directory) { + facts.extend(files_under_root( + tree, + &DeclarationRoot { + ownership_root: directory, + directory, + owning_path: None, + scope_kind: ScopeKind::Global, + }, + )?); + } + } + for root_parts in CANONICAL_AUTHORED_ROOTS { + let root: String = root_parts.join("/"); + if tree.is_dir(&root) { + facts.extend(scoped_root_files(tree, &root, root_parts.len())?); + } + } + facts.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + Ok(facts) +} + +fn scoped_root_files( + tree: &ProjectTree, + root: &str, + root_length: usize, +) -> Result, ScanError> { + let mut facts: Vec = Vec::new(); + for directory in directories(tree, root)? { + let Some((_, scope_kind)) = directory_facts(name(&directory)) else { + continue; + }; + let parts: Vec<&str> = directory.split('/').collect(); + let between: &[&str] = &parts[root_length..parts.len() - 1]; + if between + .iter() + .any(|part| SCOPED_DECLARATION_DIRECTORIES.contains(part)) + { + return Err(nested_root_failure(&directory)); + } + let mut owning_path: &str = parent(&directory); + if name(owning_path) == DECLARATION_GROUP_DIRECTORY { + owning_path = parent(owning_path); + } + facts.extend(files_under_root( + tree, + &DeclarationRoot { + ownership_root: root, + directory: &directory, + owning_path: Some(owning_path), + scope_kind, + }, + )?); + } + Ok(facts) +} + +fn nested_root_failure(directory: &str) -> ScanError { + declaration_failure(format!( + "Declaration root {directory}/ is nested inside another declaration tree" + )) +} + +fn files_under_root( + tree: &ProjectTree, + root: &DeclarationRoot<'_>, +) -> Result, ScanError> { + if let Some(nested) = directories(tree, root.directory)? + .into_iter() + .find(|directory| SCOPED_DECLARATION_DIRECTORIES.contains(&name(directory))) + { + return Err(nested_root_failure(&nested)); + } + let Some((kind, _)) = directory_facts(name(root.directory)) else { + return Ok(Vec::new()); + }; + let suffix: &str = if kind == DeclarationKind::Macro { + PYTHON_FILE_SUFFIX + } else { + SQL_FILE_SUFFIX + }; + Ok( + rglob(tree, root.directory, |entry| entry.name.ends_with(suffix))? + .into_iter() + .filter(|path| { + kind != DeclarationKind::Macro || stem(name(path)) != PYTHON_INIT_MODULE_STEM + }) + .map(|relative_path| DeclarationFileFact { + relative_path, + kind, + scope_kind: root.scope_kind, + ownership_root: root.ownership_root.to_owned(), + owning_path: root.owning_path.map(str::to_owned), + declaration_root: root.directory.to_owned(), + }) + .collect(), + ) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..edd0650597 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod declaration_groups; +pub(crate) mod declaration_tree; +pub(crate) mod named_layout; +pub(crate) mod paths; +pub(crate) mod scan; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs new file mode 100644 index 0000000000..073d9d6dd8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs @@ -0,0 +1,129 @@ +//! Python's `validate_named_declaration_layout`. + +use crate::constants::{ + AUDIT_ROLE_DIRECTORY, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, + GLOBAL_NAMED_DECLARATION_DIRECTORIES, LOCAL_AUDIT_ROLE_DIRECTORY, + LOCAL_DECLARATION_DIRECTORIES, SINGULAR_AUDIT_DIRECTORY, +}; +use crate::declarations::_helpers::declaration_groups::owned_groups; +use crate::declarations::_helpers::paths::{join, name}; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::DeclarationGroup; +use crate::tree::main::children::children; +use crate::tree::main::directories::directories; +use crate::tree::models::ProjectTree; + +const AUDIT_ROLES: &[&str] = &["generic", SINGULAR_AUDIT_DIRECTORY]; +const LOCAL_AUDIT_ROLES: &[&str] = &["generic"]; +const HOOK_ROLES: &[&str] = &["python", "sql"]; + +/// One role directory and the child directories it accepts, in sorted order. +struct RoleChildren<'a> { + directory: &'a str, + allowed: &'a [&'a str], + help_text: &'a str, +} + +/// The declaration groups the named roles use, once every named role layout is valid. +pub(crate) fn named_declaration_groups( + tree: &ProjectTree, +) -> Result, ScanError> { + if tree.is_dir(AUDIT_ROLE_DIRECTORY) { + require_role_children( + tree, + &RoleChildren { + directory: AUDIT_ROLE_DIRECTORY, + allowed: AUDIT_ROLES, + help_text: "audit files must live in audits/generic/ or audits/singular/", + }, + )?; + } + let mut top_levels: Vec<&str> = GLOBAL_NAMED_DECLARATION_DIRECTORIES.to_vec(); + top_levels.sort_unstable(); + for top_level in top_levels { + if !tree.is_dir(top_level) { + continue; + } + if let Some(nested) = directories(tree, top_level)? + .into_iter() + .find(|directory| is_nested_role_directory(name(directory))) + { + return Err(declaration_failure(format!( + "Declaration directory {nested}/ is not allowed inside the project-wide \ + {top_level}/ role; scoped declarations belong under \ + /{DECLARATION_GROUP_DIRECTORY}/ below a resource tree" + ))); + } + } + let groups: Vec = owned_groups(tree)?; + for group in &groups { + validate_group_roles(tree, &group.directory)?; + } + Ok(groups) +} + +fn validate_group_roles(tree: &ProjectTree, group: &str) -> Result<(), ScanError> { + let roles: [(&str, &[&str]); 4] = [ + (AUDIT_ROLE_DIRECTORY, AUDIT_ROLES), + (LOCAL_AUDIT_ROLE_DIRECTORY, LOCAL_AUDIT_ROLES), + ("hooks", HOOK_ROLES), + ("_hooks", HOOK_ROLES), + ]; + for (role, allowed) in roles { + let directory: String = join(group, role); + if !tree.is_dir(&directory) { + continue; + } + let singular: String = join(&directory, SINGULAR_AUDIT_DIRECTORY); + if role == LOCAL_AUDIT_ROLE_DIRECTORY && tree.exists(&singular) { + return Err(declaration_failure(format!( + "{singular}/ is invalid: singular audits are never used by name, so folder-only \ + visibility has no meaning; use {DECLARATION_GROUP_DIRECTORY}/audits/singular/" + ))); + } + let help_text: String = format!( + "{role}/ accepts only {}", + allowed + .iter() + .map(|name| format!("{name}/")) + .collect::>() + .join(", ") + ); + require_role_children( + tree, + &RoleChildren { + directory: &directory, + allowed, + help_text: &help_text, + }, + )?; + } + Ok(()) +} + +fn require_role_children(tree: &ProjectTree, role: &RoleChildren<'_>) -> Result<(), ScanError> { + let unsupported: Vec = children(tree, role.directory)? + .into_iter() + .filter(|(_, entry)| { + !entry.name.starts_with('.') + && (!entry.is_dir || !role.allowed.contains(&entry.name.as_str())) + }) + .map(|(path, _)| path) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + Err(declaration_failure(format!( + "Unsupported entries in {}/: {}; {}", + role.directory, + unsupported.join(", "), + role.help_text + ))) +} + +fn is_nested_role_directory(directory_name: &str) -> bool { + directory_name == DECLARATION_GROUP_DIRECTORY + || GLOBAL_DECLARATION_DIRECTORIES.contains(&directory_name) + || LOCAL_DECLARATION_DIRECTORIES.contains(&directory_name) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs new file mode 100644 index 0000000000..b71022439c --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs @@ -0,0 +1,43 @@ +//! Relative path text helpers shared by declaration discovery. + +use crate::declarations::models::{DeclarationKind, ScopeKind}; + +/// The parent of a `/`-separated relative path (`""` at the top). +pub(crate) fn parent(relative_path: &str) -> &str { + relative_path + .rsplit_once('/') + .map_or("", |(parent, _)| parent) +} + +/// The final component of a `/`-separated relative path. +pub(crate) fn name(relative_path: &str) -> &str { + relative_path + .rsplit_once('/') + .map_or(relative_path, |(_, name)| name) +} + +/// A `/`-separated relative directory joined with one more component. +pub(crate) fn join(directory: &str, name: &str) -> String { + format!("{directory}/{name}") +} + +/// Python's `Path.stem` of a file name. +pub(crate) fn stem(file_name: &str) -> &str { + match file_name.rfind('.') { + Some(index) if index > 0 && index + 1 < file_name.len() => &file_name[..index], + _ => file_name, + } +} + +/// Python's `DECLARATION_DIRECTORY_FACTS` for a scoped declaration directory name. +pub(crate) fn directory_facts(directory_name: &str) -> Option<(DeclarationKind, ScopeKind)> { + match directory_name { + "macros" => Some((DeclarationKind::Macro, ScopeKind::Inherited)), + "enums" => Some((DeclarationKind::Enum, ScopeKind::Inherited)), + "constants" => Some((DeclarationKind::Constant, ScopeKind::Inherited)), + "_macros" => Some((DeclarationKind::Macro, ScopeKind::Local)), + "_enums" => Some((DeclarationKind::Enum, ScopeKind::Local)), + "_constants" => Some((DeclarationKind::Constant, ScopeKind::Local)), + _ => None, + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs new file mode 100644 index 0000000000..ce4fce93d0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs @@ -0,0 +1,20 @@ +//! Building and splitting declaration scan errors. + +use crate::declarations::errors::ScanError; +use crate::models::{DiscoveryFailure, FailureKind, StageDeferral}; + +/// A `DeclarationParseError` with Python's message. +pub(crate) fn declaration_failure(message: String) -> ScanError { + ScanError::Failure(DiscoveryFailure::new(FailureKind::Declaration, message)) +} + +/// Split a scan result into the authored outcome, keeping a deferral for the caller. +pub(crate) fn authored_outcome( + result: Result, +) -> Result, StageDeferral> { + match result { + Ok(value) => Ok(Ok(value)), + Err(ScanError::Failure(failure)) => Ok(Err(failure)), + Err(ScanError::Deferral(deferral)) => Err(deferral), + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/errors.rs b/crates/sqlbuild-discovery/src/declarations/errors.rs new file mode 100644 index 0000000000..88ae69bf1b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/errors.rs @@ -0,0 +1,14 @@ +//! Why a declaration scan stopped: an authored layout failure or a whole-stage deferral. + +use crate::models::{DiscoveryFailure, StageDeferral}; + +pub(crate) enum ScanError { + Failure(DiscoveryFailure), + Deferral(StageDeferral), +} + +impl From for ScanError { + fn from(deferral: StageDeferral) -> Self { + Self::Deferral(deferral) + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs new file mode 100644 index 0000000000..6b775310ad --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs @@ -0,0 +1,25 @@ +//! Walk and validate the declaration layout the Python declaration collections read. + +use crate::_helpers::pool::discovery_pool; +use crate::constants::SEEDS_DIRECTORY; +use crate::declarations::_helpers::declaration_tree::declaration_file_facts; +use crate::declarations::_helpers::named_layout::named_declaration_groups; +use crate::declarations::_helpers::scan::authored_outcome; +use crate::declarations::models::DeclarationLayout; +use crate::models::StageDeferral; +use crate::tree::main::read_tree::read_tree; +use crate::tree::models::ProjectTree; + +/// The declaration file facts and named declaration groups, each valid or failing as in Python. +pub fn declaration_layout(tree: &ProjectTree) -> Result { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + pool.install(|| { + let file_facts = authored_outcome(declaration_file_facts(tree))?; + let named_groups = authored_outcome(named_declaration_groups(tree))?; + read_tree(tree, SEEDS_DIRECTORY)?; + Ok(DeclarationLayout { + file_facts, + named_groups, + }) + }) +} diff --git a/crates/sqlbuild-discovery/src/declarations/main/mod.rs b/crates/sqlbuild-discovery/src/declarations/main/mod.rs new file mode 100644 index 0000000000..32fb1c5af9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/main/mod.rs @@ -0,0 +1 @@ +pub mod declaration_layout; diff --git a/crates/sqlbuild-discovery/src/declarations/mod.rs b/crates/sqlbuild-discovery/src/declarations/mod.rs new file mode 100644 index 0000000000..a2537f3e8e --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/declarations/models.rs b/crates/sqlbuild-discovery/src/declarations/models.rs new file mode 100644 index 0000000000..33cb14ccbe --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/models.rs @@ -0,0 +1,68 @@ +//! The declaration layout facts native discovery hands to the Python discovery helpers. + +use crate::models::DiscoveryFailure; + +/// The kinds a macro, enum or constant declaration root holds (Python's `DeclarationKind`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarationKind { + Macro, + Enum, + Constant, +} + +impl DeclarationKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Macro => "macro", + Self::Enum => "enum", + Self::Constant => "constant", + } + } +} + +/// Python's `ScopeKind` for a declaration root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScopeKind { + Global, + Inherited, + Local, +} + +impl ScopeKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Global => "global", + Self::Inherited => "inherited", + Self::Local => "local", + } + } +} + +/// Python's `_DeclarationFileFacts` with `/`-separated project-relative paths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationFileFact { + pub relative_path: String, + pub kind: DeclarationKind, + pub scope_kind: ScopeKind, + pub ownership_root: String, + pub owning_path: Option, + pub declaration_root: String, +} + +/// A `_sqlbuild/` declaration group below a concrete owner, with its canonical root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationGroup { + pub root: String, + pub directory: String, +} + +/// The validated declaration layout, or the first failure Python's own scan raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationLayout { + /// Every macro, enum and constant file, by relative path. + pub file_facts: Result, DiscoveryFailure>, + /// The declaration groups the named roles use, once the named layout is valid. + pub named_groups: Result, DiscoveryFailure>, +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests.rs b/crates/sqlbuild-discovery/src/declarations/tests.rs new file mode 100644 index 0000000000..ca1db44484 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_declaration_layout.rs"] +mod declaration_layout; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs new file mode 100644 index 0000000000..e39cfcf614 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs @@ -0,0 +1,85 @@ +use crate::declarations::main::declaration_layout::declaration_layout; +use crate::declarations::models::{DeclarationFileFact, DeclarationGroup, DeclarationLayout}; +use crate::declarations::tests::test_types::{FactRow, LayoutTestCase}; +use crate::models::DiscoveryFailure; +use crate::tree::models::ProjectTree; +use std::fs; + +type Outcome = Result, String>; + +/// The case's layout as owned fact rows and group pairs, or the failure messages. +pub(super) fn layout_rows( + test_case: &LayoutTestCase, +) -> (Outcome, Outcome<(String, String)>) { + let project = tempfile::tempdir().expect("temporary project"); + for relative_path in test_case.files { + let path = project.path().join(relative_path); + fs::create_dir_all(path.parent().expect("parent directory")).expect("directories"); + fs::write(path, "").expect("file"); + } + for relative_path in test_case.directories { + fs::create_dir_all(project.path().join(relative_path)).expect("directory"); + } + let layout: DeclarationLayout = + declaration_layout(&ProjectTree::new(project.path())).expect("no deferral"); + ( + rows(layout.file_facts, fact_row), + rows(layout.named_groups, group_pair), + ) +} + +/// Expected fact rows in the owned shape `layout_rows` returns. +pub(super) fn owned_facts( + expected: Result<&'static [FactRow], &'static str>, +) -> Outcome { + expected + .map(|facts| facts.iter().map(owned_fact).collect()) + .map_err(str::to_owned) +} + +/// Expected group pairs in the owned shape `layout_rows` returns. +pub(super) fn owned_groups( + expected: Result<&'static [(&'static str, &'static str)], &'static str>, +) -> Outcome<(String, String)> { + expected + .map(|groups| groups.iter().map(owned_group).collect()) + .map_err(str::to_owned) +} + +fn owned_group(group: &(&str, &str)) -> (String, String) { + (group.0.to_owned(), group.1.to_owned()) +} + +pub(super) type OwnedFactRow = (String, String, String, String, Option, String); + +fn rows(outcome: Result, DiscoveryFailure>, row: fn(T) -> R) -> Outcome { + outcome + .map(|items| items.into_iter().map(row).collect()) + .map_err(|failure| failure.message) +} + +fn fact_row(fact: DeclarationFileFact) -> OwnedFactRow { + ( + fact.relative_path, + fact.kind.as_str().to_owned(), + fact.scope_kind.as_str().to_owned(), + fact.ownership_root, + fact.owning_path, + fact.declaration_root, + ) +} + +fn owned_fact(fact: &FactRow) -> OwnedFactRow { + ( + fact.0.to_owned(), + fact.1.to_owned(), + fact.2.to_owned(), + fact.3.to_owned(), + fact.4.map(str::to_owned), + fact.5.to_owned(), + ) +} + +fn group_pair(group: DeclarationGroup) -> (String, String) { + (group.root, group.directory) +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs new file mode 100644 index 0000000000..1cab16a64b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs @@ -0,0 +1,124 @@ +use crate::declarations::tests::helpers::{layout_rows, owned_facts, owned_groups}; +use crate::declarations::tests::test_types::LayoutTestCase; + +#[test] +fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python() { + let test_cases = [ + LayoutTestCase { + description: "global, inherited, grouped and local roots sort by relative path", + files: &[ + "macros/text.py", + "macros/__init__.py", + "enums/status.sql", + "models/marts/_enums/local.sql", + "models/marts/_sqlbuild/constants/limits.sql", + "models/marts/_sqlbuild/audits/generic/positive.sql", + ], + directories: &[], + expected_facts: Ok(&[ + ("enums/status.sql", "enum", "global", "enums", None, "enums"), + ( + "macros/text.py", + "macro", + "global", + "macros", + None, + "macros", + ), + ( + "models/marts/_enums/local.sql", + "enum", + "local", + "models", + Some("models/marts"), + "models/marts/_enums", + ), + ( + "models/marts/_sqlbuild/constants/limits.sql", + "constant", + "inherited", + "models", + Some("models/marts"), + "models/marts/_sqlbuild/constants", + ), + ]), + expected_groups: Ok(&[("models", "models/marts/_sqlbuild")]), + }, + LayoutTestCase { + description: "a declaration group directly below a canonical root fails", + files: &["models/_sqlbuild/macros/text.py"], + directories: &[], + expected_facts: Err( + "Grouped declaration root models/_sqlbuild/ must be below a concrete owner \ + directory; use the project-wide macros/, enums/, constants/, audits/, \ + schemas/, or hooks/ root instead", + ), + expected_groups: Ok(&[]), + }, + LayoutTestCase { + description: "a nested declaration root fails in both scans", + files: &["models/a/_enums/macros/x.py"], + directories: &["schemas/enums"], + expected_facts: Err( + "Declaration root models/a/_enums/macros/ is nested inside another declaration \ + tree", + ), + expected_groups: Err( + "Declaration directory schemas/enums/ is not allowed inside the project-wide \ + schemas/ role; scoped declarations belong under /_sqlbuild/ below a \ + resource tree", + ), + }, + LayoutTestCase { + description: "unsupported group and role entries list every entry", + files: &[ + "models/a/_sqlbuild/notes.txt", + "audits/x.sql", + "audits/.keep", + ], + directories: &["models/a/_sqlbuild/extra"], + expected_facts: Err( + "Declaration group models/a/_sqlbuild/ contains unsupported entries: \ + models/a/_sqlbuild/extra, models/a/_sqlbuild/notes.txt", + ), + expected_groups: Err( + "Unsupported entries in audits/: audits/x.sql; audit files must live in \ + audits/generic/ or audits/singular/", + ), + }, + LayoutTestCase { + description: "local singular audits and unknown hook roles fail the named layout", + files: &[], + directories: &[ + "models/a/_sqlbuild/_audits/singular", + "models/b/_sqlbuild/hooks/x", + ], + expected_facts: Ok(&[]), + expected_groups: Err( + "models/a/_sqlbuild/_audits/singular/ is invalid: singular audits are never used \ + by name, so folder-only visibility has no meaning; use \ + _sqlbuild/audits/singular/", + ), + }, + LayoutTestCase { + description: "a scoped root at the project root fails", + files: &["_macros/text.py"], + directories: &[], + expected_facts: Err( + "Scoped declaration root _macros/ must be below a canonical authored root", + ), + expected_groups: Ok(&[]), + }, + ]; + for test_case in test_cases { + assert_eq!( + layout_rows(&test_case), + ( + owned_facts(test_case.expected_facts), + owned_groups(test_case.expected_groups) + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs new file mode 100644 index 0000000000..458cbc4004 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs @@ -0,0 +1,18 @@ +pub(super) struct LayoutTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) directories: &'static [&'static str], + /// `(relative path, kind, scope kind, ownership root, owning path, declaration root)`. + pub(super) expected_facts: Result<&'static [FactRow], &'static str>, + /// `(canonical root, group directory)`. + pub(super) expected_groups: Result<&'static [(&'static str, &'static str)], &'static str>, +} + +pub(super) type FactRow = ( + &'static str, + &'static str, + &'static str, + &'static str, + Option<&'static str>, + &'static str, +); diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 11e9b34f54..1858da47b2 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -4,6 +4,9 @@ pub(crate) mod _helpers; pub mod constants; +pub mod declarations; pub mod model_files; pub mod models; +pub mod sql_tests; pub mod tree; +pub mod yaml_files; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index de1cf65fb0..c9e10edbd5 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod header_keys; pub(crate) mod locations; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index c23cd28716..c402ba4090 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -1,6 +1,6 @@ //! Parse one model file with the checks and messages of Python's `parse_matched_model_sql`. -use crate::model_files::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; +use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::model_files::_helpers::locations::header_column_locations; use crate::model_files::_helpers::output_columns::output_column_locations; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; @@ -54,6 +54,7 @@ pub(crate) fn parse_model_file( .collect(); if !unsupported.is_empty() { return Err(unsupported_keys_failure(&UnsupportedKeys { + kind: FailureKind::ModelSql, statement: MODEL_STATEMENT, file_path, header, diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 9a933a7b46..6e2ff16226 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -8,6 +8,12 @@ use std::path::PathBuf; pub enum FailureKind { /// `ModelSqlParseError` (D002). ModelSql, + /// `DeclarationParseError` (D013). + Declaration, + /// `SqlTestParseError` (D003). + SqlTest, + /// `SqlScenarioParseError` (D009). + SqlScenario, } impl FailureKind { @@ -15,6 +21,9 @@ impl FailureKind { pub fn as_str(self) -> &'static str { match self { Self::ModelSql => "model_sql", + Self::Declaration => "declaration", + Self::SqlTest => "sql_test", + Self::SqlScenario => "sql_scenario", } } } diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs new file mode 100644 index 0000000000..bee80b3bef --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod scenario_parsing; +pub(crate) mod selection; +pub(crate) mod test_blocks; diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs new file mode 100644 index 0000000000..428efa510c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs @@ -0,0 +1,45 @@ +//! The header half of Python's `parse_sql_scenario_file`. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::sql_tests::models::DiscoveredScenarioFile; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +const SCENARIO_KEYWORD: &str = "SCENARIO"; +const SCENARIO_STATEMENT: &str = "SCENARIO()"; + +pub(crate) fn parse_scenario_file( + file_path: &str, + contents: String, + supported_keys: &[String], +) -> Result { + let Some((header_start, header_end, sql_start)) = + match_statement_header(&contents, 0, SCENARIO_KEYWORD) + else { + return Err(DiscoveryFailure::new( + FailureKind::SqlScenario, + format!( + "SQL scenario '{file_path}' must start with a SCENARIO() header as the first \ + non-whitespace content" + ), + )); + }; + let header_values = parse_statement_header( + &StatementHeader { + kind: FailureKind::SqlScenario, + statement_name: SCENARIO_KEYWORD, + statement: SCENARIO_STATEMENT, + file_path, + supported_keys, + }, + &contents[header_start..header_end], + contents[..header_start].matches('\n').count() + 1, + )?; + let sql_body: String = python_cleandoc(&contents[sql_start..]); + Ok(DiscoveredScenarioFile { + contents, + header_values, + sql_body, + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs new file mode 100644 index 0000000000..6641b2950c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs @@ -0,0 +1,39 @@ +//! Read and parse the unscoped SQL files below one root in parallel, in Python's order. + +use crate::_helpers::pool::discovery_pool; +use crate::_helpers::reading::read_authored_text; +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::constants::SQL_FILE_SUFFIX; +use crate::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, ProjectRoot, StageDeferral}; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +/// Every unscoped `*.sql` file below `directory`, read and handed to `parse`. +pub(crate) fn discover_sql_files( + root: &ProjectRoot, + tree: &ProjectTree, + directory: &str, + parse: impl Fn(&str, String) -> Result + Sync, +) -> Result>, StageDeferral> { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + pool.install(|| { + let paths: Vec = rglob(tree, directory, |entry| { + entry.name.ends_with(SQL_FILE_SUFFIX) + })?; + Ok(paths + .into_par_iter() + .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .map(|relative_path| DiscoveredFile { + outcome: match read_authored_text(&tree.absolute(&relative_path)) { + Ok(contents) => match parse(&root.display_path(&relative_path), contents) { + Ok(parsed) => FileOutcome::Parsed(parsed), + Err(failure) => FileOutcome::Failed(failure), + }, + Err(_unreadable) => FileOutcome::Unreadable, + }, + relative_path, + }) + .collect()) + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs new file mode 100644 index 0000000000..28b285846c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs @@ -0,0 +1,114 @@ +//! Python's `_split_sql_test_blocks` and the header half of `_parse_single_sql_test_block`. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestBlock}; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +const TEST_KEYWORD: &str = "TEST"; +const TEST_STATEMENT: &str = "TEST()"; + +/// One `^\s*TEST\s*\(header\)\s*;\s*` match: start, header start, header end and match end. +struct BlockMatch { + start: usize, + header_start: usize, + header_end: usize, + end: usize, +} + +pub(crate) fn parse_sql_test_file( + file_path: &str, + contents: String, + supported_keys: &[String], +) -> Result { + let matches: Vec = block_matches(&contents); + let Some(first) = matches.first() else { + return Err(missing_header_failure(file_path)); + }; + if !python_strip(&contents[..first.start]).is_empty() { + return Err(missing_header_failure(file_path)); + } + let contract = StatementHeader { + kind: FailureKind::SqlTest, + statement_name: TEST_KEYWORD, + statement: TEST_STATEMENT, + file_path, + supported_keys, + }; + let mut blocks: Vec = Vec::with_capacity(matches.len()); + let mut failure: Option = None; + for (index, block) in matches.iter().enumerate() { + let next_start: usize = matches + .get(index + 1) + .map_or(contents.len(), |next| next.start); + let header: &str = &contents[block.header_start..block.header_end]; + let header_line: usize = contents[..block.header_start].matches('\n').count() + 1; + match parse_statement_header(&contract, header, header_line) { + Ok(header_values) => blocks.push(SqlTestBlock { + header_values, + sql_body: python_cleandoc( + contents[block.end.min(next_start)..next_start] + .trim_end_matches(is_python_space), + ), + }), + Err(block_failure) => { + failure = Some(block_failure); + break; + } + } + } + Ok(DiscoveredSqlTestFile { + contents, + blocks, + failure, + }) +} + +fn missing_header_failure(file_path: &str) -> DiscoveryFailure { + DiscoveryFailure::new( + FailureKind::SqlTest, + format!( + "SQL test '{file_path}' must start with a TEST() header as the first non-whitespace \ + content" + ), + ) +} + +/// `_TEST_HEADER_ONLY_PATTERN.finditer(contents)` (MULTILINE): attempts start at line starts. +fn block_matches(contents: &str) -> Vec { + let mut matches: Vec = Vec::new(); + let mut position: usize = 0; + while position <= contents.len() { + let candidate: usize = next_line_start(contents, position); + if candidate > contents.len() { + break; + } + match match_statement_header(contents, candidate, TEST_KEYWORD) { + Some((header_start, header_end, end)) => { + matches.push(BlockMatch { + start: candidate, + header_start, + header_end, + end, + }); + position = end; + } + None => position = candidate + 1, + } + } + matches +} + +/// The first position at or after `position` where MULTILINE `^` matches, or past the end. +fn next_line_start(contents: &str, position: usize) -> usize { + if position == 0 || contents.as_bytes().get(position - 1) == Some(&b'\n') { + return position; + } + contents.as_bytes()[position..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(contents.len() + 1, |offset| position + offset + 1) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs new file mode 100644 index 0000000000..967eb618f3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -0,0 +1,19 @@ +//! Discover, read and header-parse the SQL scenario files under `tests/scenarios/`. + +use crate::constants::SQL_SCENARIOS_ROOT; +use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; +use crate::sql_tests::_helpers::selection::discover_sql_files; +use crate::sql_tests::models::{DiscoveredScenarioFile, SqlTestFileOptions}; +use crate::tree::models::ProjectTree; + +/// Every scenario file in Python's order, header-parsed or failing. +pub fn discover_scenario_files( + root: &ProjectRoot, + tree: &ProjectTree, + options: &SqlTestFileOptions, +) -> Result>, StageDeferral> { + discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { + parse_scenario_file(file_path, contents, &options.scenario_keys) + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs new file mode 100644 index 0000000000..41c74c256c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -0,0 +1,22 @@ +//! Discover, read and split the SQL unit-test files under `tests/unit/`. + +use crate::constants::SQL_TESTS_ROOT; +use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::sql_tests::_helpers::selection::discover_sql_files; +use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; +use crate::tree::models::ProjectTree; + +/// Every SQL test file in Python's order, split into header-parsed blocks or failing. +pub fn discover_sql_test_files( + root: &ProjectRoot, + tree: &ProjectTree, + options: &SqlTestFileOptions, +) -> Result>, StageDeferral> { + discover_sql_files( + root, + tree, + &SQL_TESTS_ROOT.join("/"), + |file_path, contents| parse_sql_test_file(file_path, contents, &options.test_keys), + ) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs new file mode 100644 index 0000000000..f79f30314c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs @@ -0,0 +1,2 @@ +pub mod discover_scenario_files; +pub mod discover_sql_test_files; diff --git a/crates/sqlbuild-discovery/src/sql_tests/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/sql_tests/models.rs b/crates/sqlbuild-discovery/src/sql_tests/models.rs new file mode 100644 index 0000000000..9e04eae633 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/models.rs @@ -0,0 +1,35 @@ +//! Native results for SQL unit-test and scenario files. + +use crate::models::DiscoveryFailure; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// The keys each statement header supports, passed from Python's constants. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestFileOptions { + pub test_keys: Vec, + pub scenario_keys: Vec, +} + +/// One `TEST(...)` block whose header parsed with only supported keys. +#[derive(Clone, Debug, PartialEq)] +pub struct SqlTestBlock { + pub header_values: Vec<(String, AuthoredValue)>, + /// `inspect.cleandoc` of the SQL after the header. + pub sql_body: String, +} + +/// A test file split into blocks; parsing stops at the first block whose header fails. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredSqlTestFile { + pub contents: String, + pub blocks: Vec, + pub failure: Option, +} + +/// A scenario file whose header parsed with only supported keys. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredScenarioFile { + pub contents: String, + pub header_values: Vec<(String, AuthoredValue)>, + pub sql_body: String, +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests.rs b/crates/sqlbuild-discovery/src/sql_tests/tests.rs new file mode 100644 index 0000000000..158445fb36 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_sql_test_files.rs"] +mod sql_test_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs new file mode 100644 index 0000000000..1dd21e6c21 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -0,0 +1,78 @@ +use crate::models::DiscoveryFailure; +use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; +use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::models::SqlTestBlock; +use crate::sql_tests::tests::test_types::StatementFileTestCase; + +pub(super) type BlockRow = (Vec, String); +pub(super) const FILE_PATH: &str = "project/tests/unit/orders.sql"; + +fn keys(test_keys: &[&str]) -> Vec { + test_keys.iter().map(|key| (*key).to_owned()).collect() +} + +fn block_row(block: &SqlTestBlock) -> BlockRow { + ( + block + .header_values + .iter() + .map(|(key, _)| key.clone()) + .collect(), + block.sql_body.clone(), + ) +} + +fn message(failure: DiscoveryFailure) -> String { + failure.message +} + +fn failed_rows(failure: DiscoveryFailure) -> (Vec, Option) { + (Vec::new(), Some(failure.message)) +} + +/// The test file's parsed block rows and its failure message. +pub(super) fn test_file_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { + parse_sql_test_file( + FILE_PATH, + test_case.contents.to_owned(), + &keys(&["name", "mode", "parameters", "cases"]), + ) + .map_or_else(failed_rows, |file| { + ( + file.blocks.iter().map(block_row).collect(), + file.failure.map(message), + ) + }) +} + +/// The scenario file's block row and its failure message. +pub(super) fn scenario_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { + parse_scenario_file( + FILE_PATH, + test_case.contents.to_owned(), + &keys(&["description", "tags"]), + ) + .map_or_else(failed_rows, |file| { + ( + vec![block_row(&SqlTestBlock { + header_values: file.header_values, + sql_body: file.sql_body, + })], + None, + ) + }) +} + +/// The expected rows in the shape the parsers' rows take. +pub(super) fn expected_rows( + expected_blocks: &[(&[&str], &str)], + expected_failure: Option<&str>, +) -> (Vec, Option) { + ( + expected_blocks + .iter() + .map(|(block_keys, body)| (keys(block_keys), (*body).to_owned())) + .collect(), + expected_failure.map(str::to_owned), + ) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs new file mode 100644 index 0000000000..96e92835bd --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs @@ -0,0 +1,74 @@ +use crate::sql_tests::tests::helpers::{expected_rows, scenario_rows, test_file_rows}; +use crate::sql_tests::tests::test_types::StatementFileTestCase; + +#[test] +fn given_sql_test_files_when_splitting_then_blocks_and_failures_match_python() { + let test_cases = [ + StatementFileTestCase { + description: "blocks start at line starts and bodies are cleaned", + contents: "\n TEST (name \"a\");\n SELECT 1\n FROM t\n\n TEST(name \"b\") ;\n\tSELECT 2\n", + expected_blocks: &[(&["name"], "SELECT 1\nFROM t"), (&["name"], "SELECT 2")], + expected_failure: None, + }, + StatementFileTestCase { + description: "TEST not at a line start stays inside the previous body", + contents: "TEST (name \"a\"); SELECT 'TEST (x);'", + expected_blocks: &[(&["name"], "SELECT 'TEST (x);'")], + expected_failure: None, + }, + StatementFileTestCase { + description: "content before the first header fails the file", + contents: "-- lead\nTEST ();\nSELECT 1", + expected_blocks: &[], + expected_failure: Some( + "SQL test 'project/tests/unit/orders.sql' must start with a TEST() header as the \ + first non-whitespace content", + ), + }, + StatementFileTestCase { + description: "an unsupported key stops at its block, at the key's line", + contents: "TEST (name \"a\");\nSELECT 1\nTEST (\n name \"b\",\n nme 1\n);\nSELECT 2", + expected_blocks: &[(&["name"], "SELECT 1")], + expected_failure: Some( + "TEST() in 'project/tests/unit/orders.sql:5' has unsupported keys: nme", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + test_file_rows(&test_case), + expected_rows(test_case.expected_blocks, test_case.expected_failure), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenario_files_when_parsing_then_headers_and_failures_match_python() { + let test_cases = [ + StatementFileTestCase { + description: "the body keeps trailing indentation as cleandoc does", + contents: "SCENARIO (description \"Orders\");\n\n SELECT 1\n ", + expected_blocks: &[(&["description"], "SELECT 1\n ")], + expected_failure: None, + }, + StatementFileTestCase { + description: "a missing header fails", + contents: "SELECT 1", + expected_blocks: &[], + expected_failure: Some( + "SQL scenario 'project/tests/unit/orders.sql' must start with a SCENARIO() \ + header as the first non-whitespace content", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + scenario_rows(&test_case), + expected_rows(test_case.expected_blocks, test_case.expected_failure), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs new file mode 100644 index 0000000000..f45e83d054 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(super) struct StatementFileTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + /// Each parsed block's header keys and SQL body, then the failure message if one stopped it. + pub(super) expected_blocks: &'static [(&'static [&'static str], &'static str)], + pub(super) expected_failure: Option<&'static str>, +} diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs index 7fba2181b3..0aa7327497 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs @@ -8,14 +8,6 @@ use std::sync::Arc; pub(crate) type WalkedDirectory = (String, Arc>); -/// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. -pub(crate) fn is_dir(tree: &ProjectTree, relative_path: &str) -> bool { - match std::fs::metadata(tree.absolute(relative_path)) { - Ok(metadata) => metadata.is_dir(), - Err(_unreadable) => false, - } -} - /// Every directory below `directory`, with its listing, descending only into real directories. pub(crate) fn walk( tree: &ProjectTree, @@ -36,7 +28,10 @@ pub(crate) fn walk( Ok(walked) } -fn listing(tree: &ProjectTree, directory: &str) -> Result>, StageDeferral> { +pub(crate) fn listing( + tree: &ProjectTree, + directory: &str, +) -> Result>, StageDeferral> { if let Some(cached) = cached_listing(tree, directory) { return Ok(cached); } diff --git a/crates/sqlbuild-discovery/src/tree/main/children.rs b/crates/sqlbuild-discovery/src/tree/main/children.rs new file mode 100644 index 0000000000..f31bcf2130 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/children.rs @@ -0,0 +1,25 @@ +//! The entries of one directory, as `sorted(directory.iterdir())` sees them. + +use crate::models::StageDeferral; +use crate::tree::_helpers::listing::join_relative; +use crate::tree::_helpers::ordering::compare_relative_paths; +use crate::tree::_helpers::walking::listing; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// `sorted(directory.iterdir())` with relative paths; an unreadable directory defers. +pub fn children( + tree: &ProjectTree, + directory: &str, +) -> Result, StageDeferral> { + if let Err(error) = std::fs::read_dir(tree.absolute(directory)) { + return Err(StageDeferral { + reason: format!("cannot list {directory}: {error}"), + }); + } + let mut entries: Vec<(String, TreeEntry)> = listing(tree, directory)? + .iter() + .map(|entry| (join_relative(directory, &entry.name), entry.clone())) + .collect(); + entries.sort_by(|left, right| compare_relative_paths(&left.0, &right.0)); + Ok(entries) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/directories.rs b/crates/sqlbuild-discovery/src/tree/main/directories.rs new file mode 100644 index 0000000000..b9928dbe5c --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/directories.rs @@ -0,0 +1,10 @@ +//! `sorted(path for path in DirectorySnapshot.directories(root))` over the shared snapshot. + +use crate::models::StageDeferral; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; + +/// Every directory below `root`, including unwalked directory links, in Python's path order. +pub fn directories(tree: &ProjectTree, root: &str) -> Result, StageDeferral> { + rglob(tree, root, |entry| entry.is_dir) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs index cb7cc0dc9a..d45f892679 100644 --- a/crates/sqlbuild-discovery/src/tree/main/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -1,2 +1,5 @@ +pub mod children; +pub mod directories; pub mod listings; +pub mod read_tree; pub mod rglob; diff --git a/crates/sqlbuild-discovery/src/tree/main/read_tree.rs b/crates/sqlbuild-discovery/src/tree/main/read_tree.rs new file mode 100644 index 0000000000..00046ff610 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/read_tree.rs @@ -0,0 +1,13 @@ +//! Read every listing below a root into the shared snapshot. + +use crate::models::StageDeferral; +use crate::tree::_helpers::walking::walk; +use crate::tree::models::ProjectTree; + +/// List `root` and every real directory below it; a missing root reads nothing. +pub fn read_tree(tree: &ProjectTree, root: &str) -> Result<(), StageDeferral> { + if tree.is_dir(root) { + let _walked = walk(tree, root)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/rglob.rs b/crates/sqlbuild-discovery/src/tree/main/rglob.rs index 223cc056c1..ef9c44cec5 100644 --- a/crates/sqlbuild-discovery/src/tree/main/rglob.rs +++ b/crates/sqlbuild-discovery/src/tree/main/rglob.rs @@ -3,7 +3,7 @@ use crate::models::StageDeferral; use crate::tree::_helpers::listing::join_relative; use crate::tree::_helpers::ordering::compare_relative_paths; -use crate::tree::_helpers::walking::{WalkedDirectory, is_dir, walk}; +use crate::tree::_helpers::walking::{WalkedDirectory, walk}; use crate::tree::models::{ProjectTree, TreeEntry}; /// Every entry below `root` that `accept` keeps, in Python's sorted path order. @@ -12,7 +12,7 @@ pub fn rglob( root: &str, accept: impl Fn(&TreeEntry) -> bool, ) -> Result, StageDeferral> { - if !is_dir(tree, root) { + if !tree.is_dir(root) { return Ok(Vec::new()); } let mut selected: Vec = Vec::new(); diff --git a/crates/sqlbuild-discovery/src/tree/models.rs b/crates/sqlbuild-discovery/src/tree/models.rs index 96d24a7eca..93ebadc3a2 100644 --- a/crates/sqlbuild-discovery/src/tree/models.rs +++ b/crates/sqlbuild-discovery/src/tree/models.rs @@ -37,4 +37,20 @@ impl ProjectTree { .filter(|part| !part.is_empty()) .fold(self.directory.clone(), |path, part| path.join(part)) } + + /// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. + pub fn is_dir(&self, relative_path: &str) -> bool { + match std::fs::metadata(self.absolute(relative_path)) { + Ok(metadata) => metadata.is_dir(), + Err(_unreadable) => false, + } + } + + /// Python's `Path.exists()`: follows symbolic links and reports any failure as `false`. + pub fn exists(&self, relative_path: &str) -> bool { + match std::fs::metadata(self.absolute(relative_path)) { + Ok(_metadata) => true, + Err(_missing) => false, + } + } } diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs new file mode 100644 index 0000000000..85bd8a1291 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs @@ -0,0 +1,31 @@ +//! Read and load the given YAML files in parallel, deferring each file Python must load. + +use crate::_helpers::pool::discovery_pool; +use crate::_helpers::reading::read_authored_text; +use crate::models::StageDeferral; +use crate::tree::models::ProjectTree; +use crate::yaml_files::models::YamlFileOutcome; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use sqlbuild_config::yaml::main::safe_load::safe_load; + +/// Each file's outcome, in the order of `relative_paths`. +pub fn load_yaml_files( + tree: &ProjectTree, + relative_paths: &[String], +) -> Result, StageDeferral> { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + Ok(pool.install(|| { + relative_paths + .par_iter() + .map( + |relative_path| match read_authored_text(&tree.absolute(relative_path)) { + Ok(contents) => match safe_load(&contents) { + Ok(value) => YamlFileOutcome::Loaded { contents, value }, + Err(_deferred) => YamlFileOutcome::LoadInPython { contents }, + }, + Err(_unreadable) => YamlFileOutcome::Unreadable, + }, + ) + .collect() + })) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs new file mode 100644 index 0000000000..d96b3a0c0b --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -0,0 +1 @@ +pub mod load_yaml_files; diff --git a/crates/sqlbuild-discovery/src/yaml_files/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/mod.rs new file mode 100644 index 0000000000..1a70a48981 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/mod.rs @@ -0,0 +1,5 @@ +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/yaml_files/models.rs b/crates/sqlbuild-discovery/src/yaml_files/models.rs new file mode 100644 index 0000000000..7aa974d0d5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/models.rs @@ -0,0 +1,16 @@ +//! Native outcomes of reading and loading authored YAML files. + +use sqlbuild_config::models::ConfigValue; + +/// One YAML file read and loaded as PyYAML `safe_load` would, or what Python must redo. +#[derive(Clone, Debug, PartialEq)] +pub enum YamlFileOutcome { + Loaded { + contents: String, + value: ConfigValue, + }, + /// Read, but the native loader cannot reproduce PyYAML here; Python loads the contents. + LoadInPython { contents: String }, + /// The bytes could not be read or decoded; Python re-reads the file to raise its error. + Unreadable, +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests.rs b/crates/sqlbuild-discovery/src/yaml_files/tests.rs new file mode 100644 index 0000000000..929b7b00fa --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_yaml_files.rs"] +mod load_yaml_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs new file mode 100644 index 0000000000..28a1ca493a --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs @@ -0,0 +1,8 @@ +use crate::yaml_files::models::YamlFileOutcome; + +/// An expected "Python loads the contents" outcome; only its variant is compared. +pub(super) fn python() -> YamlFileOutcome { + YamlFileOutcome::LoadInPython { + contents: String::new(), + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs new file mode 100644 index 0000000000..9f019458a0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs @@ -0,0 +1,49 @@ +use crate::tree::models::ProjectTree; +use crate::yaml_files::main::load_yaml_files::load_yaml_files; +use crate::yaml_files::models::YamlFileOutcome; +use crate::yaml_files::tests::test_types::YamlFileTestCase; + +use crate::yaml_files::tests::helpers::python; +use sqlbuild_config::models::ConfigValue; +use std::mem::discriminant; + +#[test] +fn given_yaml_files_when_loading_then_each_file_loads_or_defers() { + let test_cases = [ + YamlFileTestCase { + description: "a plain mapping loads natively", + contents: b"sources:\n - name: orders\n", + expected_outcome: YamlFileOutcome::Loaded { + contents: String::new(), + value: ConfigValue::Null, + }, + }, + YamlFileTestCase { + description: "invalid YAML is left to Python for its exact error", + contents: b"sources: [\n", + expected_outcome: python(), + }, + YamlFileTestCase { + description: "a Python-only tag is left to Python", + contents: b"value: !!set {a: null}\n", + expected_outcome: python(), + }, + YamlFileTestCase { + description: "undecodable bytes are re-read by Python", + contents: b"name: \xff\n", + expected_outcome: YamlFileOutcome::Unreadable, + }, + ]; + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + std::fs::write(project.path().join("file.yml"), test_case.contents).expect("file"); + let outcomes = load_yaml_files(&ProjectTree::new(project.path()), &["file.yml".to_owned()]) + .expect("no deferral"); + assert_eq!( + outcomes.iter().map(discriminant).collect::>(), + vec![discriminant(&test_case.expected_outcome)], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs new file mode 100644 index 0000000000..99eba3e5a5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::yaml_files::models::YamlFileOutcome; + +pub(super) struct YamlFileTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static [u8], + /// Only the variant is compared. + pub(super) expected_outcome: YamlFileOutcome, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs new file mode 100644 index 0000000000..140497c9f4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs @@ -0,0 +1,108 @@ +//! Loaded configuration values as the Python objects PyYAML and `tomllib` build. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +type PyObject = Py; + +/// Decimal digits per `int(str)` call, well below Python's default 4300-digit limit. +const BIG_INTEGER_CHUNK_DIGITS: usize = 1000; + +fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { + value.into_py_any(py) +} + +fn date(datetime: &Bound<'_, PyModule>, value: ConfigDate) -> PyResult { + Ok(datetime + .getattr("date")? + .call1((value.year, value.month, value.day))? + .unbind()) +} + +fn time(datetime: &Bound<'_, PyModule>, value: ConfigTime) -> PyResult { + Ok(datetime + .getattr("time")? + .call1((value.hour, value.minute, value.second, value.microsecond))? + .unbind()) +} + +fn date_time(datetime: &Bound<'_, PyModule>, value: ConfigDateTime) -> PyResult { + let tzinfo: Option> = match value.utc_offset_seconds { + Some(seconds) => { + let delta = datetime.getattr("timedelta")?.call( + (), + Some(&[("seconds", seconds)].into_py_dict(datetime.py())?), + )?; + Some(datetime.getattr("timezone")?.call1((delta,))?) + } + None => None, + }; + let (date, time) = (value.date, value.time); + Ok(datetime + .getattr("datetime")? + .call1(( + date.year, + date.month, + date.day, + time.hour, + time.minute, + time.second, + time.microsecond, + tzinfo, + ))? + .unbind()) +} + +/// Python's `int` of a decimal text of any length, built from chunks below the conversion limit. +fn big_integer(py: Python<'_>, text: &str) -> PyResult { + let (negative, digits) = match text.strip_prefix('-') { + Some(digits) => (true, digits), + None => (false, text), + }; + let int = py.import("builtins")?.getattr("int")?; + let mut value: Bound<'_, PyAny> = int.call1((0,))?; + for chunk in digits.as_bytes().chunks(BIG_INTEGER_CHUNK_DIGITS) { + let chunk_text = std::str::from_utf8(chunk) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let scale = int.call1((10,))?.pow(chunk.len(), py.None())?; + value = value.mul(scale)?.add(int.call1((chunk_text,))?)?; + } + if negative { + value = value.neg()?; + } + Ok(value.unbind()) +} + +/// Build the Python object for one loaded value; mapping entries keep their document order. +pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyResult { + match value { + ConfigValue::Null => Ok(py.None()), + ConfigValue::Bool(flag) => object(py, flag), + ConfigValue::Integer(number) => object(py, number), + ConfigValue::BigInteger(text) => big_integer(py, &text), + ConfigValue::Float(number) => object(py, number), + ConfigValue::String(text) => object(py, text), + ConfigValue::Date(value) => date(&py.import("datetime")?, value), + ConfigValue::DateTime(value) => date_time(&py.import("datetime")?, value), + ConfigValue::Time(value) => time(&py.import("datetime")?, value), + ConfigValue::List(items) => { + let list = PyList::empty(py); + for item in items { + list.append(config_value_to_python(py, item)?)?; + } + Ok(list.unbind().into_any()) + } + ConfigValue::Map(entries) => { + let mapping = PyDict::new(py); + for (key, item) in entries { + mapping.set_item( + config_value_to_python(py, key)?, + config_value_to_python(py, item)?, + )?; + } + Ok(mapping.unbind().into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index 30ddb6d43a..52261b6dff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -2,19 +2,33 @@ use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; +use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; +use sqlbuild_discovery::declarations::models::{ + DeclarationFileFact, DeclarationGroup, DeclarationLayout, +}; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{ - DiscoveredFile, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, + DiscoveredFile, DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, +}; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::models::{ + DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, }; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; -use std::path::PathBuf; +use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use crate::bindings::_helpers::config_values::config_value_to_python; use crate::bindings::_helpers::functions::map_to_python; -use crate::bindings::models::ModelDiscoveryRequest; +use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree, SqlTestDiscoveryRequest}; use crate::bindings::types::CompilerDetach; type PyObject = Py; @@ -43,15 +57,7 @@ fn outcome_object( match outcome { FileOutcome::Parsed(value) => parsed(py, value), FileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), - FileOutcome::Failed(failure) => tuple_object( - py, - vec![ - object(py, "error")?, - object(py, failure.kind.as_str())?, - object(py, failure.message)?, - object(py, failure.help)?, - ], - ), + FileOutcome::Failed(failure) => failure_object(py, failure), } } @@ -86,7 +92,15 @@ fn files_object( } type Listing = (String, Vec<(String, bool, bool)>); -type ModelDiscovery = (Vec<(String, PyObject)>, Vec); +type FactRow = ( + String, + &'static str, + &'static str, + String, + Option, + String, +); +type DeclarationLayoutRows = (Option>, Option>); fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { entries @@ -95,19 +109,58 @@ fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { .collect() } -fn listing_rows(tree: &ProjectTree) -> Vec { - read_listings(tree) - .into_iter() - .map(|(directory, entries)| (directory, entry_rows(&entries))) - .collect() +fn fact_row(fact: DeclarationFileFact) -> FactRow { + ( + fact.relative_path, + fact.kind.as_str(), + fact.scope_kind.as_str(), + fact.ownership_root, + fact.owning_path, + fact.declaration_root, + ) +} + +fn group_row(group: DeclarationGroup) -> (String, String) { + (group.root, group.directory) +} + +fn valid_rows(outcome: Result, DiscoveryFailure>, row: fn(T) -> R) -> Option> { + match outcome { + Ok(items) => Some(items.into_iter().map(row).collect()), + Err(_failure) => None, + } } -/// Discover the model files natively with the listings the walk read, or `None` to defer. +#[pymethods] +impl NativeProjectTree { + #[new] + fn new(project_dir: &str) -> Self { + Self { + inner: ProjectTree::new(Path::new(project_dir)), + exported: Mutex::new(HashSet::new()), + } + } + + /// The listings read since the last call, for the pass's Python directory snapshot. + fn listings(&self) -> Vec { + let Ok(mut exported) = self.exported.lock() else { + return Vec::new(); + }; + read_listings(&self.inner) + .into_iter() + .filter(|(directory, _)| exported.insert(directory.clone())) + .map(|(directory, entries)| (directory, entry_rows(&entries))) + .collect() + } +} + +/// Discover the model files natively, or return `None` when Python must run the stage. #[pyfunction] fn discover_model_files( py: Python<'_>, request: ModelDiscoveryRequest, -) -> PyResult> { + tree: &NativeProjectTree, +) -> PyResult>> { let root = ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -118,24 +171,179 @@ fn discover_model_files( extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, }; - let (discovered, listings): ( - Result>, StageDeferral>, - Vec, - ) = py - .compiler_detach(|| { - let tree = ProjectTree::new(&root.directory); - let discovered = discover_models(&root, &tree, &options); - Ok((discovered, listing_rows(&tree))) - }) + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::panics::compiler_error)?; match discovered { - Ok(files) => Ok(Some((files_object(py, files, model_object)?, listings))), + Ok(files) => Ok(Some(files_object(py, files, model_object)?)), + Err(_deferral) => Ok(None), + } +} + +/// The valid declaration file facts and named declaration groups; `None` where Python must scan. +#[pyfunction] +fn discover_declaration_layout( + py: Python<'_>, + tree: &NativeProjectTree, +) -> PyResult> { + let layout: Result = py + .compiler_detach(|| Ok(declaration_layout(&tree.inner))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match layout { + Ok(layout) => Ok(Some(( + valid_rows(layout.file_facts, fact_row), + valid_rows(layout.named_groups, group_row), + ))), + Err(_deferral) => Ok(None), + } +} + +fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { + tuple_object( + py, + vec![ + object(py, "error")?, + object(py, failure.kind.as_str())?, + object(py, failure.message)?, + object(py, failure.help)?, + ], + ) +} + +fn test_block_object(py: Python<'_>, block: SqlTestBlock) -> PyResult { + tuple_object( + py, + vec![ + map_to_python(py, block.header_values)?, + object(py, block.sql_body)?, + ], + ) +} + +fn test_file_object(py: Python<'_>, file: DiscoveredSqlTestFile) -> PyResult { + let blocks: Vec = file + .blocks + .into_iter() + .map(|block| test_block_object(py, block)) + .collect::>()?; + let failure: Option = file + .failure + .map(|failure| failure_object(py, failure)) + .transpose()?; + tuple_object( + py, + vec![ + object(py, "ok")?, + object(py, file.contents)?, + object(py, blocks)?, + object(py, failure)?, + ], + ) +} + +fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult { + tuple_object( + py, + vec![ + object(py, "ok")?, + object(py, file.contents)?, + map_to_python(py, file.header_values)?, + object(py, file.sql_body)?, + ], + ) +} + +fn sql_test_inputs(request: SqlTestDiscoveryRequest) -> (ProjectRoot, SqlTestFileOptions) { + ( + ProjectRoot { + directory: PathBuf::from(request.project_dir), + display_prefix: request.display_prefix, + }, + SqlTestFileOptions { + test_keys: request.test_keys, + scenario_keys: request.scenario_keys, + }, + ) +} + +/// Discover and split the SQL test files natively, or return `None` when Python must run. +#[pyfunction] +fn discover_sql_test_files( + py: Python<'_>, + request: SqlTestDiscoveryRequest, + tree: &NativeProjectTree, +) -> PyResult>> { + let (root, options) = sql_test_inputs(request); + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match discovered { + Ok(files) => Ok(Some(files_object(py, files, test_file_object)?)), + Err(_deferral) => Ok(None), + } +} + +/// Discover and header-parse the scenario files natively, or return `None` when Python must run. +#[pyfunction] +fn discover_scenario_files( + py: Python<'_>, + request: SqlTestDiscoveryRequest, + tree: &NativeProjectTree, +) -> PyResult>> { + let (root, options) = sql_test_inputs(request); + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match discovered { + Ok(files) => Ok(Some(files_object(py, files, scenario_object)?)), + Err(_deferral) => Ok(None), + } +} + +/// One file's payload; a value Python cannot hold natively makes Python load that file. +fn yaml_file_object(py: Python<'_>, outcome: YamlFileOutcome) -> PyResult { + match outcome { + YamlFileOutcome::Loaded { contents, value } => match config_value_to_python(py, value) { + Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), + Err(_unconvertible) => { + tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) + } + }, + YamlFileOutcome::LoadInPython { contents } => { + tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) + } + YamlFileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), + } +} + +/// Read and load YAML files natively, one payload per path, or `None` when Python must run. +#[pyfunction] +fn load_yaml_files( + py: Python<'_>, + relative_paths: Vec, + tree: &NativeProjectTree, +) -> PyResult>> { + let loaded: Result, StageDeferral> = py + .compiler_detach(|| Ok(load_yaml(&tree.inner, &relative_paths))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match loaded { + Ok(outcomes) => Ok(Some( + outcomes + .into_iter() + .map(|outcome| yaml_file_object(py, outcome)) + .collect::>()?, + )), Err(_deferral) => Ok(None), } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; module.add( "PYTHON_ALNUM_UNICODE_VERSION", python_alnum_unicode_version(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 5e8b1378a6..8c2763913b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,5 +1,6 @@ mod allocator; mod catalog; +mod config_values; pub(crate) mod conversions; mod discovery; pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index 05a6f267c3..8697cff25d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -18,6 +18,7 @@ use sqlbuild_core::text::main::close_matches::close_matches; use sqlbuild_core::text::main::decode_python_text::decode_python_text; use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::models::LineIndex; use std::path::Path; @@ -251,8 +252,14 @@ fn _oracle_close_matches( close_matches(word, &candidates, count, cutoff) } +#[pyfunction] +fn _oracle_cleandoc(texts: Vec) -> Vec { + texts.iter().map(|text| python_cleandoc(text)).collect() +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_python_alnum, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_cleandoc, module)?)?; module.add_function(wrap_pyfunction!(_oracle_close_matches, module)?)?; module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index b9982f238c..800836b3df 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -97,6 +97,15 @@ impl From for validation::PositionInput { } } +/// The directory listings one Python discovery pass shares across its native calls. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeProjectTree { + pub(crate) inner: sqlbuild_discovery::tree::models::ProjectTree, + /// Directories whose listings were already handed to the Python snapshot. + pub(crate) exported: Mutex>, +} + /// One native model discovery request from the Python discovery facade. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] @@ -108,3 +117,13 @@ pub(crate) struct ModelDiscoveryRequest { pub(crate) extract_implicit_alias_columns: bool, pub(crate) extract_output_column_locations: bool, } + +/// One native SQL test or scenario discovery request from the Python discovery facade. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct SqlTestDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) test_keys: Vec, + pub(crate) scenario_keys: Vec, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs index cd07d750a3..6b913e3e02 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs @@ -15,12 +15,21 @@ pub(crate) fn match_one(text: &str) -> Option { /// UTF-8 byte offsets of one matched header: header start, header end, and SQL start. pub(crate) fn match_one_bytes(text: &str) -> Option { + match_statement_bytes(text, 0, MODEL_KEYWORD) +} + +/// `\s*KEYWORD\s*\((?P
...)\)\s*;\s*` matched at byte `start`, as UTF-8 byte offsets. +pub(crate) fn match_statement_bytes( + text: &str, + start: usize, + keyword: &[u8], +) -> Option { let bytes = text.as_bytes(); - let mut index = skip_whitespace(text, 0); - if !bytes[index..].starts_with(MODEL_KEYWORD) { + let mut index = skip_whitespace(text, start); + if !bytes[index..].starts_with(keyword) { return None; } - index = skip_whitespace(text, index + MODEL_KEYWORD.len()); + index = skip_whitespace(text, index + keyword.len()); if bytes.get(index) != Some(&b'(') { return None; } diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index e4d9f28ab3..0050ddac66 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -5,3 +5,4 @@ pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod sql_interpolation; pub mod sql_references; +pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs b/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs new file mode 100644 index 0000000000..807bbd0117 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs @@ -0,0 +1,10 @@ +use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; + +/// Byte offsets (header start, header end, match end) of `\s*KEYWORD\s*\(header\)\s*;\s*`. +pub fn match_statement_header( + text: &str, + start: usize, + keyword: &str, +) -> Option { + matching::match_statement_bytes(text, start, keyword.as_bytes()) +} diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index c2cf26daa0..858e2864ee 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -25,6 +25,7 @@ ' """Log one refresh."""\n' " return None\n" ) +_MACRO_MODULE: str = "def cents(value):\n return f'{value} * 100'\n" _SEED_DECLARATION: str = ( "seeds:\n - name: order_channels\n description: Order channels.\n" " columns:\n - name: id\n type: INTEGER\n" @@ -48,6 +49,7 @@ def discovery_failure_cases() -> tuple[FailureCase, ...]: return ( *_config_cases(), *_sql_file_cases(), + *_layout_cases(), *_yaml_file_cases(), *_conflict_cases(), *_python_cases(), @@ -185,6 +187,22 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) +def _layout_cases() -> tuple[FailureCase, ...]: + return tuple( + _case(name=name, expected_code="D013", files={path: contents}) + for name, path, contents in ( + ("scoped-root-at-project-root", "_enums/order_status.sql", ""), + ("declaration-group-below-root", "models/_sqlbuild/macros/money.py", _MACRO_MODULE), + ("declaration-root-nested", "models/marts/_enums/_constants/limits.sql", ""), + ("declaration-group-unsupported-entry", "models/marts/_sqlbuild/notes.txt", ""), + ("named-role-nested-declaration", "schemas/enums/order_status.sql", ""), + ("audit-role-unsupported-entry", "audits/at_least.sql", ""), + ("local-singular-audit", "models/marts/_sqlbuild/_audits/singular/at_least.sql", ""), + ("hook-role-unsupported-entry", "models/marts/_sqlbuild/hooks/notes.sql", ""), + ) + ) + + def _yaml_file_cases() -> tuple[FailureCase, ...]: return ( _case( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 70e969ca26..2026ce2ab1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -46,10 +46,27 @@ def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... PYTHON_ALNUM_UNICODE_VERSION: str +class NativeProjectTree: + def __init__(self, project_dir: str) -> None: ... + def listings(self) -> list[tuple[str, list[tuple[str, bool, bool]]]]: ... + def discover_model_files( - request: dict[str, object], + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... +def discover_sql_test_files( + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... +def discover_scenario_files( + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... +def load_yaml_files(relative_paths: list[str], tree: NativeProjectTree) -> list[object] | None: ... +def discover_declaration_layout( + tree: NativeProjectTree, ) -> ( - tuple[list[tuple[str, tuple[object, ...]]], list[tuple[str, list[tuple[str, bool, bool]]]]] + tuple[ + list[tuple[str, str, str, str, str | None, str]] | None, + list[tuple[str, str]] | None, + ] | None ): ... def substitute_static_project_vars( @@ -68,6 +85,7 @@ def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... def _oracle_project_config(project_dir: str) -> str: ... def _oracle_python_alnum(code_points: list[int]) -> tuple[str, list[bool]]: ... +def _oracle_cleandoc(texts: list[str]) -> list[str]: ... def _oracle_close_matches( word: str, possibilities: list[str], count: int, cutoff: float ) -> list[str]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 6de7587bd5..c8e4494128 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -34,9 +34,20 @@ from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( build_integration_loader_functions, ) +from sqlbuild.compiler.discovery._helpers.native.declarations import ( + prepare_native_declaration_layout, +) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, + discover_native_test_files, +) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( + discover_native_schema_files, + discover_native_source_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( @@ -55,6 +66,7 @@ DiscoveredProjectInputs, DiscoveredProvider, DiscoveredPythonNodeFunctions, + DiscoveredSourceFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestFile, @@ -96,6 +108,7 @@ def build_discovered_project_inputs( project_dir=project_dir, discover_models=discover_models, fact_cache=fact_cache, + native=native, ) declarations: DiscoveredDeclarationFiles = ( discover() @@ -214,21 +227,42 @@ def _discover_declaration_files( project_dir: Path, discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], fact_cache: FactCacheStore | None, + native: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): + source_files: tuple[DiscoveredSourceFile, ...] = ( + discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache) + if native + else discover_source_files(project_dir=project_dir, fact_cache=fact_cache) + ) + model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() + if native: + prepare_native_declaration_layout(project_dir=project_dir) return DiscoveredDeclarationFiles( - source_files=discover_source_files(project_dir=project_dir, fact_cache=fact_cache), - model_files=discover_models(), + source_files=source_files, + model_files=model_files, enum_files=discover_enum_files(project_dir=project_dir), constant_files=discover_constant_files(project_dir=project_dir), model_schema_files=discover_model_schema_files(project_dir=project_dir), sql_function_files=discover_sql_function_files(project_dir=project_dir), sql_hook_files=discover_sql_hook_files(project_dir=project_dir), python_function_files=discover_python_function_files(project_dir=project_dir), - schema_files=discover_schema_files(project_dir=project_dir), + schema_files=( + discover_native_schema_files(project_dir=project_dir) + if native + else discover_schema_files(project_dir=project_dir) + ), seed_files=discover_seed_files(project_dir=project_dir), - test_files=discover_test_files(project_dir=project_dir, fact_cache=fact_cache), - scenario_files=discover_scenario_files(project_dir=project_dir), + test_files=( + discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache) + if native + else discover_test_files(project_dir=project_dir, fact_cache=fact_cache) + ), + scenario_files=( + discover_native_scenario_files(project_dir=project_dir) + if native + else discover_scenario_files(project_dir=project_dir) + ), audit_files=discover_audit_files(project_dir=project_dir), macro_files=discover_macro_files(project_dir=project_dir), adapter_file=discover_adapter_file(project_dir=project_dir), diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index a4ec31738f..6b74f2b4b1 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -43,15 +43,17 @@ prepare_matched_model_file_headers, ) from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file +from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( + schema_file_paths, + source_file_paths, +) from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - SCHEMA_FILE_NAME, SEED_FILE_SUFFIX, - YAML_FILE_SUFFIXES, ) from sqlbuild.compiler.discovery.exceptions import ( DeclarationParseError, @@ -59,7 +61,6 @@ ModelSqlParseError, ProviderDiscoveryError, PythonNodeDiscoveryError, - SchemaParseError, ) from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, @@ -94,7 +95,7 @@ ModelHeaderMatch, NamedDeclarationRoot, ) -from sqlbuild.compiler.discovery.types import ScopedDeclarationFile +from sqlbuild.compiler.discovery.types import NativeDeclarationFact, ScopedDeclarationFile from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, @@ -195,6 +196,25 @@ def _discover_declaration_file_facts( return facts +def remember_declaration_file_facts( + *, project_dir: Path, facts: Iterable[NativeDeclarationFact] +) -> None: + """Record a scan of every declaration kind another walk of this pass already validated.""" + + DirectorySnapshot.current(project_dir=project_dir).memo[(_FACTS_MEMO_KEY, None)] = tuple( + _DeclarationFileFacts( + file_path=project_dir / relative_path, + relative_path=Path(relative_path), + declaration_kind=DeclarationKind(kind), + scope_kind=ScopeKind(scope_kind), + ownership_root=Path(ownership_root), + owning_path=None if owning_path is None else Path(owning_path), + declaration_root=Path(declaration_root), + ) + for relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root in facts + ) + + def _scan_declaration_file_facts( *, project_dir: Path, declaration_kind: DeclarationKind | None, tree: DirectorySnapshot ) -> tuple[_DeclarationFileFacts, ...]: @@ -664,27 +684,7 @@ def discover_python_function_files( def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: """Discover model schema.yml files and seed declaration .yml files.""" - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - schema_paths: list[Path] = [] - models_root: Path = project_dir / "models" - seeds_root: Path = project_dir / "seeds" - - if models_root.is_dir(): - schema_paths.extend( - path - for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) - if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) - ) - if seeds_root.is_dir(): - yaml_path: Path - for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): - raise SchemaParseError( - f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " - ".yaml is not supported" - ) - schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) - - deduped_paths: tuple[Path, ...] = tuple(dict.fromkeys(schema_paths)) + deduped_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) discovered_schema_files: list[DiscoveredSchemaFile] = [] file_path: Path for file_path in deduped_paths: @@ -712,13 +712,7 @@ def discover_source_files( ) -> tuple[DiscoveredSourceFile, ...]: """Discover source declaration YAML files under sources/.""" - sources_root: Path = project_dir / "sources" - if not sources_root.is_dir(): - return () - - yaml_paths: tuple[Path, ...] = tuple( - sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) - ) + yaml_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) def parse(file_path: Path) -> DiscoveredSourceFile: return parse_source_file_with_cache( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index 9d0156d070..0439371b7c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from pathlib import Path from typing import cast @@ -25,6 +25,7 @@ _SINGULAR_AUDIT_DIRECTORY: str = "singular" _LAYOUT_MEMO_KEY: str = "named_declaration_layout" _GROUPS_MEMO_KEY: str = "declaration_groups" +_PATH_SEPARATOR: str = "/" _NESTED_ROLE_DIRECTORIES: frozenset[str] = frozenset( {DECLARATION_GROUP_DIRECTORY} | INHERITED_DECLARATION_DIRECTORIES @@ -86,6 +87,16 @@ def named_declaration_files( yield root, file_path +def remember_declaration_groups(*, project_dir: Path, groups: Iterable[tuple[str, str]]) -> None: + """Record a named layout another walk of this pass validated, with its declaration groups.""" + + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + tree.memo[_GROUPS_MEMO_KEY] = tuple( + (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) for root, directory in groups + ) + tree.memo[_LAYOUT_MEMO_KEY] = True + + def validate_named_declaration_layout(*, project_dir: Path) -> None: """Reject entries that no named declaration role accepts.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py new file mode 100644 index 0000000000..2bb75590b7 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py @@ -0,0 +1,39 @@ +"""The declaration layout walked and validated natively for the Python declaration collections.""" + +from __future__ import annotations + +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import remember_declaration_file_facts +from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( + remember_declaration_groups, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery.types import NativeDeclarationFact + + +def prepare_native_declaration_layout(*, project_dir: Path) -> None: + """Record a natively validated declaration layout; Python rescans an invalid one to raise.""" + + if not native_discovery_supported( + project_dir=project_dir, display_prefix=native_display_prefix(project_dir) + ): + return + tree: _native.NativeProjectTree = native_project_tree(project_dir) + layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( + _native.discover_declaration_layout(tree) + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + if layout is None: + return + facts, groups = layout + if facts is not None: + remember_declaration_file_facts(project_dir=project_dir, facts=facts) + if groups is not None: + remember_declaration_groups(project_dir=project_dir, groups=groups) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index 9061a7e797..a50aa42d3f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -15,6 +15,7 @@ native_display_prefix, native_failure, native_locations, + native_project_tree, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -32,7 +33,7 @@ SQL_MODEL_HEADER_KEYS, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.discovery.types import NativeLocation, NativeModelDiscovery +from sqlbuild.compiler.discovery.types import NativeLocation def discover_native_model_files( @@ -44,28 +45,22 @@ def discover_native_model_files( """Discover model files natively; Python runs when native discovery cannot match it.""" display_prefix: str = native_display_prefix(project_dir) - discovered: NativeModelDiscovery | None = ( - _native.discover_model_files( - { - "project_dir": str(project_dir), - "display_prefix": display_prefix, - "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), - "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), - "extract_implicit_alias_columns": extract_implicit_alias_columns, - "extract_output_column_locations": extract_output_column_locations, - } + files: list[tuple[str, tuple[object, ...]]] | None = ( + _discover_native_files( + project_dir=project_dir, + display_prefix=display_prefix, + extract_implicit_alias_columns=extract_implicit_alias_columns, + extract_output_column_locations=extract_output_column_locations, ) if native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix) else None ) - if discovered is None: + if files is None: return discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=extract_implicit_alias_columns, extract_output_column_locations=extract_output_column_locations, ) - files, listings = discovered - seed_snapshot_listings(project_dir=project_dir, listings=listings) return tuple( _model_file( project_dir=project_dir, @@ -78,6 +73,29 @@ def discover_native_model_files( ) +def _discover_native_files( + *, + project_dir: Path, + display_prefix: str, + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, +) -> list[tuple[str, tuple[object, ...]]] | None: + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: list[tuple[str, tuple[object, ...]]] | None = _native.discover_model_files( + { + "project_dir": str(project_dir), + "display_prefix": display_prefix, + "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), + "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), + "extract_implicit_alias_columns": extract_implicit_alias_columns, + "extract_output_column_locations": extract_output_column_locations, + }, + tree, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return files + + def _model_file( *, project_dir: Path, diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 23d13f459a..aeff70119d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -8,7 +8,13 @@ import sqlbuild._native as _native from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError +from sqlbuild.compiler.discovery.exceptions import ( + DeclarationParseError, + DiscoveryError, + ModelSqlParseError, + SqlScenarioParseError, + SqlTestParseError, +) from sqlbuild.compiler.discovery.types import ( DirectorySnapshotEntry, NativeListing, @@ -19,7 +25,13 @@ _DISPLAY_PROBE: str = "_" _WINDOWS_OS_NAME: str = "nt" _PATH_ENCODING: str = "utf-8" -_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = {"model_sql": ModelSqlParseError} +_NATIVE_TREE_MEMO_KEY: str = "native_project_tree" +_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = { + "model_sql": ModelSqlParseError, + "declaration": DeclarationParseError, + "sql_test": SqlTestParseError, + "sql_scenario": SqlScenarioParseError, +} def native_display_prefix(project_dir: Path) -> str: @@ -64,9 +76,22 @@ def native_discovery_supported(*, project_dir: Path, display_prefix: str) -> boo ) -def seed_snapshot_listings(*, project_dir: Path, listings: list[NativeListing]) -> None: +def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: + """Return the native listings shared by every native call of this discovery pass.""" + + snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + tree: object = snapshot.memo.get(_NATIVE_TREE_MEMO_KEY) + if isinstance(tree, _native.NativeProjectTree): + return tree + created: _native.NativeProjectTree = _native.NativeProjectTree(str(project_dir)) + snapshot.memo[_NATIVE_TREE_MEMO_KEY] = created + return created + + +def seed_snapshot_listings(*, project_dir: Path, tree: _native.NativeProjectTree) -> None: """Share the native walk's listings with the pass's Python directory snapshot.""" + listings: list[NativeListing] = tree.listings() DirectorySnapshot.current(project_dir=project_dir).seed_listings( {project_dir / directory: _snapshot_entries(entries) for directory, entries in listings} ) @@ -81,6 +106,12 @@ def _snapshot_entries( ) +def native_path_text_supported(text: str) -> bool: + """Whether a path's text crosses to the native engine unchanged (no surrogate escapes).""" + + return _is_utf8_text(text) + + def _is_utf8_text(text: str) -> bool: try: _ = text.encode(_PATH_ENCODING) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py new file mode 100644 index 0000000000..eb10063887 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py @@ -0,0 +1,169 @@ +"""SQL unit-test and scenario files discovered by the native engine, as Python records.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_scenario_files, + discover_test_files, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_failure, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values +from sqlbuild.compiler.discovery._helpers.sql.scenarios import ( + build_sql_scenario_file, + parse_sql_scenario_file, +) +from sqlbuild.compiler.discovery._helpers.sql.tests import ( + build_sql_test_block, + parse_sql_test_file, + validate_sql_test_names, +) +from sqlbuild.compiler.discovery.constants import ( + NATIVE_FAILED_TAG, + NATIVE_UNREADABLE_TAG, + SQL_SCENARIO_HEADER_KEYS, + SQL_TEST_HEADER_KEYS, + SQL_TESTS_OWNERSHIP_ROOT, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, +) +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore + +type _NativeFiles = list[tuple[str, tuple[object, ...]]] +type _NativeDiscovery = Callable[ + [dict[str, object], _native.NativeProjectTree], _NativeFiles | None +] + + +def discover_native_test_files( + *, project_dir: Path, fact_cache: FactCacheStore | None = None +) -> tuple[DiscoveredSqlTestFile, ...]: + """Discover SQL test files natively; Python, with its fact cache, runs where native cannot.""" + + files: _NativeFiles | None = _native_files( + project_dir=project_dir, discover=_native.discover_sql_test_files + ) + if files is None: + return discover_test_files(project_dir=project_dir, fact_cache=fact_cache) + return tuple( + _test_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) + for relative_path, payload in files + ) + + +def discover_native_scenario_files(*, project_dir: Path) -> tuple[DiscoveredSqlScenarioFile, ...]: + """Discover scenario files natively; Python runs when native discovery cannot match it.""" + + files: _NativeFiles | None = _native_files( + project_dir=project_dir, discover=_native.discover_scenario_files + ) + if files is None: + return discover_scenario_files(project_dir=project_dir) + return tuple( + _scenario_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) + for relative_path, payload in files + ) + + +def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> _NativeFiles | None: + display_prefix: str = native_display_prefix(project_dir) + if not native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix): + return None + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: _NativeFiles | None = discover( + { + "project_dir": str(project_dir), + "display_prefix": display_prefix, + "test_keys": sorted(SQL_TEST_HEADER_KEYS), + "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + }, + tree, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return files + + +def _test_file( + *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] +) -> DiscoveredSqlTestFile: + file_path: Path = project_dir / relative_path + tag: object = payload[0] + if tag == NATIVE_FAILED_TAG: + raise native_failure(payload) + contents: str + blocks: tuple[DiscoveredSqlTestBlock, ...] + if tag == NATIVE_UNREADABLE_TAG: + contents = file_path.read_text(encoding="utf-8") + blocks = parse_sql_test_file(contents=contents, file_path=file_path) + else: + _tag, native_contents, native_blocks, block_failure = payload + contents = str(native_contents) + blocks = _test_blocks( + file_path=file_path, + native_blocks=cast(list[tuple[dict[str, object], str]], native_blocks), + block_failure=cast(tuple[object, ...] | None, block_failure), + ) + return DiscoveredSqlTestFile( + file_path=file_path, + relative_path=relative_path, + contents=contents, + blocks=blocks, + ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), + ) + + +def _test_blocks( + *, + file_path: Path, + native_blocks: list[tuple[dict[str, object], str]], + block_failure: tuple[object, ...] | None, +) -> tuple[DiscoveredSqlTestBlock, ...]: + blocks: tuple[DiscoveredSqlTestBlock, ...] = tuple( + build_sql_test_block( + header_values=project_native_header_values(values), + sql_body=sql_body, + file_path=file_path, + test_index=test_index, + ) + for test_index, (values, sql_body) in enumerate(native_blocks, start=1) + ) + if block_failure is not None: + raise native_failure(block_failure) + validate_sql_test_names(file_path=file_path, blocks=blocks) + return blocks + + +def _scenario_file( + *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] +) -> DiscoveredSqlScenarioFile: + file_path: Path = project_dir / relative_path + tag: object = payload[0] + if tag == NATIVE_FAILED_TAG: + raise native_failure(payload) + if tag == NATIVE_UNREADABLE_TAG: + return parse_sql_scenario_file( + contents=file_path.read_text(encoding="utf-8"), + file_path=file_path, + relative_path=relative_path, + ) + _tag, contents, values, sql_body = payload + return build_sql_scenario_file( + header_values=project_native_header_values(cast(dict[str, object], values)), + sql_body=str(sql_body), + contents=str(contents), + file_path=file_path, + relative_path=relative_path, + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py new file mode 100644 index 0000000000..a374d1b213 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py @@ -0,0 +1,172 @@ +"""Source and schema YAML files loaded by the native engine, with per-file Python loading.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( + parse_source_file_with_cache, +) +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_source_files +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import project_relative_path +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_path_text_supported, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( + schema_file_paths, + source_file_paths, +) +from sqlbuild.compiler.discovery._helpers.yml.schema import ( + parse_loaded_schema_yml, + parse_schema_yml, +) +from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery.constants import ( + NATIVE_LOADED_TAG, + NATIVE_UNREADABLE_TAG, + NATIVE_YAML_BATCH_BYTES, +) +from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSourceFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry + +type _Payload = tuple[object, ...] + + +def discover_native_source_files( + *, project_dir: Path, fact_cache: FactCacheStore | None = None +) -> tuple[DiscoveredSourceFile, ...]: + """Discover source YAML files natively; Python, with its fact cache, loads the rest.""" + + file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) + if not _native_supported(project_dir=project_dir, file_paths=file_paths): + return discover_source_files(project_dir=project_dir, fact_cache=fact_cache) + discovered: list[DiscoveredSourceFile] = [] + for file_path, payload in _native_payloads(project_dir=project_dir, file_paths=file_paths): + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) + if payload[0] == NATIVE_LOADED_TAG: + _tag, contents, loaded = payload + discovered.append( + DiscoveredSourceFile( + file_path=file_path, + relative_path=relative_path, + contents=str(contents), + source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), + ) + ) + else: + discovered.append( + parse_source_file_with_cache( + project_dir=project_dir, + file_path=file_path, + relative_path=relative_path, + fact_cache=fact_cache, + ) + ) + return tuple(discovered) + + +def discover_native_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: + """Discover schema YAML files natively; Python loads any file native cannot reproduce.""" + + file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) + payloads: Iterator[tuple[Path, _Payload]] = ( + _native_payloads(project_dir=project_dir, file_paths=file_paths) + if _native_supported(project_dir=project_dir, file_paths=file_paths) + else ((file_path, (NATIVE_UNREADABLE_TAG,)) for file_path in file_paths) + ) + discovered: list[DiscoveredSchemaFile] = [] + for file_path, payload in payloads: + contents: str + entries: tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]] + if payload[0] == NATIVE_LOADED_TAG: + _tag, native_contents, loaded = payload + contents = str(native_contents) + entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) + else: + contents = _python_contents(file_path=file_path, payload=payload) + entries = parse_schema_yml(contents=contents, file_path=file_path) + discovered.append( + DiscoveredSchemaFile( + file_path=file_path, + relative_path=project_relative_path(path=file_path, project_dir=project_dir), + contents=contents, + model_entries=entries[0], + seed_entries=entries[1], + ) + ) + return tuple(discovered) + + +def _python_contents(*, file_path: Path, payload: _Payload) -> str: + read: Callable[[], str] = { + True: lambda: file_path.read_text(encoding="utf-8"), + False: lambda: str(payload[1]), + }[payload[0] == NATIVE_UNREADABLE_TAG] + return read() + + +def _native_supported(*, project_dir: Path, file_paths: tuple[Path, ...]) -> bool: + return ( + bool(file_paths) + and native_discovery_supported( + project_dir=project_dir, display_prefix=native_display_prefix(project_dir) + ) + and all( + native_path_text_supported(_relative_text(project_dir=project_dir, file_path=file_path)) + for file_path in file_paths + ) + ) + + +def _relative_text(*, project_dir: Path, file_path: Path) -> str: + return project_relative_path(path=file_path, project_dir=project_dir).as_posix() + + +def _native_payloads( + *, project_dir: Path, file_paths: tuple[Path, ...] +) -> Iterator[tuple[Path, _Payload]]: + """Load the files natively in batches bounded by size, releasing each file once used.""" + + tree: _native.NativeProjectTree = native_project_tree(project_dir) + for batch in _batches(file_paths=file_paths): + payloads: list[object] | None = _native.load_yaml_files( + [_relative_text(project_dir=project_dir, file_path=path) for path in batch], tree + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + pending: list[_Payload] = ( + [(NATIVE_UNREADABLE_TAG,)] * len(batch) + if payloads is None + else [cast(_Payload, payload) for payload in reversed(payloads)] + ) + del payloads + for file_path in batch: + yield file_path, pending.pop() + + +def _batches(*, file_paths: tuple[Path, ...]) -> Iterator[tuple[Path, ...]]: + batch: list[Path] = [] + batch_bytes: int = 0 + for file_path in file_paths: + size: int = _file_size(file_path) + if batch and batch_bytes + size > NATIVE_YAML_BATCH_BYTES: + yield tuple(batch) + batch, batch_bytes = [], 0 + batch.append(file_path) + batch_bytes += size + if batch: + yield tuple(batch) + + +def _file_size(file_path: Path) -> int: + try: + return file_path.stat().st_size + except OSError: + return 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py index 431ad0b816..5ac51932c9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values from sqlbuild.compiler.discovery.constants import ( + SQL_SCENARIO_HEADER_KEYS, SQL_SCENARIOS_OWNERSHIP_ROOT, STATEMENT_HEADER_BODY_PATTERN, ) @@ -21,9 +22,6 @@ ) _SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" _SCENARIO_TAGS_HEADER_KEY: str = "tags" -_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset( - {_SCENARIO_DESCRIPTION_HEADER_KEY, _SCENARIO_TAGS_HEADER_KEY} -) def parse_sql_scenario_file( @@ -43,7 +41,33 @@ def parse_sql_scenario_file( header_line=contents.count("\n", 0, header_match.start("header")) + 1, file_path=file_path, ) - sql_body: str = cleandoc(header_match.group("sql")) + return build_sql_scenario_file( + header_values=header_values, + sql_body=cleandoc(header_match.group("sql")), + contents=contents, + file_path=file_path, + relative_path=relative_path, + ) + + +def build_sql_scenario_file( + *, + header_values: dict[str, object], + sql_body: str, + contents: str, + file_path: Path, + relative_path: Path, +) -> DiscoveredSqlScenarioFile: + """Validate a header-parsed SCENARIO file with only supported keys and build its record.""" + + description_value: object | None = header_values.get(_SCENARIO_DESCRIPTION_HEADER_KEY) + if _SCENARIO_DESCRIPTION_HEADER_KEY in header_values and not isinstance(description_value, str): + raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") + tags_value: object | None = header_values.get(_SCENARIO_TAGS_HEADER_KEY) + if _SCENARIO_TAGS_HEADER_KEY in header_values and ( + not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) + ): + raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") if not sql_body: raise SqlScenarioParseError( f"SQL scenario '{file_path}' must define SQL after SCENARIO(...)" @@ -70,21 +94,11 @@ def _parse_scenario_header(*, header: str, header_line: int, file_path: Path) -> reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_SCENARIO_HEADER_KEYS, + supported_keys=SQL_SCENARIO_HEADER_KEYS, statement="SCENARIO()", header=header, header_line=header_line, file_path=file_path, error_class=SqlScenarioParseError, ) - - description_value: object | None = parsed_header.get(_SCENARIO_DESCRIPTION_HEADER_KEY) - if _SCENARIO_DESCRIPTION_HEADER_KEY in parsed_header and not isinstance(description_value, str): - raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") - tags_value: object | None = parsed_header.get(_SCENARIO_TAGS_HEADER_KEY) - if _SCENARIO_TAGS_HEADER_KEY in parsed_header and ( - not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) - ): - raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") - return parsed_header diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index 3cd3e72c19..a8dac94e45 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -17,6 +17,7 @@ ) from sqlbuild.compiler.discovery.constants import ( SQL_ANALYSIS_CONFIG_KEY, + SQL_TEST_HEADER_KEYS, STATEMENT_HEADER_BODY_PATTERN, ) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError @@ -44,17 +45,6 @@ _TEST_CASES_HEADER_KEY: str = "cases" _TEST_CURSOR_START_HEADER_KEY: str = "cursor_start" _TEST_CURSOR_END_HEADER_KEY: str = "cursor_end" -_TEST_HEADER_KEYS: frozenset[str] = frozenset( - { - _TEST_NAME_HEADER_KEY, - _TEST_MODE_HEADER_KEY, - _TEST_PARAMETERS_HEADER_KEY, - _TEST_CASES_HEADER_KEY, - _TEST_CURSOR_START_HEADER_KEY, - _TEST_CURSOR_END_HEADER_KEY, - SQL_ANALYSIS_CONFIG_KEY, - } -) _PARAMETER_TYPES: tuple[SqlValueKind, ...] = ( SqlValueKind.STRING, SqlValueKind.INTEGER, @@ -102,7 +92,7 @@ def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSq ) ) - _validate_test_names(file_path=file_path, blocks=tuple(discovered_blocks)) + validate_sql_test_names(file_path=file_path, blocks=tuple(discovered_blocks)) return tuple(discovered_blocks) @@ -148,7 +138,27 @@ def _parse_single_sql_test_block( header_line=block_line + raw_test_block.count("\n", 0, header_match.start("header")), file_path=file_path, ) - sql_body: str = cleandoc(header_match.group("sql")) + return build_sql_test_block( + header_values=header_values, + sql_body=cleandoc(header_match.group("sql")), + file_path=file_path, + test_index=test_index, + ) + + +def build_sql_test_block( + *, header_values: dict[str, object], sql_body: str, file_path: Path, test_index: int +) -> DiscoveredSqlTestBlock: + """Validate a header-parsed TEST block with only supported keys and build its record.""" + + if _TEST_NAME_HEADER_KEY in header_values: + _validate_test_name(name_value=header_values[_TEST_NAME_HEADER_KEY], file_path=file_path) + if _TEST_MODE_HEADER_KEY in header_values: + _validate_test_mode(mode_value=header_values[_TEST_MODE_HEADER_KEY], file_path=file_path) + if SQL_ANALYSIS_CONFIG_KEY in header_values and not isinstance( + header_values[SQL_ANALYSIS_CONFIG_KEY], bool + ): + raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") if not sql_body: raise SqlTestParseError(f"SQL test '{file_path}' must define SQL after TEST(...)") @@ -211,23 +221,13 @@ def _parse_test_header(*, header: str, header_line: int, file_path: Path) -> dic reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_TEST_HEADER_KEYS, + supported_keys=SQL_TEST_HEADER_KEYS, statement="TEST()", header=header, header_line=header_line, file_path=file_path, error_class=SqlTestParseError, ) - - if _TEST_NAME_HEADER_KEY in parsed_header: - _validate_test_name(name_value=parsed_header[_TEST_NAME_HEADER_KEY], file_path=file_path) - if _TEST_MODE_HEADER_KEY in parsed_header: - _validate_test_mode(mode_value=parsed_header[_TEST_MODE_HEADER_KEY], file_path=file_path) - if SQL_ANALYSIS_CONFIG_KEY in parsed_header and not isinstance( - parsed_header[SQL_ANALYSIS_CONFIG_KEY], bool - ): - raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") - return parsed_header @@ -391,7 +391,9 @@ def _validate_test_mode(*, mode_value: object, file_path: Path) -> None: raise SqlTestParseError(f"TEST() mode in '{file_path}' must be one of: {allowed_modes}") -def _validate_test_names(*, file_path: Path, blocks: tuple[DiscoveredSqlTestBlock, ...]) -> None: +def validate_sql_test_names(*, file_path: Path, blocks: tuple[DiscoveredSqlTestBlock, ...]) -> None: + """Reject multi-block files whose blocks lack unique names.""" + if len(blocks) <= 1: return diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py new file mode 100644 index 0000000000..39c9289c7a --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py @@ -0,0 +1,48 @@ +"""The source and schema YAML files discovery reads, in Python's discovery order.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( + is_in_scoped_declaration_tree, +) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.constants import SCHEMA_FILE_NAME, YAML_FILE_SUFFIXES +from sqlbuild.compiler.discovery.exceptions import SchemaParseError + + +def source_file_paths(*, project_dir: Path) -> tuple[Path, ...]: + """Return the YAML files directly below sources/, in path order.""" + + sources_root: Path = project_dir / "sources" + if not sources_root.is_dir(): + return () + return tuple( + sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) + ) + + +def schema_file_paths(*, project_dir: Path) -> tuple[Path, ...]: + """Return the unscoped model schema.yml files and seed declaration files, in Python's order.""" + + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + schema_paths: list[Path] = [] + models_root: Path = project_dir / "models" + seeds_root: Path = project_dir / "seeds" + + if models_root.is_dir(): + schema_paths.extend( + path + for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) + if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) + ) + if seeds_root.is_dir(): + yaml_path: Path + for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): + raise SchemaParseError( + f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " + ".yaml is not supported" + ) + schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) + return tuple(dict.fromkeys(schema_paths)) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py index 5feeaf44ff..2ae6db08d8 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py @@ -76,19 +76,33 @@ def parse_schema_yml( ) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: """Parse one schema.yml file into raw model and seed metadata.""" - payload: dict[str, object] = _load_schema_payload(contents=contents, file_path=file_path) + return parse_loaded_schema_yml( + loaded=_load_schema_yaml(contents=contents, file_path=file_path), file_path=file_path + ) + + +def parse_loaded_schema_yml( + *, loaded: object, file_path: Path +) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: + """Parse the document one schema.yml file loads to into raw model and seed metadata.""" + + payload: dict[str, object] = _schema_mapping(loaded=loaded, file_path=file_path) return ( _parse_model_entries(payload=payload, file_path=file_path), _parse_seed_entries(payload=payload, file_path=file_path), ) -def _load_schema_payload(*, contents: str, file_path: Path) -> dict[str, object]: +def _load_schema_yaml(*, contents: str, file_path: Path) -> object: try: loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - payload: object = yaml.load(contents, Loader=loader) + return yaml.load(contents, Loader=loader) except YAMLError as error: raise SchemaParseError(f"{file_path} contains invalid YAML: {error}") from error + + +def _schema_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: + payload: object = loaded if payload is None: return {} if not isinstance(payload, dict): diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py index 0813d61a05..2a6bcd2c68 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py @@ -92,7 +92,15 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: """Parse one sources/*.yml file into raw source declarations.""" - payload: dict[str, object] = _load_sources_payload(contents=contents, file_path=file_path) + return parse_loaded_sources_yml( + loaded=_load_sources_yaml(contents=contents, file_path=file_path), file_path=file_path + ) + + +def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[SourceEntry, ...]: + """Parse the document one sources/*.yml file loads to into raw source declarations.""" + + payload: dict[str, object] = _sources_mapping(loaded=loaded, file_path=file_path) _reject_unknown_source_keys( mapping=payload, allowed=_SOURCE_FILE_KEYS, file_path=file_path, label="sources file" ) @@ -112,12 +120,16 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, . return tuple(parsed_sources) -def _load_sources_payload(*, contents: str, file_path: Path) -> dict[str, object]: +def _load_sources_yaml(*, contents: str, file_path: Path) -> object: try: loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - payload: object = yaml.load(contents, Loader=loader) + return yaml.load(contents, Loader=loader) except YAMLError as error: raise SourceParseError(f"{file_path} contains invalid YAML: {error}") from error + + +def _sources_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: + payload: object = loaded if payload is None: return {} if not isinstance(payload, dict): diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 451aeaea91..2b02d9f48b 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -121,6 +121,10 @@ CONFIG_CONCURRENCY_KEY: str = "concurrency" LEGACY_CONFIG_CONCURRENCY_KEY: str = "max_concurrency" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" +SQL_TEST_HEADER_KEYS: frozenset[str] = frozenset( + {"name", "mode", "parameters", "cases", "cursor_start", "cursor_end", SQL_ANALYSIS_CONFIG_KEY} +) +SQL_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset({"description", "tags"}) LEGACY_SQL_VALIDATION_CONFIG_KEY: str = "sql_validation" SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" REQUIRE_SQL_ANALYSIS_SETTING_KEY: str = "require_sql_analysis" @@ -182,3 +186,5 @@ DISCOVERY_SQL_TEST_FACT_KIND: str = "sql_test" NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" +NATIVE_LOADED_TAG: str = "ok" +NATIVE_YAML_BATCH_BYTES: int = 4 * 1024 * 1024 diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index c603cfce2f..af91ecf729 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -114,36 +114,26 @@ def _assemble_discovered_project_inputs( else project_config.settings.sql_analysis ) ) - if native: + with FactCacheStore( + root=discovery_cache_root( + project_dir=project_dir, + project_config=project_config, + local_config=local_config, + cache_request=cache_request, + ), + namespace=DISCOVERY_FACT_CACHE_NAMESPACE, + algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, + ) as fact_cache: discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( project_dir=project_dir, project_config=project_config, local_config=local_config, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, + fact_cache=fact_cache, declaration_reuse=declaration_reuse, - native=True, + native=native, ) - else: - with FactCacheStore( - root=discovery_cache_root( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - cache_request=cache_request, - ), - namespace=DISCOVERY_FACT_CACHE_NAMESPACE, - algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, - ) as fact_cache: - discovered_inputs = build_discovered_project_inputs( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - fact_cache=fact_cache, - declaration_reuse=declaration_reuse, - ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( configure_discovered_event_exporters, diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 2c91d5429a..980b716ff4 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -20,7 +20,7 @@ type ProjectProvider = Provider type NativeLocation = tuple[str, int, int, int, int] type NativeListing = tuple[str, list[tuple[str, bool, bool]]] -type NativeModelDiscovery = tuple[list[tuple[str, tuple[object, ...]]], list[NativeListing]] +type NativeDeclarationFact = tuple[str, str, str, str, str | None, str] class LoaderConnectionMode(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/_test_types.py b/tests/integration/src/sqlbuild/compiler/_test_types.py index 4c549cca28..6bf8097f5b 100644 --- a/tests/integration/src/sqlbuild/compiler/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/_test_types.py @@ -51,6 +51,16 @@ class CloseMatchesOracleTestCase: expected_mismatches: tuple[tuple[object, object, object], ...] = () +@dataclass(frozen=True) +class CleandocOracleTestCase: + """Seeded SQL bodies compared between the native port and `inspect.cleandoc`.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + @dataclass(frozen=True) class CharacterClassOracleTestCase: """A code point range classified natively and by Python's `str` methods.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 92f2242ec9..37de05fda3 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -94,3 +94,58 @@ class SharedSnapshotTestCase: created_file: str pattern: str expected_matches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class GeneratedLayoutParityTestCase: + """Seeded declaration layouts whose native validation and facts must match Python's.""" + + description: str + seed: int + count: int + expected_mismatches: tuple[str, ...] = () + expected_minimum_valid: int = 0 + expected_minimum_invalid: int = 0 + + +@dataclass(frozen=True) +class GeneratedSqlTestParityTestCase: + """Seeded SQL test and scenario files discovered by both compiler engines.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + expected_minimum_parsed: int = 0 + expected_minimum_failed: int = 0 + + +@dataclass(frozen=True) +class GeneratedYamlFileParityTestCase: + """Seeded source and seed declaration files discovered by both compiler engines.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + expected_minimum_parsed: int = 0 + expected_minimum_failed: int = 0 + + +@dataclass(frozen=True) +class NativeYamlLoadTestCase: + """Source files whose native load outcome and engine parity are both checked.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_native_tags: tuple[str, ...] + + +@dataclass(frozen=True) +class FactCacheFallbackTestCase: + """A runtime under which native source and test discovery must use the fact cache or not.""" + + description: str + unidata_version: str + expected_same_keys_as_python: bool + expected_native_keys: int diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 69df632906..c894609cb0 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -12,19 +12,52 @@ from contextlib import suppress from itertools import compress from pathlib import Path -from typing import Any +from typing import Any, cast +import pytest import yaml from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_constant_files, + discover_enum_files, + discover_macro_files, + discover_model_files, + discover_scenario_files, + discover_schema_files, + discover_source_files, + discover_test_files, +) +from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( + named_declaration_roots, + remember_declaration_groups, +) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, + discover_native_test_files, +) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( + discover_native_schema_files, + discover_native_source_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.discovery.exceptions import DeclarationParseError +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import ( + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredSqlModelFile, +) +from sqlbuild.compiler.discovery.types import NativeDeclarationFact +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.scopes.constants import NAMED_DECLARATION_KINDS from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text @@ -252,7 +285,21 @@ def finished_document(*, rng: random.Random, text: str) -> str: return rng.choice(BYTE_ORDER_MARKS) + ending +LONG_KEY_CONTEXTS: tuple[str, ...] = ( + "{key}: 1\n", "{{{key}: 1}}\n", "[{key}: 1]\n", "a:\n {key} : 1\n", "'{key}': 1\n", + "&k {key}: 1\n", "? {key}\n: 1\n", "x: {{{key}: 1, b: 2}}\n", +) # fmt: skip + + +def long_key_document(*, rng: random.Random) -> str: + """Return a document whose key length straddles the 1024-character simple-key limit.""" + + key: str = rng.choice(("a", "0x", "é")) * rng.randint(990, 1040) + return rng.choice(LONG_KEY_CONTEXTS).format(key=key) + + YAML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "long keys": lambda rng: long_key_document(rng=rng), "scalars": lambda rng: finished_document(rng=rng, text=scalar_document(rng=rng)), "anchors": lambda rng: finished_document(rng=rng, text=anchored_document(rng=rng)), "dumped": lambda rng: finished_document(rng=rng, text=dumped_document(rng=rng)), @@ -638,6 +685,241 @@ def harmful_mismatches( SEPARATORS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", " ", "\u3000", "\n\n") +TEST_HEADER_ENTRIES: tuple[str, ...] = ( + 'name "keeps_orders"', "mode model", 'name "second"', "name ''", "name 1", "mode query", + "mode audit", "mode bogus", "sql_analysis false", "sql_analysis 1", "nme 1", + "parameters (status string), cases (placed (status 'placed'))", + "parameters (amount (type integer, nullable true)), cases (empty (amount null))", + "parameters (amount integer), cases (bad (amount 'x'))", "parameters (amount integer)", + "cursor_start '2024-01-01'", "cursor_end 3", "materialized table", "description (", + "tags [a, b]", "'unterminated", +) # fmt: skip +SCENARIO_HEADER_ENTRIES: tuple[str, ...] = ( + 'description "Orders world"', "tags [north, south]", "description 1", "tags [1]", + "tags north", "name x", "description (", '"unterminated', +) # fmt: skip +STATEMENT_BODIES: tuple[str, ...] = ( + "SELECT 1", "SELECT 2 AS amount", "\n SELECT 3\n FROM orders\n", "", " ", + "\n SELECT 1\n FROM t\n", "\tSELECT 1\n\tFROM\tt", + "SELECT 'TEST (x);' AS label", "SELECT 1\n ", "-- only a comment", "SELECT ')' AS c;", + "\u3000SELECT 1", +) # fmt: skip +STATEMENT_PREFIXES: tuple[str, ...] = ("",) * 10 + (" \n", "\ufeff", "-- lead\n", "\n\n ") + + +def generated_statement_text(*, rng: random.Random, keyword: str, entries: tuple[str, ...]) -> str: + """Return one statement header with random entries, separators and a body.""" + + separator: str = rng.choice(SEPARATORS) + chosen: list[str] = rng.choices( + (rng.sample(entries, k=rng.choice((0, 1, 1, 1, 2))), rng.sample(entries[:2], k=1)), + weights=(3, 2), + )[0] + header: str = ("," + separator).join(chosen) + closing: str = rng.choice((");",) * 8 + (") ;", ")\n;", ")", "));", ");;")) + return ( + f"{rng.choice(('', ' ', ' '))}{keyword}{rng.choice(('', ' '))}({separator}{header}" + f"{separator}{closing}{rng.choice(SEPARATORS)}{rng.choice(STATEMENT_BODIES)}" + ) + + +def generated_test_bytes(*, rng: random.Random) -> bytes: + """Return one SQL test file with one to three blocks, joined at or inside lines.""" + + blocks: list[str] = [ + generated_statement_text(rng=rng, keyword="TEST", entries=TEST_HEADER_ENTRIES) + for _ in range(rng.choice((1, 1, 1, 1, 2, 3))) + ] + joined: str = "".join(rng.choice(("\n", "\n\n", " ", "\r\n")) + block for block in blocks) + return (rng.choice(STATEMENT_PREFIXES) + joined.lstrip("\n\r ")).encode("utf-8") + + +def generated_scenario_bytes(*, rng: random.Random) -> bytes: + """Return one SQL scenario file.""" + + text: str = generated_statement_text( + rng=rng, keyword="SCENARIO", entries=SCENARIO_HEADER_ENTRIES + ) + return (rng.choice(STATEMENT_PREFIXES) + text).encode("utf-8") + + +def sql_test_discovery_outcome(*, project_dir: Path, native: bool) -> object: + """Return the rendered test and scenario files, or the error type, message and help.""" + + discover: Callable[[], tuple[object, object]] = { + True: lambda: ( + discover_native_test_files(project_dir=project_dir), + discover_native_scenario_files(project_dir=project_dir), + ), + False: lambda: ( + discover_test_files(project_dir=project_dir), + discover_scenario_files(project_dir=project_dir), + ), + }[native] + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with DirectorySnapshot.scope(project_dir=project_dir), capture: + rendered[0] = render_stage_capture(discover()) + failure: BaseException | None = capture.failure + return { + True: rendered[0], + False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), + }[failure is None] + + +SOURCE_FIELD_CHOICES: dict[str, tuple[object, ...]] = { + "description": ("Orders feed.", "", 3, None), + "database": ("raw", ""), + "schema": ("shop", None), + "table": ("orders", 7), + "managed": (True, False, "yes"), + "write_strategy": ("append", "merge", "table", "bogus"), + "unique_key": ("order_id", ["order_id", "line"], [], ""), + "cursor_column": ("updated_at",), + "load_batch_size": (100, 0, True), + "type_enforcement": (True, "no"), + "contract": ("enforced", "none", "strict"), + "freshness": ( + {"strategy": "adapter"}, + {"strategy": "column", "column": "updated_at", "type": "timestamp", "lag_tolerance": "2h"}, + {"strategy": "sql", "query": "SELECT 1", "type": "timestamp"}, + {"strategy": "column"}, + ), + "audits": (["not_null"], [{"accepted_values": {"values": [1, 2]}}], "x"), + "freshnes": ("1d",), +} +COLUMN_FIELD_CHOICES: dict[str, tuple[object, ...]] = { + "type": ("INTEGER", "VARCHAR", ""), + "nullable": (True, False, "yes", None), + "description": ("Order id.", 5), + "audits": (["not_null"], ["unique"], []), +} +SCALAR_META_TEMPLATES: tuple[str, ...] = ( + "\n meta: {{flag: {plain}}}", "\n meta:\n seen: {plain}", + "\n meta: &shared{n} {{k: {plain}}}", "\n description: '{single}'", + "\n <<: {{description: '{single}'}}", "\n meta: {{k: {plain}, j: [{plain}]}}", +) # fmt: skip + + +def _random_fields( + *, rng: random.Random, choices: dict[str, tuple[object, ...]] +) -> dict[str, object]: + keys: list[str] = rng.sample(sorted(choices), k=rng.randint(0, 3)) + return { + key: rng.choices(choices[key], weights=(6, *(1,) * (len(choices[key]) - 1)))[0] + for key in keys + } + + +def _generated_column(*, rng: random.Random, names: tuple[str, ...]) -> dict[str, object]: + return {"name": rng.choice(names), **_random_fields(rng=rng, choices=COLUMN_FIELD_CHOICES)} + + +def _generated_entry( + *, + rng: random.Random, + names: tuple[str, ...], + column_names: tuple[str, ...], + fields: dict[str, object], +) -> dict[str, object]: + return { + "name": rng.choice(names), + **fields, + "columns": [ + _generated_column(rng=rng, names=column_names) for _ in range(rng.randint(0, 3)) + ], + "meta": {"owner": random_yaml_value(rng=rng, depth=2)}, + } + + +def generated_source_document(*, rng: random.Random) -> str: + """Return a sources file of random entries in a random dump style, with resolver scalars.""" + + sources: list[object] = [ + _generated_entry( + rng=rng, + names=("orders", "customers", "line_items", ""), + column_names=("order_id", "amount", "status"), + fields=_random_fields(rng=rng, choices=SOURCE_FIELD_CHOICES), + ) + for _ in range(rng.randint(0, 3)) + ] + flow_style: bool | None = rng.choice((None, False, False, True)) + dumped: str = yaml.safe_dump( + {"sources": sources}, + default_flow_style=flow_style, + default_style=rng.choice((None, None, None, '"', "'")), + width=rng.choice((40, 80, 4096)), + allow_unicode=rng.random() < 0.5, + sort_keys=False, + ) + scalar: str = random_scalar(rng=rng) + tail: str = rng.choice(SCALAR_META_TEMPLATES).format( + plain=scalar, single=scalar.replace("'", "''"), n=rng.randint(0, 9) + ) + extra: str = rng.choices((f"- name: extra{tail}\n", "other: [\n"), weights=(8, 1))[0] + appendable: bool = flow_style is False and bool(sources) + return rng.choice(BYTE_ORDER_MARKS) + dumped + {True: extra, False: ""}[appendable] + + +def generated_seed_declaration(*, rng: random.Random) -> str: + """Return a seed declaration file of random entries in a random dump style.""" + + seeds: list[object] = [ + _generated_entry( + rng=rng, + names=("channels", "regions", ""), + column_names=("id", "label"), + fields={"description": rng.choice(("Order channels.", 1))}, + ) + for _ in range(rng.randint(0, 2)) + ] + return yaml.safe_dump( + {"seeds": seeds}, + default_flow_style=rng.choice((None, False, True)), + default_style=rng.choice((None, None, '"')), + sort_keys=False, + ) + + +def native_yaml_tags_and_values( + *, project_dir: Path, relative_paths: list[str] +) -> tuple[list[object], bool]: + """Return each file's native tag and whether every natively loaded value equals LibYAML's.""" + + payloads: list[object] = ( + _native.load_yaml_files(relative_paths, _native.NativeProjectTree(str(project_dir))) or [] + ) + rows: list[tuple[object, ...]] = [cast(tuple[object, ...], payload) for payload in payloads] + loaded: list[tuple[object, ...]] = list(compress(rows, [row[0] == "ok" for row in rows])) + expected: list[object] = [yaml.load(str(row[1]), Loader=yaml.CSafeLoader) for row in loaded] + return [row[0] for row in rows], [row[2] for row in loaded] == expected + + +def yaml_discovery_outcome(*, project_dir: Path, native: bool) -> object: + """Return the rendered source and schema files, or the error type, message and help.""" + + discover: Callable[[], tuple[object, object]] = { + True: lambda: ( + discover_native_source_files(project_dir=project_dir), + discover_native_schema_files(project_dir=project_dir), + ), + False: lambda: ( + discover_source_files(project_dir=project_dir), + discover_schema_files(project_dir=project_dir), + ), + }[native] + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with DirectorySnapshot.scope(project_dir=project_dir), capture: + rendered[0] = render_stage_capture(discover()) + failure: BaseException | None = capture.failure + return { + True: rendered[0], + False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), + }[failure is None] + + class FailureCapture: """Swallow and keep the discovery failure a block raises.""" @@ -706,6 +988,199 @@ def generated_model_bytes(*, rng: random.Random) -> bytes: return text.encode("utf-8") +LAYOUT_ROOTS: tuple[str, ...] = ("models",) * 8 + ( + "tests/unit", "tests/scenarios", "functions/sql", "sources", "macros", "enums", "constants", + "audits", "schemas", "hooks", "_macros", "_sqlbuild", "seeds", +) # fmt: skip +LAYOUT_SEGMENTS: tuple[str, ...] = ("marts", "core") * 6 + ( + "_sqlbuild", "_sqlbuild", "_sqlbuild", "macros", "enums", "constants", "_macros", "_enums", + "_constants", "audits", "_audits", "generic", "singular", "schemas", "_schemas", "hooks", + "_hooks", "sql", "python", ".cache", +) # fmt: skip +LAYOUT_TEMPLATES: tuple[str, ...] = ( + "{root}/{owner}/_sqlbuild/{declaration}", "{root}/{owner}/{local}", "{declaration}", + "{declaration}/{owner}", "{root}/{owner}/_sqlbuild/{named}", "{named}", "{root}/{owner}", +) # fmt: skip +LAYOUT_DECLARATIONS: tuple[str, ...] = ( + "macros", "enums", "constants", "_macros", "_enums", "_constants", +) # fmt: skip +LAYOUT_NAMED_ROLES: tuple[str, ...] = ( + "audits/generic", "audits/singular", "_audits/generic", "schemas", "_schemas", "hooks/sql", + "hooks/python", "_hooks/sql", +) # fmt: skip +LAYOUT_FILE_NAMES: tuple[str, ...] = ("a.py", "b.sql", "c.sql", "__init__.py", "notes.txt", ".keep") +LAYOUT_CONTENTS: dict[str, str] = { + "enums": "ENUM (name e{n}, members [A]);\n", + "_enums": "ENUM (name e{n}, members [A]);\n", + "constants": "CONSTANT (name c{n}, value 1);\n", + "_constants": "CONSTANT (name c{n}, value 1);\n", +} +_FACT_KINDS: dict[type[object], str] = { + DiscoveredMacroFile: "macro", + DiscoveredEnumFile: "enum", + DiscoveredConstantFile: "constant", +} + + +def generated_layout_files(*, rng: random.Random) -> tuple[tuple[str, bytes], ...]: + """Return files spread over canonical, global and grouped declaration roots.""" + + files: dict[str, bytes] = {} + for index in range(rng.randint(1, 6)): + directory: str = rng.choices( + ( + rng.choice(LAYOUT_TEMPLATES).format( + root=rng.choice(LAYOUT_ROOTS[:12]), + owner=rng.choice(("marts", "core", "marts/core")), + declaration=rng.choice(LAYOUT_DECLARATIONS), + local=rng.choice(LAYOUT_DECLARATIONS[3:]), + named=rng.choice(LAYOUT_NAMED_ROLES), + ), + "/".join( + (rng.choice(LAYOUT_ROOTS), *rng.choices(LAYOUT_SEGMENTS, k=rng.randint(0, 3))) + ), + ), + weights=(6, 1), + )[0] + parts: list[str] = [*directory.split("/"), rng.choice(LAYOUT_FILE_NAMES)] + template: str = next( + filter(None, (LAYOUT_CONTENTS.get(part) for part in reversed(parts))), "" + ) + files["/".join(parts)] = template.format(n=index).encode("utf-8") + return tuple(files.items()) + + +def python_layout_outcome(*, project_dir: Path) -> tuple[object, object]: + """Return Python's declaration file facts and named roots, `None` where its scan raises.""" + + return ( + _outcome_or_none(lambda: _python_fact_rows(project_dir=project_dir)), + _outcome_or_none(lambda: _named_root_rows(project_dir=project_dir, seed=lambda: None)), + ) + + +def native_layout_outcome(*, project_dir: Path) -> tuple[object, object]: + """Return the native facts and the named roots its groups give, `None` where it fails.""" + + layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( + _native.discover_declaration_layout(_native.NativeProjectTree(str(project_dir))) + ) + facts, groups = layout or (None, None) + return ( + _mapped(facts, lambda rows: [_native_fact_row(fact) for fact in rows]), + _mapped( + groups, + lambda rows: _named_root_rows( + project_dir=project_dir, + seed=lambda: remember_declaration_groups(project_dir=project_dir, groups=rows), + ), + ), + ) + + +def _mapped[T](value: T | None, transform: Callable[[T], object]) -> object: + return {True: lambda: None, False: lambda: transform(cast(T, value))}[value is None]() + + +def _outcome_or_none(compute: Callable[[], object]) -> object: + outcome: list[object] = [None] + with suppress(DeclarationParseError): + outcome[0] = compute() + return outcome[0] + + +def _python_fact_rows(*, project_dir: Path) -> list[tuple[str, ...]]: + with DirectorySnapshot.scope(project_dir=project_dir): + files: tuple[DiscoveredMacroFile | DiscoveredEnumFile | DiscoveredConstantFile, ...] = ( + *discover_macro_files(project_dir=project_dir), + *discover_enum_files(project_dir=project_dir), + *discover_constant_files(project_dir=project_dir), + ) + return sorted( + ( + file.relative_path.as_posix(), + _FACT_KINDS[type(file)], + file.scope_kind.value, + str(file.ownership_root), + str(file.owning_path), + str(file.declaration_root), + ) + for file in files + ) + + +def _native_fact_row(fact: NativeDeclarationFact) -> tuple[str, ...]: + relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root = fact + return (relative_path, kind, scope_kind, ownership_root, str(owning_path), declaration_root) + + +def _named_root_rows(*, project_dir: Path, seed: Callable[[], None]) -> list[tuple[str, ...]]: + with DirectorySnapshot.scope(project_dir=project_dir): + seed() + return [ + ( + root.relative_directory.as_posix(), + root.kind.value, + root.scope_kind.value, + str(root.ownership_root), + str(root.owning_path), + ) + for root in named_declaration_roots( + project_dir=project_dir, kinds=NAMED_DECLARATION_KINDS + ) + ] + + +def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch) -> object: + """Return the rendered discovery stage under `engine` with its failure type and message.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with capture: + rendered[0] = render_stage_capture(discover_project_inputs(project_dir=project_dir)) + return (rendered[0], type(capture.failure).__name__, str(capture.failure)) + + +class RecordingFactCache: + """A fact cache that records the fact keys discovery asks for and caches nothing.""" + + enabled: bool = True + + def __init__(self) -> None: + self.keys: list[tuple[str, ...]] = [] + + def key(self, *parts: str) -> str: + self.keys.append(parts) + return str(len(self.keys)) + + def read_many(self, entries: object) -> dict[str, object]: + return {} + + def stage(self, *, key: str, slot: str, value: object) -> None: + return None + + +def fact_cache_keys(*, project_dir: Path, native: bool) -> list[tuple[str, ...]]: + """Return the fact keys source and test discovery request under the chosen path.""" + + cache: RecordingFactCache = RecordingFactCache() + fact_cache: FactCacheStore = cast(FactCacheStore, cache) + discover: Callable[[], object] = { + True: lambda: ( + discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache), + discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache), + ), + False: lambda: ( + discover_source_files(project_dir=project_dir, fact_cache=fact_cache), + discover_test_files(project_dir=project_dir, fact_cache=fact_cache), + ), + }[native] + with DirectorySnapshot.scope(project_dir=project_dir): + _ = discover() + return cache.keys + + class CallCounter: """Count calls and delegate to the wrapped function.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py new file mode 100644 index 0000000000..a3b9b3b16d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py @@ -0,0 +1,117 @@ +"""Engine parity for the declaration layout native discovery walks and validates.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + GeneratedLayoutParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + generated_layout_files, + native_layout_outcome, + python_layout_outcome, + stage_outcome, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_AUDIT: bytes = b'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' +_SCHEMA: bytes = ( + b'SCHEMA (\n name order_shape,\n description "Order shape",\n' + b" columns (\n id (type INTEGER),\n ),\n);\n" +) +_HOOK: bytes = b'HOOK (\n description "Record a refresh"\n);\n\nSELECT 1 AS refreshed\n' +_MACRO: bytes = b"def cents(value):\n return f'{value} * 100'\n" +_MODEL: bytes = b'MODEL (description "Orders");\nSELECT 1 AS order_id' + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="global, inherited, local and grouped declarations of every kind", + files=( + ("models/marts/orders.sql", _MODEL), + ("macros/money.py", _MACRO), + ("macros/__init__.py", b""), + ("enums/status.sql", b"ENUM (name order_status, members [PLACED]);\n"), + ("models/marts/_enums/tier.sql", b"ENUM (name tier, members [GOLD]);\n"), + ("models/marts/_sqlbuild/constants/limits.sql", b"CONSTANT (name cap, value 3);\n"), + ("models/marts/_sqlbuild/_audits/generic/is_true.sql", _AUDIT), + ("models/marts/_sqlbuild/_schemas/order_shape.sql", _SCHEMA), + ("models/marts/_sqlbuild/hooks/sql/record_refresh.sql", _HOOK), + ("audits/generic/is_true.sql", _AUDIT), + ("schemas/order_shape.sql", _SCHEMA), + ("hooks/sql/_private.sql", _HOOK), + ("functions/sql/_macros/helpers.py", _MACRO), + ("seeds/channels.csv", b"id\n1\n"), + ), + ), + EngineSwitchParityTestCase( + description="a nested declaration root fails with Python's error", + files=(("models/marts/_enums/macros/money.py", _MACRO),), + ), + EngineSwitchParityTestCase( + description="an unsupported audit role entry fails with Python's error", + files=(("models/marts/orders.sql", _MODEL), ("audits/is_true.sql", _AUDIT)), + ), + EngineSwitchParityTestCase( + description="an unreadable declaration file fails with Python's read error", + files=(("enums/status.sql", b"\xff"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_layout_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedLayoutParityTestCase( + description="seeded layouts over every declaration role", + seed=20261007, + count=400, + expected_minimum_valid=60, + expected_minimum_invalid=60, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_layouts_when_validating_natively_then_facts_and_failures_match_python( + test_case: GeneratedLayoutParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + project_dirs: list[Path] = [tmp_path / f"project_{index}" for index in range(test_case.count)] + for project_dir in project_dirs: + write_project(project_dir=project_dir, files=generated_layout_files(rng=rng)) + expected: list[tuple[object, object]] = [ + python_layout_outcome(project_dir=path) for path in project_dirs + ] + + actual: list[tuple[object, object]] = [ + native_layout_outcome(project_dir=path) for path in project_dirs + ] + + valid: int = sum(None not in outcome for outcome in expected) + inputs: list[object] = [path.name for path in project_dirs] + expected_outcomes: list[object] = [*expected] + actual_outcomes: list[object] = [*actual] + assert ( + mismatches(inputs=inputs, expected=expected_outcomes, actual=actual_outcomes), + valid >= test_case.expected_minimum_valid, + test_case.count - valid >= test_case.expected_minimum_invalid, + ) == (list(test_case.expected_mismatches), True, True), test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py new file mode 100644 index 0000000000..d4c478137f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py @@ -0,0 +1,117 @@ +"""Engine parity for SQL unit-test and scenario files discovered natively.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + GeneratedSqlTestParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + generated_scenario_bytes, + generated_test_bytes, + sql_test_discovery_outcome, + stage_outcome, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_TEST: bytes = b'TEST (name "keeps_orders");\n\nSELECT 1\n' +_SCENARIO: bytes = b'SCENARIO (description "Orders world");\n\nSELECT 1\n' + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedSqlTestParityTestCase( + description="seeded test blocks and scenario headers", + seed=71, + case_count=1000, + expected_minimum_parsed=80, + expected_minimum_failed=150, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_test_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: GeneratedSqlTestParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[tuple[tuple[str, bytes], ...]] = [ + ( + ("tests/unit/orders.sql", generated_test_bytes(rng=rng)), + ("tests/scenarios/orders.sql", generated_scenario_bytes(rng=rng)), + ) + for _ in range(test_case.case_count) + ] + project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] + for project_dir, project_files in zip(project_dirs, files, strict=True): + write_project(project_dir=project_dir, files=project_files) + expected: list[object] = [ + sql_test_discovery_outcome(project_dir=project_dir, native=False) + for project_dir in project_dirs + ] + + actual: list[object] = [ + sql_test_discovery_outcome(project_dir=project_dir, native=True) + for project_dir in project_dirs + ] + + parsed: int = sum(isinstance(outcome, str) for outcome in expected) + assert ( + mismatches(inputs=list(files), expected=expected, actual=actual), + parsed >= test_case.expected_minimum_parsed, + len(expected) - parsed >= test_case.expected_minimum_failed, + ) == (list(test_case.expected_mismatches), True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="test and scenario files in nested, macro and scoped folders", + files=( + ("models/orders.sql", b'MODEL (description "Orders");\nSELECT 1 AS order_id'), + ("tests/unit/orders.sql", _TEST), + ("tests/unit/marts/b.sql", _TEST), + ("tests/unit/macros/money.sql", _TEST), + ("tests/unit/marts/_macros/helpers.py", b"def cents(value):\n return value\n"), + ("tests/unit/marts/_sqlbuild/enums/status.sql", b"ENUM (name s, members [A]);\n"), + ("tests/scenarios/north/orders.sql", _SCENARIO), + ("tests/scenarios/south.sql", _SCENARIO), + ), + ), + EngineSwitchParityTestCase( + description="the first failing test file in path order wins", + files=( + ("tests/unit/a.sql", _TEST), + ("tests/unit/b.sql", b"\xff"), + ("tests/unit/c.sql", b"TEST (nme 1);\nSELECT 1"), + ), + ), + EngineSwitchParityTestCase( + description="a semantic failure in an earlier block beats a later syntax failure", + files=( + ("tests/unit/a.sql", b"TEST (mode bogus);\nSELECT 1\nTEST (name ();\nSELECT 2"), + ), + ), + EngineSwitchParityTestCase( + description="an unreadable scenario fails with Python's read error", + files=(("tests/scenarios/a.sql", b"SCENARIO ();\n\xfe"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_test_files_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py new file mode 100644 index 0000000000..6f6ce814e9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py @@ -0,0 +1,219 @@ +"""Engine parity for source and schema YAML files loaded natively with per-file deferral.""" + +from __future__ import annotations + +import random +import unicodedata +from pathlib import Path + +import pytest + +import sqlbuild._native as _native +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + FactCacheFallbackTestCase, + GeneratedYamlFileParityTestCase, + NativeYamlLoadTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + fact_cache_keys, + generated_seed_declaration, + generated_source_document, + native_yaml_tags_and_values, + stage_outcome, + write_project, + yaml_discovery_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_SOURCES: bytes = ( + b"sources:\n - name: raw_orders\n description: Orders feed.\n" + b" columns:\n - name: order_id\n type: INTEGER\n" +) +_BIG_HEX: str = "0x" + "f" * 4000 +_BIG_OCTAL: str = "0" + "7" * 4200 +_BIG_BINARY: str = "0b" + "1" * 4200 +_SEED: bytes = ( + b"seeds:\n - name: channels\n description: Channels.\n" + b" columns:\n - name: id\n type: INTEGER\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedYamlFileParityTestCase( + description="seeded source and seed declarations in random YAML styles", + seed=81, + case_count=800, + expected_minimum_parsed=60, + expected_minimum_failed=100, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_yaml_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: GeneratedYamlFileParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[tuple[tuple[str, bytes], ...]] = [ + ( + ("sources/orders.yml", generated_source_document(rng=rng).encode("utf-8")), + ("seeds/channels.yml", generated_seed_declaration(rng=rng).encode("utf-8")), + ("seeds/channels.csv", b"id,label\n1,web\n"), + ) + for _ in range(test_case.case_count) + ] + project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] + for project_dir, project_files in zip(project_dirs, files, strict=True): + write_project(project_dir=project_dir, files=project_files) + expected: list[object] = [ + yaml_discovery_outcome(project_dir=project_dir, native=False) + for project_dir in project_dirs + ] + + actual: list[object] = [ + yaml_discovery_outcome(project_dir=project_dir, native=True) for project_dir in project_dirs + ] + + parsed: int = sum(isinstance(outcome, str) for outcome in expected) + assert ( + mismatches(inputs=list(files), expected=expected, actual=actual), + parsed >= test_case.expected_minimum_parsed, + len(expected) - parsed >= test_case.expected_minimum_failed, + ) == (list(test_case.expected_mismatches), True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="sources, model schema and seed declarations", + files=( + ("sources/orders.yml", _SOURCES), + ("sources/more.yaml", _SOURCES.replace(b"raw_orders", b"raw_customers")), + ("sources/notes.txt", b"ignored"), + ("models/orders.sql", b'MODEL (description "Orders");\nSELECT 1 AS order_id'), + ("models/marts/schema.yml", b"models: []\n"), + ("models/marts/_sqlbuild/schemas/schema.yml", b"models: []\n"), + ("seeds/channels.yml", _SEED), + ("seeds/channels.csv", b"id\n1\n"), + ), + ), + EngineSwitchParityTestCase( + description="a Python-only tag and invalid YAML load in Python with its error", + files=( + ("sources/a.yml", b"sources: []\nextra: !!set {a: null}\n"), + ("sources/b.yml", b"sources: [\n"), + ), + ), + EngineSwitchParityTestCase( + description="an unreadable source file fails with Python's read error", + files=(("sources/a.yml", b"sources: []\n# \xff\n"),), + ), + EngineSwitchParityTestCase( + description="a directory named like a source file fails as Python reads it", + files=(("sources/folder.yml/inner.txt", b""),), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_files_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + NativeYamlLoadTestCase( + description="big hexadecimal, octal and binary integers load as values and explicit keys", + files=( + ( + "sources/a.yml", + ( + f"sources:\n - name: orders\n meta:\n hex: {_BIG_HEX}\n" + f" octal: {_BIG_OCTAL}\n binary: {_BIG_BINARY}\n" + f" ? {_BIG_HEX}\n : key\n ? {_BIG_BINARY}\n : key\n" + ).encode(), + ), + ), + expected_native_tags=("ok",), + ), + NativeYamlLoadTestCase( + description="an implicit key beyond PyYAML's simple-key limit is left to Python", + files=(("sources/a.yml", f"sources: []\nmeta: {{{_BIG_HEX}: key}}\n".encode()),), + expected_native_tags=("load",), + ), + NativeYamlLoadTestCase( + description="an earlier invalid file still fails before a later big integer", + files=( + ("sources/a.yml", b"sources: [\n"), + ("sources/b.yml", f"sources: []\nmeta: {{v: {_BIG_HEX}}}\n".encode()), + ), + expected_native_tags=("load", "ok"), + ), + ], + ids=lambda case: case.description, +) +def test_given_big_integers_when_loading_natively_then_values_and_error_order_match_python( + test_case: NativeYamlLoadTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = yaml_discovery_outcome(project_dir=tmp_path, native=False) + + native: object = yaml_discovery_outcome(project_dir=tmp_path, native=True) + tags, values_match = native_yaml_tags_and_values( + project_dir=tmp_path, relative_paths=[path for path, _contents in test_case.files] + ) + + assert (native, tags, values_match) == ( + python, + list(test_case.expected_native_tags), + True, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + FactCacheFallbackTestCase( + description="when native defers, Python discovery uses the fact cache as usual", + unidata_version="0.0.0", + expected_same_keys_as_python=True, + expected_native_keys=2, + ), + FactCacheFallbackTestCase( + description="native discovery itself reads no cached facts", + unidata_version=_native.PYTHON_ALNUM_UNICODE_VERSION, + expected_same_keys_as_python=False, + expected_native_keys=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_native_deferral_when_discovering_sources_and_tests_then_fact_cache_is_used( + test_case: FactCacheFallbackTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project( + project_dir=tmp_path, + files=( + ("sources/orders.yml", _SOURCES), + ("tests/unit/orders.sql", b'TEST (name "keeps_orders");\nSELECT 1\n'), + ), + ) + python: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=False) + monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) + + native: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=True) + + assert ((native == python), len(native)) == ( + test_case.expected_same_keys_as_python, + test_case.expected_native_keys, + ), test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py index 132ce747ef..33913e49c1 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py @@ -47,6 +47,13 @@ case_count=1500, expected_maximum_deferred=20, ), + GeneratedDocumentOracleTestCase( + description="keys straddling the 1024-character simple-key limit", + generator="long keys", + seed=44, + case_count=600, + expected_maximum_deferred=400, + ), GeneratedDocumentOracleTestCase( description="documents written by PyYAML's safe dumper in random styles", generator="dumped", diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py index 0d34d92e5f..0a0e398415 100644 --- a/tests/integration/src/sqlbuild/compiler/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -302,6 +302,10 @@ def python_read_text(*, path: Path, data: bytes) -> str: TABLE_PYTHON_VERSION: tuple[int, int] = (3, 12) +CLEANDOC_FRAGMENTS: tuple[str, ...] = ( + "", " ", " ", "\t", " \t", "\u3000", "\x0b", "\x1c", "\xa0", "\r", "\n", "\n", "\n\n", + "SELECT 1", "a", "é", "x\ty", "--c", +) # fmt: skip MUTATION_ALPHABET: str = "abcdefghijklmnopqrstuvwxyz_0é" @@ -321,6 +325,12 @@ def _substitute(*, rng: random.Random, word: list[str], position: int) -> None: MUTATIONS: tuple[Callable[..., None], ...] = (_insert, _delete, _substitute) +def cleandoc_text(*, rng: random.Random) -> str: + """Return a body mixing indentation, tabs, Unicode spaces and blank lines.""" + + return "".join(rng.choices(CLEANDOC_FRAGMENTS, k=rng.randint(0, 24))) + + def mutated_word(*, rng: random.Random, candidates: list[str]) -> str: """Return a candidate with a few random insertions, deletions and substitutions.""" diff --git a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py index b8f59494e3..0a3f83f3e8 100644 --- a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py @@ -3,6 +3,7 @@ from __future__ import annotations import difflib +import inspect import random import sys import unicodedata @@ -13,10 +14,12 @@ from sqlbuild.compiler.discovery.constants import SQL_MODEL_HEADER_KEYS from tests.integration.src.sqlbuild.compiler._test_types import ( CharacterClassOracleTestCase, + CleandocOracleTestCase, CloseMatchesOracleTestCase, ) from tests.integration.src.sqlbuild.compiler.helpers import ( TABLE_PYTHON_VERSION, + cleandoc_text, mismatches, mutated_word, ) @@ -86,3 +89,30 @@ def test_given_words_when_finding_close_matches_natively_then_difflib_agrees( assert mismatches(inputs=list(words), expected=expected, actual=actual) == list( test_case.expected_mismatches ) + + +@pytest.mark.skipif( + sys.version_info[:2] != TABLE_PYTHON_VERSION, + reason="native discovery runs only on the CI Python, whose cleandoc the port reproduces", +) +@pytest.mark.parametrize( + "test_case", + [ + CleandocOracleTestCase( + description="indented bodies with tabs and blank lines", seed=53, case_count=3000 + ) + ], + ids=lambda case: case.description, +) +def test_given_bodies_when_cleaning_natively_then_inspect_cleandoc_agrees( + test_case: CleandocOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + texts: list[str] = [cleandoc_text(rng=rng) for _ in range(test_case.case_count)] + + expected: list[object] = [inspect.cleandoc(text) for text in texts] + actual: list[object] = list(_native._oracle_cleandoc(texts)) + + assert mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + )