From 53282026a262708a756e3fcead02b9002599f32f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:52:07 +0100 Subject: [PATCH 1/3] feat(native): walk and validate the declaration layout natively With the native engine, Rust walks the canonical, global and named declaration roots on the pass's shared tree, validates the declaration group, scoped root and named role layout, and returns the macro, enum and constant file facts and the declaration groups. Python records them only for a valid layout and rescans an invalid one to raise its own error. Reading and parsing declaration files stays in Python. --- crates/sqlbuild-discovery/src/constants.rs | 22 +++ .../_helpers/declaration_groups.rs | 82 ++++++++ .../declarations/_helpers/declaration_tree.rs | 138 ++++++++++++++ .../src/declarations/_helpers/mod.rs | 5 + .../src/declarations/_helpers/named_layout.rs | 129 +++++++++++++ .../src/declarations/_helpers/paths.rs | 43 +++++ .../src/declarations/_helpers/scan.rs | 20 ++ .../src/declarations/errors.rs | 14 ++ .../declarations/main/declaration_layout.rs | 25 +++ .../src/declarations/main/mod.rs | 1 + .../src/declarations/mod.rs | 7 + .../src/declarations/models.rs | 68 +++++++ .../src/declarations/tests.rs | 6 + .../src/declarations/tests/helpers.rs | 85 +++++++++ .../tests/test_declaration_layout.rs | 124 ++++++++++++ .../src/declarations/tests/test_types.rs | 18 ++ crates/sqlbuild-discovery/src/lib.rs | 1 + crates/sqlbuild-discovery/src/models.rs | 3 + .../src/tree/_helpers/walking.rs | 13 +- .../src/tree/main/children.rs | 25 +++ .../src/tree/main/directories.rs | 10 + .../sqlbuild-discovery/src/tree/main/mod.rs | 3 + .../src/tree/main/read_tree.rs | 13 ++ .../sqlbuild-discovery/src/tree/main/rglob.rs | 4 +- crates/sqlbuild-discovery/src/tree/models.rs | 16 ++ .../src/bindings/_helpers/discovery.rs | 110 ++++++++--- crates/sqlbuild-python/src/bindings/models.rs | 9 + .../corpus/discovery_failure_cases.py | 18 ++ src/sqlbuild/_native.pyi | 14 +- .../_helpers/filesystem/aggregation.py | 16 +- .../discovery/_helpers/filesystem/core.py | 21 +- .../_helpers/filesystem/named_declarations.py | 13 +- .../discovery/_helpers/native/declarations.py | 39 ++++ .../discovery/_helpers/native/model_files.py | 46 +++-- .../discovery/_helpers/native/payloads.py | 16 +- src/sqlbuild/compiler/discovery/types.py | 2 +- .../compiler/discovery/_test_types.py | 12 ++ .../sqlbuild/compiler/discovery/helpers.py | 180 +++++++++++++++++- .../test_native_declaration_discovery.py | 117 ++++++++++++ 39 files changed, 1430 insertions(+), 58 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/errors.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/models.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/tests.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs create mode 100644 crates/sqlbuild-discovery/src/declarations/tests/test_types.rs create mode 100644 crates/sqlbuild-discovery/src/tree/main/children.rs create mode 100644 crates/sqlbuild-discovery/src/tree/main/directories.rs create mode 100644 crates/sqlbuild-discovery/src/tree/main/read_tree.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/declarations.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index b4828a179a..c6d8b50f71 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -27,3 +27,25 @@ pub const CANONICAL_AUTHORED_ROOTS: [&[&str]; 5] = [ pub const SQL_TESTS_ROOT: &[&str] = &["tests", "unit"]; /// The directory of macro tests below the SQL test root. pub const MACRO_TESTS_DIRECTORY: &str = "macros"; +/// Inherited declaration directories, which may also be project-wide global roots. +pub const GLOBAL_DECLARATION_DIRECTORIES: [&str; 3] = ["macros", "enums", "constants"]; +/// Folder-local declaration directories. +pub const LOCAL_DECLARATION_DIRECTORIES: [&str; 3] = ["_macros", "_enums", "_constants"]; +/// Directories a declaration group may hold for named declarations. +pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [ + "audits", "_audits", "schemas", "_schemas", "hooks", "_hooks", +]; +/// Project-wide named declaration roles. +pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"]; +/// Root of seed files, walked so the Python snapshot shares its listings. +pub const SEEDS_DIRECTORY: &str = "seeds"; +/// Suffix of macro files. +pub const PYTHON_FILE_SUFFIX: &str = ".py"; +/// The stem of package initialisers, which are not macro files. +pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; +/// The project-wide audit role directory. +pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; +/// The folder-local audit role directory inside a declaration group. +pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; +/// The singular audit directory below an audit role. +pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs new file mode 100644 index 0000000000..7172da607a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs @@ -0,0 +1,82 @@ +//! Python's `_validate_declaration_groups` and `_declaration_groups`. + +use crate::constants::{ + CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GROUPED_NAMED_DECLARATION_DIRECTORIES, + SCOPED_DECLARATION_DIRECTORIES, +}; +use crate::declarations::_helpers::paths::parent; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::DeclarationGroup; +use crate::models::StageDeferral; +use crate::tree::main::children::children; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// Every `_sqlbuild/` directory below a canonical root, root by root, in path order. +fn all_groups(tree: &ProjectTree) -> Result, StageDeferral> { + let mut groups: Vec = Vec::new(); + for root_parts in CANONICAL_AUTHORED_ROOTS { + let root: String = root_parts.join("/"); + if !tree.is_dir(&root) { + continue; + } + for directory in rglob(tree, &root, |entry| { + entry.name == DECLARATION_GROUP_DIRECTORY + })? { + if tree.is_dir(&directory) { + groups.push(DeclarationGroup { + root: root.clone(), + directory, + }); + } + } + } + Ok(groups) +} + +/// Python's `_declaration_groups`: the groups below a concrete owner directory. +pub(crate) fn owned_groups(tree: &ProjectTree) -> Result, StageDeferral> { + Ok(all_groups(tree)? + .into_iter() + .filter(|group| parent(&group.directory) != group.root) + .collect()) +} + +pub(crate) fn validate_declaration_groups(tree: &ProjectTree) -> Result<(), ScanError> { + if tree.exists(DECLARATION_GROUP_DIRECTORY) { + return Err(declaration_failure(format!( + "Grouped declaration root {DECLARATION_GROUP_DIRECTORY}/ must be below a canonical \ + authored root" + ))); + } + for group in all_groups(tree)? { + if parent(&group.directory) == group.root { + return Err(declaration_failure(format!( + "Grouped declaration root {}/ must be below a concrete owner directory; use the \ + project-wide macros/, enums/, constants/, audits/, schemas/, or hooks/ root \ + instead", + group.directory + ))); + } + let unsupported: Vec = children(tree, &group.directory)? + .into_iter() + .filter(|(_, entry)| !is_group_role(entry)) + .map(|(path, _)| path) + .collect(); + if !unsupported.is_empty() { + return Err(declaration_failure(format!( + "Declaration group {}/ contains unsupported entries: {}", + group.directory, + unsupported.join(", ") + ))); + } + } + Ok(()) +} + +fn is_group_role(entry: &TreeEntry) -> bool { + entry.is_dir + && (SCOPED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str()) + || GROUPED_NAMED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str())) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs new file mode 100644 index 0000000000..8f99a20d08 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -0,0 +1,138 @@ +//! Python's `_scan_declaration_file_facts`: macro, enum and constant files with scope facts. + +use crate::constants::{ + CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, + LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, PYTHON_INIT_MODULE_STEM, + SCOPED_DECLARATION_DIRECTORIES, SQL_FILE_SUFFIX, +}; +use crate::declarations::_helpers::declaration_groups::validate_declaration_groups; +use crate::declarations::_helpers::paths::{directory_facts, name, parent, stem}; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::{DeclarationFileFact, DeclarationKind, ScopeKind}; +use crate::tree::main::directories::directories; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; + +/// Where a declaration root sits and who owns it. +struct DeclarationRoot<'a> { + ownership_root: &'a str, + directory: &'a str, + owning_path: Option<&'a str>, + scope_kind: ScopeKind, +} + +/// Every declaration file below every macro, enum and constant root, by relative path. +pub(crate) fn declaration_file_facts( + tree: &ProjectTree, +) -> Result, ScanError> { + validate_declaration_groups(tree)?; + let mut local: Vec<&str> = LOCAL_DECLARATION_DIRECTORIES.to_vec(); + local.sort_unstable(); + if let Some(directory) = local.into_iter().find(|directory| tree.is_dir(directory)) { + return Err(declaration_failure(format!( + "Scoped declaration root {directory}/ must be below a canonical authored root" + ))); + } + let mut facts: Vec = Vec::new(); + let mut global: Vec<&str> = GLOBAL_DECLARATION_DIRECTORIES.to_vec(); + global.sort_unstable(); + for directory in global { + if tree.is_dir(directory) { + facts.extend(files_under_root( + tree, + &DeclarationRoot { + ownership_root: directory, + directory, + owning_path: None, + scope_kind: ScopeKind::Global, + }, + )?); + } + } + for root_parts in CANONICAL_AUTHORED_ROOTS { + let root: String = root_parts.join("/"); + if tree.is_dir(&root) { + facts.extend(scoped_root_files(tree, &root, root_parts.len())?); + } + } + facts.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + Ok(facts) +} + +fn scoped_root_files( + tree: &ProjectTree, + root: &str, + root_length: usize, +) -> Result, ScanError> { + let mut facts: Vec = Vec::new(); + for directory in directories(tree, root)? { + let Some((_, scope_kind)) = directory_facts(name(&directory)) else { + continue; + }; + let parts: Vec<&str> = directory.split('/').collect(); + let between: &[&str] = &parts[root_length..parts.len() - 1]; + if between + .iter() + .any(|part| SCOPED_DECLARATION_DIRECTORIES.contains(part)) + { + return Err(nested_root_failure(&directory)); + } + let mut owning_path: &str = parent(&directory); + if name(owning_path) == DECLARATION_GROUP_DIRECTORY { + owning_path = parent(owning_path); + } + facts.extend(files_under_root( + tree, + &DeclarationRoot { + ownership_root: root, + directory: &directory, + owning_path: Some(owning_path), + scope_kind, + }, + )?); + } + Ok(facts) +} + +fn nested_root_failure(directory: &str) -> ScanError { + declaration_failure(format!( + "Declaration root {directory}/ is nested inside another declaration tree" + )) +} + +fn files_under_root( + tree: &ProjectTree, + root: &DeclarationRoot<'_>, +) -> Result, ScanError> { + if let Some(nested) = directories(tree, root.directory)? + .into_iter() + .find(|directory| SCOPED_DECLARATION_DIRECTORIES.contains(&name(directory))) + { + return Err(nested_root_failure(&nested)); + } + let Some((kind, _)) = directory_facts(name(root.directory)) else { + return Ok(Vec::new()); + }; + let suffix: &str = if kind == DeclarationKind::Macro { + PYTHON_FILE_SUFFIX + } else { + SQL_FILE_SUFFIX + }; + Ok( + rglob(tree, root.directory, |entry| entry.name.ends_with(suffix))? + .into_iter() + .filter(|path| { + kind != DeclarationKind::Macro || stem(name(path)) != PYTHON_INIT_MODULE_STEM + }) + .map(|relative_path| DeclarationFileFact { + relative_path, + kind, + scope_kind: root.scope_kind, + ownership_root: root.ownership_root.to_owned(), + owning_path: root.owning_path.map(str::to_owned), + declaration_root: root.directory.to_owned(), + }) + .collect(), + ) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..edd0650597 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod declaration_groups; +pub(crate) mod declaration_tree; +pub(crate) mod named_layout; +pub(crate) mod paths; +pub(crate) mod scan; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs new file mode 100644 index 0000000000..073d9d6dd8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs @@ -0,0 +1,129 @@ +//! Python's `validate_named_declaration_layout`. + +use crate::constants::{ + AUDIT_ROLE_DIRECTORY, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, + GLOBAL_NAMED_DECLARATION_DIRECTORIES, LOCAL_AUDIT_ROLE_DIRECTORY, + LOCAL_DECLARATION_DIRECTORIES, SINGULAR_AUDIT_DIRECTORY, +}; +use crate::declarations::_helpers::declaration_groups::owned_groups; +use crate::declarations::_helpers::paths::{join, name}; +use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::errors::ScanError; +use crate::declarations::models::DeclarationGroup; +use crate::tree::main::children::children; +use crate::tree::main::directories::directories; +use crate::tree::models::ProjectTree; + +const AUDIT_ROLES: &[&str] = &["generic", SINGULAR_AUDIT_DIRECTORY]; +const LOCAL_AUDIT_ROLES: &[&str] = &["generic"]; +const HOOK_ROLES: &[&str] = &["python", "sql"]; + +/// One role directory and the child directories it accepts, in sorted order. +struct RoleChildren<'a> { + directory: &'a str, + allowed: &'a [&'a str], + help_text: &'a str, +} + +/// The declaration groups the named roles use, once every named role layout is valid. +pub(crate) fn named_declaration_groups( + tree: &ProjectTree, +) -> Result, ScanError> { + if tree.is_dir(AUDIT_ROLE_DIRECTORY) { + require_role_children( + tree, + &RoleChildren { + directory: AUDIT_ROLE_DIRECTORY, + allowed: AUDIT_ROLES, + help_text: "audit files must live in audits/generic/ or audits/singular/", + }, + )?; + } + let mut top_levels: Vec<&str> = GLOBAL_NAMED_DECLARATION_DIRECTORIES.to_vec(); + top_levels.sort_unstable(); + for top_level in top_levels { + if !tree.is_dir(top_level) { + continue; + } + if let Some(nested) = directories(tree, top_level)? + .into_iter() + .find(|directory| is_nested_role_directory(name(directory))) + { + return Err(declaration_failure(format!( + "Declaration directory {nested}/ is not allowed inside the project-wide \ + {top_level}/ role; scoped declarations belong under \ + /{DECLARATION_GROUP_DIRECTORY}/ below a resource tree" + ))); + } + } + let groups: Vec = owned_groups(tree)?; + for group in &groups { + validate_group_roles(tree, &group.directory)?; + } + Ok(groups) +} + +fn validate_group_roles(tree: &ProjectTree, group: &str) -> Result<(), ScanError> { + let roles: [(&str, &[&str]); 4] = [ + (AUDIT_ROLE_DIRECTORY, AUDIT_ROLES), + (LOCAL_AUDIT_ROLE_DIRECTORY, LOCAL_AUDIT_ROLES), + ("hooks", HOOK_ROLES), + ("_hooks", HOOK_ROLES), + ]; + for (role, allowed) in roles { + let directory: String = join(group, role); + if !tree.is_dir(&directory) { + continue; + } + let singular: String = join(&directory, SINGULAR_AUDIT_DIRECTORY); + if role == LOCAL_AUDIT_ROLE_DIRECTORY && tree.exists(&singular) { + return Err(declaration_failure(format!( + "{singular}/ is invalid: singular audits are never used by name, so folder-only \ + visibility has no meaning; use {DECLARATION_GROUP_DIRECTORY}/audits/singular/" + ))); + } + let help_text: String = format!( + "{role}/ accepts only {}", + allowed + .iter() + .map(|name| format!("{name}/")) + .collect::>() + .join(", ") + ); + require_role_children( + tree, + &RoleChildren { + directory: &directory, + allowed, + help_text: &help_text, + }, + )?; + } + Ok(()) +} + +fn require_role_children(tree: &ProjectTree, role: &RoleChildren<'_>) -> Result<(), ScanError> { + let unsupported: Vec = children(tree, role.directory)? + .into_iter() + .filter(|(_, entry)| { + !entry.name.starts_with('.') + && (!entry.is_dir || !role.allowed.contains(&entry.name.as_str())) + }) + .map(|(path, _)| path) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + Err(declaration_failure(format!( + "Unsupported entries in {}/: {}; {}", + role.directory, + unsupported.join(", "), + role.help_text + ))) +} + +fn is_nested_role_directory(directory_name: &str) -> bool { + directory_name == DECLARATION_GROUP_DIRECTORY + || GLOBAL_DECLARATION_DIRECTORIES.contains(&directory_name) + || LOCAL_DECLARATION_DIRECTORIES.contains(&directory_name) +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs new file mode 100644 index 0000000000..b71022439c --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/paths.rs @@ -0,0 +1,43 @@ +//! Relative path text helpers shared by declaration discovery. + +use crate::declarations::models::{DeclarationKind, ScopeKind}; + +/// The parent of a `/`-separated relative path (`""` at the top). +pub(crate) fn parent(relative_path: &str) -> &str { + relative_path + .rsplit_once('/') + .map_or("", |(parent, _)| parent) +} + +/// The final component of a `/`-separated relative path. +pub(crate) fn name(relative_path: &str) -> &str { + relative_path + .rsplit_once('/') + .map_or(relative_path, |(_, name)| name) +} + +/// A `/`-separated relative directory joined with one more component. +pub(crate) fn join(directory: &str, name: &str) -> String { + format!("{directory}/{name}") +} + +/// Python's `Path.stem` of a file name. +pub(crate) fn stem(file_name: &str) -> &str { + match file_name.rfind('.') { + Some(index) if index > 0 && index + 1 < file_name.len() => &file_name[..index], + _ => file_name, + } +} + +/// Python's `DECLARATION_DIRECTORY_FACTS` for a scoped declaration directory name. +pub(crate) fn directory_facts(directory_name: &str) -> Option<(DeclarationKind, ScopeKind)> { + match directory_name { + "macros" => Some((DeclarationKind::Macro, ScopeKind::Inherited)), + "enums" => Some((DeclarationKind::Enum, ScopeKind::Inherited)), + "constants" => Some((DeclarationKind::Constant, ScopeKind::Inherited)), + "_macros" => Some((DeclarationKind::Macro, ScopeKind::Local)), + "_enums" => Some((DeclarationKind::Enum, ScopeKind::Local)), + "_constants" => Some((DeclarationKind::Constant, ScopeKind::Local)), + _ => None, + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs new file mode 100644 index 0000000000..ce4fce93d0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs @@ -0,0 +1,20 @@ +//! Building and splitting declaration scan errors. + +use crate::declarations::errors::ScanError; +use crate::models::{DiscoveryFailure, FailureKind, StageDeferral}; + +/// A `DeclarationParseError` with Python's message. +pub(crate) fn declaration_failure(message: String) -> ScanError { + ScanError::Failure(DiscoveryFailure::new(FailureKind::Declaration, message)) +} + +/// Split a scan result into the authored outcome, keeping a deferral for the caller. +pub(crate) fn authored_outcome( + result: Result, +) -> Result, StageDeferral> { + match result { + Ok(value) => Ok(Ok(value)), + Err(ScanError::Failure(failure)) => Ok(Err(failure)), + Err(ScanError::Deferral(deferral)) => Err(deferral), + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/errors.rs b/crates/sqlbuild-discovery/src/declarations/errors.rs new file mode 100644 index 0000000000..88ae69bf1b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/errors.rs @@ -0,0 +1,14 @@ +//! Why a declaration scan stopped: an authored layout failure or a whole-stage deferral. + +use crate::models::{DiscoveryFailure, StageDeferral}; + +pub(crate) enum ScanError { + Failure(DiscoveryFailure), + Deferral(StageDeferral), +} + +impl From for ScanError { + fn from(deferral: StageDeferral) -> Self { + Self::Deferral(deferral) + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs new file mode 100644 index 0000000000..6b775310ad --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs @@ -0,0 +1,25 @@ +//! Walk and validate the declaration layout the Python declaration collections read. + +use crate::_helpers::pool::discovery_pool; +use crate::constants::SEEDS_DIRECTORY; +use crate::declarations::_helpers::declaration_tree::declaration_file_facts; +use crate::declarations::_helpers::named_layout::named_declaration_groups; +use crate::declarations::_helpers::scan::authored_outcome; +use crate::declarations::models::DeclarationLayout; +use crate::models::StageDeferral; +use crate::tree::main::read_tree::read_tree; +use crate::tree::models::ProjectTree; + +/// The declaration file facts and named declaration groups, each valid or failing as in Python. +pub fn declaration_layout(tree: &ProjectTree) -> Result { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + pool.install(|| { + let file_facts = authored_outcome(declaration_file_facts(tree))?; + let named_groups = authored_outcome(named_declaration_groups(tree))?; + read_tree(tree, SEEDS_DIRECTORY)?; + Ok(DeclarationLayout { + file_facts, + named_groups, + }) + }) +} diff --git a/crates/sqlbuild-discovery/src/declarations/main/mod.rs b/crates/sqlbuild-discovery/src/declarations/main/mod.rs new file mode 100644 index 0000000000..32fb1c5af9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/main/mod.rs @@ -0,0 +1 @@ +pub mod declaration_layout; diff --git a/crates/sqlbuild-discovery/src/declarations/mod.rs b/crates/sqlbuild-discovery/src/declarations/mod.rs new file mode 100644 index 0000000000..a2537f3e8e --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/declarations/models.rs b/crates/sqlbuild-discovery/src/declarations/models.rs new file mode 100644 index 0000000000..33cb14ccbe --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/models.rs @@ -0,0 +1,68 @@ +//! The declaration layout facts native discovery hands to the Python discovery helpers. + +use crate::models::DiscoveryFailure; + +/// The kinds a macro, enum or constant declaration root holds (Python's `DeclarationKind`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarationKind { + Macro, + Enum, + Constant, +} + +impl DeclarationKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Macro => "macro", + Self::Enum => "enum", + Self::Constant => "constant", + } + } +} + +/// Python's `ScopeKind` for a declaration root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScopeKind { + Global, + Inherited, + Local, +} + +impl ScopeKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Global => "global", + Self::Inherited => "inherited", + Self::Local => "local", + } + } +} + +/// Python's `_DeclarationFileFacts` with `/`-separated project-relative paths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationFileFact { + pub relative_path: String, + pub kind: DeclarationKind, + pub scope_kind: ScopeKind, + pub ownership_root: String, + pub owning_path: Option, + pub declaration_root: String, +} + +/// A `_sqlbuild/` declaration group below a concrete owner, with its canonical root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationGroup { + pub root: String, + pub directory: String, +} + +/// The validated declaration layout, or the first failure Python's own scan raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationLayout { + /// Every macro, enum and constant file, by relative path. + pub file_facts: Result, DiscoveryFailure>, + /// The declaration groups the named roles use, once the named layout is valid. + pub named_groups: Result, DiscoveryFailure>, +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests.rs b/crates/sqlbuild-discovery/src/declarations/tests.rs new file mode 100644 index 0000000000..ca1db44484 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_declaration_layout.rs"] +mod declaration_layout; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs new file mode 100644 index 0000000000..e39cfcf614 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs @@ -0,0 +1,85 @@ +use crate::declarations::main::declaration_layout::declaration_layout; +use crate::declarations::models::{DeclarationFileFact, DeclarationGroup, DeclarationLayout}; +use crate::declarations::tests::test_types::{FactRow, LayoutTestCase}; +use crate::models::DiscoveryFailure; +use crate::tree::models::ProjectTree; +use std::fs; + +type Outcome = Result, String>; + +/// The case's layout as owned fact rows and group pairs, or the failure messages. +pub(super) fn layout_rows( + test_case: &LayoutTestCase, +) -> (Outcome, Outcome<(String, String)>) { + let project = tempfile::tempdir().expect("temporary project"); + for relative_path in test_case.files { + let path = project.path().join(relative_path); + fs::create_dir_all(path.parent().expect("parent directory")).expect("directories"); + fs::write(path, "").expect("file"); + } + for relative_path in test_case.directories { + fs::create_dir_all(project.path().join(relative_path)).expect("directory"); + } + let layout: DeclarationLayout = + declaration_layout(&ProjectTree::new(project.path())).expect("no deferral"); + ( + rows(layout.file_facts, fact_row), + rows(layout.named_groups, group_pair), + ) +} + +/// Expected fact rows in the owned shape `layout_rows` returns. +pub(super) fn owned_facts( + expected: Result<&'static [FactRow], &'static str>, +) -> Outcome { + expected + .map(|facts| facts.iter().map(owned_fact).collect()) + .map_err(str::to_owned) +} + +/// Expected group pairs in the owned shape `layout_rows` returns. +pub(super) fn owned_groups( + expected: Result<&'static [(&'static str, &'static str)], &'static str>, +) -> Outcome<(String, String)> { + expected + .map(|groups| groups.iter().map(owned_group).collect()) + .map_err(str::to_owned) +} + +fn owned_group(group: &(&str, &str)) -> (String, String) { + (group.0.to_owned(), group.1.to_owned()) +} + +pub(super) type OwnedFactRow = (String, String, String, String, Option, String); + +fn rows(outcome: Result, DiscoveryFailure>, row: fn(T) -> R) -> Outcome { + outcome + .map(|items| items.into_iter().map(row).collect()) + .map_err(|failure| failure.message) +} + +fn fact_row(fact: DeclarationFileFact) -> OwnedFactRow { + ( + fact.relative_path, + fact.kind.as_str().to_owned(), + fact.scope_kind.as_str().to_owned(), + fact.ownership_root, + fact.owning_path, + fact.declaration_root, + ) +} + +fn owned_fact(fact: &FactRow) -> OwnedFactRow { + ( + fact.0.to_owned(), + fact.1.to_owned(), + fact.2.to_owned(), + fact.3.to_owned(), + fact.4.map(str::to_owned), + fact.5.to_owned(), + ) +} + +fn group_pair(group: DeclarationGroup) -> (String, String) { + (group.root, group.directory) +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs new file mode 100644 index 0000000000..1cab16a64b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs @@ -0,0 +1,124 @@ +use crate::declarations::tests::helpers::{layout_rows, owned_facts, owned_groups}; +use crate::declarations::tests::test_types::LayoutTestCase; + +#[test] +fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python() { + let test_cases = [ + LayoutTestCase { + description: "global, inherited, grouped and local roots sort by relative path", + files: &[ + "macros/text.py", + "macros/__init__.py", + "enums/status.sql", + "models/marts/_enums/local.sql", + "models/marts/_sqlbuild/constants/limits.sql", + "models/marts/_sqlbuild/audits/generic/positive.sql", + ], + directories: &[], + expected_facts: Ok(&[ + ("enums/status.sql", "enum", "global", "enums", None, "enums"), + ( + "macros/text.py", + "macro", + "global", + "macros", + None, + "macros", + ), + ( + "models/marts/_enums/local.sql", + "enum", + "local", + "models", + Some("models/marts"), + "models/marts/_enums", + ), + ( + "models/marts/_sqlbuild/constants/limits.sql", + "constant", + "inherited", + "models", + Some("models/marts"), + "models/marts/_sqlbuild/constants", + ), + ]), + expected_groups: Ok(&[("models", "models/marts/_sqlbuild")]), + }, + LayoutTestCase { + description: "a declaration group directly below a canonical root fails", + files: &["models/_sqlbuild/macros/text.py"], + directories: &[], + expected_facts: Err( + "Grouped declaration root models/_sqlbuild/ must be below a concrete owner \ + directory; use the project-wide macros/, enums/, constants/, audits/, \ + schemas/, or hooks/ root instead", + ), + expected_groups: Ok(&[]), + }, + LayoutTestCase { + description: "a nested declaration root fails in both scans", + files: &["models/a/_enums/macros/x.py"], + directories: &["schemas/enums"], + expected_facts: Err( + "Declaration root models/a/_enums/macros/ is nested inside another declaration \ + tree", + ), + expected_groups: Err( + "Declaration directory schemas/enums/ is not allowed inside the project-wide \ + schemas/ role; scoped declarations belong under /_sqlbuild/ below a \ + resource tree", + ), + }, + LayoutTestCase { + description: "unsupported group and role entries list every entry", + files: &[ + "models/a/_sqlbuild/notes.txt", + "audits/x.sql", + "audits/.keep", + ], + directories: &["models/a/_sqlbuild/extra"], + expected_facts: Err( + "Declaration group models/a/_sqlbuild/ contains unsupported entries: \ + models/a/_sqlbuild/extra, models/a/_sqlbuild/notes.txt", + ), + expected_groups: Err( + "Unsupported entries in audits/: audits/x.sql; audit files must live in \ + audits/generic/ or audits/singular/", + ), + }, + LayoutTestCase { + description: "local singular audits and unknown hook roles fail the named layout", + files: &[], + directories: &[ + "models/a/_sqlbuild/_audits/singular", + "models/b/_sqlbuild/hooks/x", + ], + expected_facts: Ok(&[]), + expected_groups: Err( + "models/a/_sqlbuild/_audits/singular/ is invalid: singular audits are never used \ + by name, so folder-only visibility has no meaning; use \ + _sqlbuild/audits/singular/", + ), + }, + LayoutTestCase { + description: "a scoped root at the project root fails", + files: &["_macros/text.py"], + directories: &[], + expected_facts: Err( + "Scoped declaration root _macros/ must be below a canonical authored root", + ), + expected_groups: Ok(&[]), + }, + ]; + for test_case in test_cases { + assert_eq!( + layout_rows(&test_case), + ( + owned_facts(test_case.expected_facts), + owned_groups(test_case.expected_groups) + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs new file mode 100644 index 0000000000..458cbc4004 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs @@ -0,0 +1,18 @@ +pub(super) struct LayoutTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) directories: &'static [&'static str], + /// `(relative path, kind, scope kind, ownership root, owning path, declaration root)`. + pub(super) expected_facts: Result<&'static [FactRow], &'static str>, + /// `(canonical root, group directory)`. + pub(super) expected_groups: Result<&'static [(&'static str, &'static str)], &'static str>, +} + +pub(super) type FactRow = ( + &'static str, + &'static str, + &'static str, + &'static str, + Option<&'static str>, + &'static str, +); diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 11e9b34f54..a24a60344a 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -4,6 +4,7 @@ pub(crate) mod _helpers; pub mod constants; +pub mod declarations; pub mod model_files; pub mod models; pub mod tree; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 9a933a7b46..a8a4f167e5 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -8,6 +8,8 @@ use std::path::PathBuf; pub enum FailureKind { /// `ModelSqlParseError` (D002). ModelSql, + /// `DeclarationParseError` (D013). + Declaration, } impl FailureKind { @@ -15,6 +17,7 @@ impl FailureKind { pub fn as_str(self) -> &'static str { match self { Self::ModelSql => "model_sql", + Self::Declaration => "declaration", } } } diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs index 7fba2181b3..0aa7327497 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs @@ -8,14 +8,6 @@ use std::sync::Arc; pub(crate) type WalkedDirectory = (String, Arc>); -/// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. -pub(crate) fn is_dir(tree: &ProjectTree, relative_path: &str) -> bool { - match std::fs::metadata(tree.absolute(relative_path)) { - Ok(metadata) => metadata.is_dir(), - Err(_unreadable) => false, - } -} - /// Every directory below `directory`, with its listing, descending only into real directories. pub(crate) fn walk( tree: &ProjectTree, @@ -36,7 +28,10 @@ pub(crate) fn walk( Ok(walked) } -fn listing(tree: &ProjectTree, directory: &str) -> Result>, StageDeferral> { +pub(crate) fn listing( + tree: &ProjectTree, + directory: &str, +) -> Result>, StageDeferral> { if let Some(cached) = cached_listing(tree, directory) { return Ok(cached); } diff --git a/crates/sqlbuild-discovery/src/tree/main/children.rs b/crates/sqlbuild-discovery/src/tree/main/children.rs new file mode 100644 index 0000000000..f31bcf2130 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/children.rs @@ -0,0 +1,25 @@ +//! The entries of one directory, as `sorted(directory.iterdir())` sees them. + +use crate::models::StageDeferral; +use crate::tree::_helpers::listing::join_relative; +use crate::tree::_helpers::ordering::compare_relative_paths; +use crate::tree::_helpers::walking::listing; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// `sorted(directory.iterdir())` with relative paths; an unreadable directory defers. +pub fn children( + tree: &ProjectTree, + directory: &str, +) -> Result, StageDeferral> { + if let Err(error) = std::fs::read_dir(tree.absolute(directory)) { + return Err(StageDeferral { + reason: format!("cannot list {directory}: {error}"), + }); + } + let mut entries: Vec<(String, TreeEntry)> = listing(tree, directory)? + .iter() + .map(|entry| (join_relative(directory, &entry.name), entry.clone())) + .collect(); + entries.sort_by(|left, right| compare_relative_paths(&left.0, &right.0)); + Ok(entries) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/directories.rs b/crates/sqlbuild-discovery/src/tree/main/directories.rs new file mode 100644 index 0000000000..b9928dbe5c --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/directories.rs @@ -0,0 +1,10 @@ +//! `sorted(path for path in DirectorySnapshot.directories(root))` over the shared snapshot. + +use crate::models::StageDeferral; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; + +/// Every directory below `root`, including unwalked directory links, in Python's path order. +pub fn directories(tree: &ProjectTree, root: &str) -> Result, StageDeferral> { + rglob(tree, root, |entry| entry.is_dir) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs index cb7cc0dc9a..d45f892679 100644 --- a/crates/sqlbuild-discovery/src/tree/main/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -1,2 +1,5 @@ +pub mod children; +pub mod directories; pub mod listings; +pub mod read_tree; pub mod rglob; diff --git a/crates/sqlbuild-discovery/src/tree/main/read_tree.rs b/crates/sqlbuild-discovery/src/tree/main/read_tree.rs new file mode 100644 index 0000000000..00046ff610 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/read_tree.rs @@ -0,0 +1,13 @@ +//! Read every listing below a root into the shared snapshot. + +use crate::models::StageDeferral; +use crate::tree::_helpers::walking::walk; +use crate::tree::models::ProjectTree; + +/// List `root` and every real directory below it; a missing root reads nothing. +pub fn read_tree(tree: &ProjectTree, root: &str) -> Result<(), StageDeferral> { + if tree.is_dir(root) { + let _walked = walk(tree, root)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/rglob.rs b/crates/sqlbuild-discovery/src/tree/main/rglob.rs index 223cc056c1..ef9c44cec5 100644 --- a/crates/sqlbuild-discovery/src/tree/main/rglob.rs +++ b/crates/sqlbuild-discovery/src/tree/main/rglob.rs @@ -3,7 +3,7 @@ use crate::models::StageDeferral; use crate::tree::_helpers::listing::join_relative; use crate::tree::_helpers::ordering::compare_relative_paths; -use crate::tree::_helpers::walking::{WalkedDirectory, is_dir, walk}; +use crate::tree::_helpers::walking::{WalkedDirectory, walk}; use crate::tree::models::{ProjectTree, TreeEntry}; /// Every entry below `root` that `accept` keeps, in Python's sorted path order. @@ -12,7 +12,7 @@ pub fn rglob( root: &str, accept: impl Fn(&TreeEntry) -> bool, ) -> Result, StageDeferral> { - if !is_dir(tree, root) { + if !tree.is_dir(root) { return Ok(Vec::new()); } let mut selected: Vec = Vec::new(); diff --git a/crates/sqlbuild-discovery/src/tree/models.rs b/crates/sqlbuild-discovery/src/tree/models.rs index 96d24a7eca..93ebadc3a2 100644 --- a/crates/sqlbuild-discovery/src/tree/models.rs +++ b/crates/sqlbuild-discovery/src/tree/models.rs @@ -37,4 +37,20 @@ impl ProjectTree { .filter(|part| !part.is_empty()) .fold(self.directory.clone(), |path, part| path.join(part)) } + + /// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. + pub fn is_dir(&self, relative_path: &str) -> bool { + match std::fs::metadata(self.absolute(relative_path)) { + Ok(metadata) => metadata.is_dir(), + Err(_unreadable) => false, + } + } + + /// Python's `Path.exists()`: follows symbolic links and reports any failure as `false`. + pub fn exists(&self, relative_path: &str) -> bool { + match std::fs::metadata(self.absolute(relative_path)) { + Ok(_metadata) => true, + Err(_missing) => false, + } + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index 30ddb6d43a..b356bab861 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -2,19 +2,25 @@ use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; +use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; +use sqlbuild_discovery::declarations::models::{ + DeclarationFileFact, DeclarationGroup, DeclarationLayout, +}; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{ - DiscoveredFile, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, + DiscoveredFile, DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, }; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; -use std::path::PathBuf; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; use crate::bindings::_helpers::functions::map_to_python; -use crate::bindings::models::ModelDiscoveryRequest; +use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree}; use crate::bindings::types::CompilerDetach; type PyObject = Py; @@ -86,7 +92,15 @@ fn files_object( } type Listing = (String, Vec<(String, bool, bool)>); -type ModelDiscovery = (Vec<(String, PyObject)>, Vec); +type FactRow = ( + String, + &'static str, + &'static str, + String, + Option, + String, +); +type DeclarationLayoutRows = (Option>, Option>); fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { entries @@ -95,19 +109,58 @@ fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { .collect() } -fn listing_rows(tree: &ProjectTree) -> Vec { - read_listings(tree) - .into_iter() - .map(|(directory, entries)| (directory, entry_rows(&entries))) - .collect() +fn fact_row(fact: DeclarationFileFact) -> FactRow { + ( + fact.relative_path, + fact.kind.as_str(), + fact.scope_kind.as_str(), + fact.ownership_root, + fact.owning_path, + fact.declaration_root, + ) +} + +fn group_row(group: DeclarationGroup) -> (String, String) { + (group.root, group.directory) } -/// Discover the model files natively with the listings the walk read, or `None` to defer. +fn valid_rows(outcome: Result, DiscoveryFailure>, row: fn(T) -> R) -> Option> { + match outcome { + Ok(items) => Some(items.into_iter().map(row).collect()), + Err(_failure) => None, + } +} + +#[pymethods] +impl NativeProjectTree { + #[new] + fn new(project_dir: &str) -> Self { + Self { + inner: ProjectTree::new(Path::new(project_dir)), + exported: Mutex::new(HashSet::new()), + } + } + + /// The listings read since the last call, for the pass's Python directory snapshot. + fn listings(&self) -> Vec { + let Ok(mut exported) = self.exported.lock() else { + return Vec::new(); + }; + read_listings(&self.inner) + .into_iter() + .filter(|(directory, _)| exported.insert(directory.clone())) + .map(|(directory, entries)| (directory, entry_rows(&entries))) + .collect() + } +} + +/// Discover the model files natively, or return `None` when Python must run the stage. #[pyfunction] fn discover_model_files( py: Python<'_>, request: ModelDiscoveryRequest, -) -> PyResult> { + tree: &NativeProjectTree, +) -> PyResult>> { let root = ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -118,24 +171,37 @@ fn discover_model_files( extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, }; - let (discovered, listings): ( - Result>, StageDeferral>, - Vec, - ) = py - .compiler_detach(|| { - let tree = ProjectTree::new(&root.directory); - let discovered = discover_models(&root, &tree, &options); - Ok((discovered, listing_rows(&tree))) - }) + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::panics::compiler_error)?; match discovered { - Ok(files) => Ok(Some((files_object(py, files, model_object)?, listings))), + Ok(files) => Ok(Some(files_object(py, files, model_object)?)), + Err(_deferral) => Ok(None), + } +} + +/// The valid declaration file facts and named declaration groups; `None` where Python must scan. +#[pyfunction] +fn discover_declaration_layout( + py: Python<'_>, + tree: &NativeProjectTree, +) -> PyResult> { + let layout: Result = py + .compiler_detach(|| Ok(declaration_layout(&tree.inner))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match layout { + Ok(layout) => Ok(Some(( + valid_rows(layout.file_facts, fact_row), + valid_rows(layout.named_groups, group_row), + ))), Err(_deferral) => Ok(None), } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; module.add( "PYTHON_ALNUM_UNICODE_VERSION", python_alnum_unicode_version(), diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index b9982f238c..72ab04cfc7 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -97,6 +97,15 @@ impl From for validation::PositionInput { } } +/// The directory listings one Python discovery pass shares across its native calls. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeProjectTree { + pub(crate) inner: sqlbuild_discovery::tree::models::ProjectTree, + /// Directories whose listings were already handed to the Python snapshot. + pub(crate) exported: Mutex>, +} + /// One native model discovery request from the Python discovery facade. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index c2cf26daa0..858e2864ee 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -25,6 +25,7 @@ ' """Log one refresh."""\n' " return None\n" ) +_MACRO_MODULE: str = "def cents(value):\n return f'{value} * 100'\n" _SEED_DECLARATION: str = ( "seeds:\n - name: order_channels\n description: Order channels.\n" " columns:\n - name: id\n type: INTEGER\n" @@ -48,6 +49,7 @@ def discovery_failure_cases() -> tuple[FailureCase, ...]: return ( *_config_cases(), *_sql_file_cases(), + *_layout_cases(), *_yaml_file_cases(), *_conflict_cases(), *_python_cases(), @@ -185,6 +187,22 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) +def _layout_cases() -> tuple[FailureCase, ...]: + return tuple( + _case(name=name, expected_code="D013", files={path: contents}) + for name, path, contents in ( + ("scoped-root-at-project-root", "_enums/order_status.sql", ""), + ("declaration-group-below-root", "models/_sqlbuild/macros/money.py", _MACRO_MODULE), + ("declaration-root-nested", "models/marts/_enums/_constants/limits.sql", ""), + ("declaration-group-unsupported-entry", "models/marts/_sqlbuild/notes.txt", ""), + ("named-role-nested-declaration", "schemas/enums/order_status.sql", ""), + ("audit-role-unsupported-entry", "audits/at_least.sql", ""), + ("local-singular-audit", "models/marts/_sqlbuild/_audits/singular/at_least.sql", ""), + ("hook-role-unsupported-entry", "models/marts/_sqlbuild/hooks/notes.sql", ""), + ) + ) + + def _yaml_file_cases() -> tuple[FailureCase, ...]: return ( _case( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 70e969ca26..8e89ea555a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -46,10 +46,20 @@ def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... PYTHON_ALNUM_UNICODE_VERSION: str +class NativeProjectTree: + def __init__(self, project_dir: str) -> None: ... + def listings(self) -> list[tuple[str, list[tuple[str, bool, bool]]]]: ... + def discover_model_files( - request: dict[str, object], + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... +def discover_declaration_layout( + tree: NativeProjectTree, ) -> ( - tuple[list[tuple[str, tuple[object, ...]]], list[tuple[str, list[tuple[str, bool, bool]]]]] + tuple[ + list[tuple[str, str, str, str, str | None, str]] | None, + list[tuple[str, str]] | None, + ] | None ): ... def substitute_static_project_vars( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 6de7587bd5..07844fe457 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -34,6 +34,9 @@ from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( build_integration_loader_functions, ) +from sqlbuild.compiler.discovery._helpers.native.declarations import ( + prepare_native_declaration_layout, +) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) @@ -55,6 +58,7 @@ DiscoveredProjectInputs, DiscoveredProvider, DiscoveredPythonNodeFunctions, + DiscoveredSourceFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestFile, @@ -96,6 +100,7 @@ def build_discovered_project_inputs( project_dir=project_dir, discover_models=discover_models, fact_cache=fact_cache, + native=native, ) declarations: DiscoveredDeclarationFiles = ( discover() @@ -214,11 +219,18 @@ def _discover_declaration_files( project_dir: Path, discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], fact_cache: FactCacheStore | None, + native: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): + source_files: tuple[DiscoveredSourceFile, ...] = discover_source_files( + project_dir=project_dir, fact_cache=fact_cache + ) + model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() + if native: + prepare_native_declaration_layout(project_dir=project_dir) return DiscoveredDeclarationFiles( - source_files=discover_source_files(project_dir=project_dir, fact_cache=fact_cache), - model_files=discover_models(), + source_files=source_files, + model_files=model_files, enum_files=discover_enum_files(project_dir=project_dir), constant_files=discover_constant_files(project_dir=project_dir), model_schema_files=discover_model_schema_files(project_dir=project_dir), diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index a4ec31738f..a93247e3a1 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -94,7 +94,7 @@ ModelHeaderMatch, NamedDeclarationRoot, ) -from sqlbuild.compiler.discovery.types import ScopedDeclarationFile +from sqlbuild.compiler.discovery.types import NativeDeclarationFact, ScopedDeclarationFile from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, @@ -195,6 +195,25 @@ def _discover_declaration_file_facts( return facts +def remember_declaration_file_facts( + *, project_dir: Path, facts: Iterable[NativeDeclarationFact] +) -> None: + """Record a scan of every declaration kind another walk of this pass already validated.""" + + DirectorySnapshot.current(project_dir=project_dir).memo[(_FACTS_MEMO_KEY, None)] = tuple( + _DeclarationFileFacts( + file_path=project_dir / relative_path, + relative_path=Path(relative_path), + declaration_kind=DeclarationKind(kind), + scope_kind=ScopeKind(scope_kind), + ownership_root=Path(ownership_root), + owning_path=None if owning_path is None else Path(owning_path), + declaration_root=Path(declaration_root), + ) + for relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root in facts + ) + + def _scan_declaration_file_facts( *, project_dir: Path, declaration_kind: DeclarationKind | None, tree: DirectorySnapshot ) -> tuple[_DeclarationFileFacts, ...]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index 9d0156d070..0439371b7c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from pathlib import Path from typing import cast @@ -25,6 +25,7 @@ _SINGULAR_AUDIT_DIRECTORY: str = "singular" _LAYOUT_MEMO_KEY: str = "named_declaration_layout" _GROUPS_MEMO_KEY: str = "declaration_groups" +_PATH_SEPARATOR: str = "/" _NESTED_ROLE_DIRECTORIES: frozenset[str] = frozenset( {DECLARATION_GROUP_DIRECTORY} | INHERITED_DECLARATION_DIRECTORIES @@ -86,6 +87,16 @@ def named_declaration_files( yield root, file_path +def remember_declaration_groups(*, project_dir: Path, groups: Iterable[tuple[str, str]]) -> None: + """Record a named layout another walk of this pass validated, with its declaration groups.""" + + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + tree.memo[_GROUPS_MEMO_KEY] = tuple( + (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) for root, directory in groups + ) + tree.memo[_LAYOUT_MEMO_KEY] = True + + def validate_named_declaration_layout(*, project_dir: Path) -> None: """Reject entries that no named declaration role accepts.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py new file mode 100644 index 0000000000..2bb75590b7 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py @@ -0,0 +1,39 @@ +"""The declaration layout walked and validated natively for the Python declaration collections.""" + +from __future__ import annotations + +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import remember_declaration_file_facts +from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( + remember_declaration_groups, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery.types import NativeDeclarationFact + + +def prepare_native_declaration_layout(*, project_dir: Path) -> None: + """Record a natively validated declaration layout; Python rescans an invalid one to raise.""" + + if not native_discovery_supported( + project_dir=project_dir, display_prefix=native_display_prefix(project_dir) + ): + return + tree: _native.NativeProjectTree = native_project_tree(project_dir) + layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( + _native.discover_declaration_layout(tree) + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + if layout is None: + return + facts, groups = layout + if facts is not None: + remember_declaration_file_facts(project_dir=project_dir, facts=facts) + if groups is not None: + remember_declaration_groups(project_dir=project_dir, groups=groups) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index 9061a7e797..a50aa42d3f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -15,6 +15,7 @@ native_display_prefix, native_failure, native_locations, + native_project_tree, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -32,7 +33,7 @@ SQL_MODEL_HEADER_KEYS, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.discovery.types import NativeLocation, NativeModelDiscovery +from sqlbuild.compiler.discovery.types import NativeLocation def discover_native_model_files( @@ -44,28 +45,22 @@ def discover_native_model_files( """Discover model files natively; Python runs when native discovery cannot match it.""" display_prefix: str = native_display_prefix(project_dir) - discovered: NativeModelDiscovery | None = ( - _native.discover_model_files( - { - "project_dir": str(project_dir), - "display_prefix": display_prefix, - "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), - "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), - "extract_implicit_alias_columns": extract_implicit_alias_columns, - "extract_output_column_locations": extract_output_column_locations, - } + files: list[tuple[str, tuple[object, ...]]] | None = ( + _discover_native_files( + project_dir=project_dir, + display_prefix=display_prefix, + extract_implicit_alias_columns=extract_implicit_alias_columns, + extract_output_column_locations=extract_output_column_locations, ) if native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix) else None ) - if discovered is None: + if files is None: return discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=extract_implicit_alias_columns, extract_output_column_locations=extract_output_column_locations, ) - files, listings = discovered - seed_snapshot_listings(project_dir=project_dir, listings=listings) return tuple( _model_file( project_dir=project_dir, @@ -78,6 +73,29 @@ def discover_native_model_files( ) +def _discover_native_files( + *, + project_dir: Path, + display_prefix: str, + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, +) -> list[tuple[str, tuple[object, ...]]] | None: + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: list[tuple[str, tuple[object, ...]]] | None = _native.discover_model_files( + { + "project_dir": str(project_dir), + "display_prefix": display_prefix, + "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), + "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), + "extract_implicit_alias_columns": extract_implicit_alias_columns, + "extract_output_column_locations": extract_output_column_locations, + }, + tree, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return files + + def _model_file( *, project_dir: Path, diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 23d13f459a..f37989a615 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -19,6 +19,7 @@ _DISPLAY_PROBE: str = "_" _WINDOWS_OS_NAME: str = "nt" _PATH_ENCODING: str = "utf-8" +_NATIVE_TREE_MEMO_KEY: str = "native_project_tree" _FAILURE_CLASSES: dict[str, type[DiscoveryError]] = {"model_sql": ModelSqlParseError} @@ -64,9 +65,22 @@ def native_discovery_supported(*, project_dir: Path, display_prefix: str) -> boo ) -def seed_snapshot_listings(*, project_dir: Path, listings: list[NativeListing]) -> None: +def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: + """Return the native listings shared by every native call of this discovery pass.""" + + snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + tree: object = snapshot.memo.get(_NATIVE_TREE_MEMO_KEY) + if isinstance(tree, _native.NativeProjectTree): + return tree + created: _native.NativeProjectTree = _native.NativeProjectTree(str(project_dir)) + snapshot.memo[_NATIVE_TREE_MEMO_KEY] = created + return created + + +def seed_snapshot_listings(*, project_dir: Path, tree: _native.NativeProjectTree) -> None: """Share the native walk's listings with the pass's Python directory snapshot.""" + listings: list[NativeListing] = tree.listings() DirectorySnapshot.current(project_dir=project_dir).seed_listings( {project_dir / directory: _snapshot_entries(entries) for directory, entries in listings} ) diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 2c91d5429a..980b716ff4 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -20,7 +20,7 @@ type ProjectProvider = Provider type NativeLocation = tuple[str, int, int, int, int] type NativeListing = tuple[str, list[tuple[str, bool, bool]]] -type NativeModelDiscovery = tuple[list[tuple[str, tuple[object, ...]]], list[NativeListing]] +type NativeDeclarationFact = tuple[str, str, str, str, str | None, str] class LoaderConnectionMode(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 92f2242ec9..a2229f4dc0 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -94,3 +94,15 @@ class SharedSnapshotTestCase: created_file: str pattern: str expected_matches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class GeneratedLayoutParityTestCase: + """Seeded declaration layouts whose native validation and facts must match Python's.""" + + description: str + seed: int + count: int + expected_mismatches: tuple[str, ...] = () + expected_minimum_valid: int = 0 + expected_minimum_invalid: int = 0 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 69df632906..39e05eb34f 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -12,19 +12,39 @@ from contextlib import suppress from itertools import compress from pathlib import Path -from typing import Any +from typing import Any, cast +import pytest import yaml from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_constant_files, + discover_enum_files, + discover_macro_files, + discover_model_files, +) +from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( + named_declaration_roots, + remember_declaration_groups, +) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.discovery.exceptions import DeclarationParseError +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import ( + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredSqlModelFile, +) +from sqlbuild.compiler.discovery.types import NativeDeclarationFact from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.scopes.constants import NAMED_DECLARATION_KINDS from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text @@ -706,6 +726,160 @@ def generated_model_bytes(*, rng: random.Random) -> bytes: return text.encode("utf-8") +LAYOUT_ROOTS: tuple[str, ...] = ("models",) * 8 + ( + "tests/unit", "tests/scenarios", "functions/sql", "sources", "macros", "enums", "constants", + "audits", "schemas", "hooks", "_macros", "_sqlbuild", "seeds", +) # fmt: skip +LAYOUT_SEGMENTS: tuple[str, ...] = ("marts", "core") * 6 + ( + "_sqlbuild", "_sqlbuild", "_sqlbuild", "macros", "enums", "constants", "_macros", "_enums", + "_constants", "audits", "_audits", "generic", "singular", "schemas", "_schemas", "hooks", + "_hooks", "sql", "python", ".cache", +) # fmt: skip +LAYOUT_TEMPLATES: tuple[str, ...] = ( + "{root}/{owner}/_sqlbuild/{declaration}", "{root}/{owner}/{local}", "{declaration}", + "{declaration}/{owner}", "{root}/{owner}/_sqlbuild/{named}", "{named}", "{root}/{owner}", +) # fmt: skip +LAYOUT_DECLARATIONS: tuple[str, ...] = ( + "macros", "enums", "constants", "_macros", "_enums", "_constants", +) # fmt: skip +LAYOUT_NAMED_ROLES: tuple[str, ...] = ( + "audits/generic", "audits/singular", "_audits/generic", "schemas", "_schemas", "hooks/sql", + "hooks/python", "_hooks/sql", +) # fmt: skip +LAYOUT_FILE_NAMES: tuple[str, ...] = ("a.py", "b.sql", "c.sql", "__init__.py", "notes.txt", ".keep") +LAYOUT_CONTENTS: dict[str, str] = { + "enums": "ENUM (name e{n}, members [A]);\n", + "_enums": "ENUM (name e{n}, members [A]);\n", + "constants": "CONSTANT (name c{n}, value 1);\n", + "_constants": "CONSTANT (name c{n}, value 1);\n", +} +_FACT_KINDS: dict[type[object], str] = { + DiscoveredMacroFile: "macro", + DiscoveredEnumFile: "enum", + DiscoveredConstantFile: "constant", +} + + +def generated_layout_files(*, rng: random.Random) -> tuple[tuple[str, bytes], ...]: + """Return files spread over canonical, global and grouped declaration roots.""" + + files: dict[str, bytes] = {} + for index in range(rng.randint(1, 6)): + directory: str = rng.choices( + ( + rng.choice(LAYOUT_TEMPLATES).format( + root=rng.choice(LAYOUT_ROOTS[:12]), + owner=rng.choice(("marts", "core", "marts/core")), + declaration=rng.choice(LAYOUT_DECLARATIONS), + local=rng.choice(LAYOUT_DECLARATIONS[3:]), + named=rng.choice(LAYOUT_NAMED_ROLES), + ), + "/".join( + (rng.choice(LAYOUT_ROOTS), *rng.choices(LAYOUT_SEGMENTS, k=rng.randint(0, 3))) + ), + ), + weights=(6, 1), + )[0] + parts: list[str] = [*directory.split("/"), rng.choice(LAYOUT_FILE_NAMES)] + template: str = next( + filter(None, (LAYOUT_CONTENTS.get(part) for part in reversed(parts))), "" + ) + files["/".join(parts)] = template.format(n=index).encode("utf-8") + return tuple(files.items()) + + +def python_layout_outcome(*, project_dir: Path) -> tuple[object, object]: + """Return Python's declaration file facts and named roots, `None` where its scan raises.""" + + return ( + _outcome_or_none(lambda: _python_fact_rows(project_dir=project_dir)), + _outcome_or_none(lambda: _named_root_rows(project_dir=project_dir, seed=lambda: None)), + ) + + +def native_layout_outcome(*, project_dir: Path) -> tuple[object, object]: + """Return the native facts and the named roots its groups give, `None` where it fails.""" + + layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( + _native.discover_declaration_layout(_native.NativeProjectTree(str(project_dir))) + ) + facts, groups = layout or (None, None) + return ( + _mapped(facts, lambda rows: [_native_fact_row(fact) for fact in rows]), + _mapped( + groups, + lambda rows: _named_root_rows( + project_dir=project_dir, + seed=lambda: remember_declaration_groups(project_dir=project_dir, groups=rows), + ), + ), + ) + + +def _mapped[T](value: T | None, transform: Callable[[T], object]) -> object: + return {True: lambda: None, False: lambda: transform(cast(T, value))}[value is None]() + + +def _outcome_or_none(compute: Callable[[], object]) -> object: + outcome: list[object] = [None] + with suppress(DeclarationParseError): + outcome[0] = compute() + return outcome[0] + + +def _python_fact_rows(*, project_dir: Path) -> list[tuple[str, ...]]: + with DirectorySnapshot.scope(project_dir=project_dir): + files: tuple[DiscoveredMacroFile | DiscoveredEnumFile | DiscoveredConstantFile, ...] = ( + *discover_macro_files(project_dir=project_dir), + *discover_enum_files(project_dir=project_dir), + *discover_constant_files(project_dir=project_dir), + ) + return sorted( + ( + file.relative_path.as_posix(), + _FACT_KINDS[type(file)], + file.scope_kind.value, + str(file.ownership_root), + str(file.owning_path), + str(file.declaration_root), + ) + for file in files + ) + + +def _native_fact_row(fact: NativeDeclarationFact) -> tuple[str, ...]: + relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root = fact + return (relative_path, kind, scope_kind, ownership_root, str(owning_path), declaration_root) + + +def _named_root_rows(*, project_dir: Path, seed: Callable[[], None]) -> list[tuple[str, ...]]: + with DirectorySnapshot.scope(project_dir=project_dir): + seed() + return [ + ( + root.relative_directory.as_posix(), + root.kind.value, + root.scope_kind.value, + str(root.ownership_root), + str(root.owning_path), + ) + for root in named_declaration_roots( + project_dir=project_dir, kinds=NAMED_DECLARATION_KINDS + ) + ] + + +def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch) -> object: + """Return the rendered discovery stage under `engine` with its failure type and message.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with capture: + rendered[0] = render_stage_capture(discover_project_inputs(project_dir=project_dir)) + return (rendered[0], type(capture.failure).__name__, str(capture.failure)) + + class CallCounter: """Count calls and delegate to the wrapped function.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py new file mode 100644 index 0000000000..a3b9b3b16d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py @@ -0,0 +1,117 @@ +"""Engine parity for the declaration layout native discovery walks and validates.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + GeneratedLayoutParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + generated_layout_files, + native_layout_outcome, + python_layout_outcome, + stage_outcome, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_AUDIT: bytes = b'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' +_SCHEMA: bytes = ( + b'SCHEMA (\n name order_shape,\n description "Order shape",\n' + b" columns (\n id (type INTEGER),\n ),\n);\n" +) +_HOOK: bytes = b'HOOK (\n description "Record a refresh"\n);\n\nSELECT 1 AS refreshed\n' +_MACRO: bytes = b"def cents(value):\n return f'{value} * 100'\n" +_MODEL: bytes = b'MODEL (description "Orders");\nSELECT 1 AS order_id' + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="global, inherited, local and grouped declarations of every kind", + files=( + ("models/marts/orders.sql", _MODEL), + ("macros/money.py", _MACRO), + ("macros/__init__.py", b""), + ("enums/status.sql", b"ENUM (name order_status, members [PLACED]);\n"), + ("models/marts/_enums/tier.sql", b"ENUM (name tier, members [GOLD]);\n"), + ("models/marts/_sqlbuild/constants/limits.sql", b"CONSTANT (name cap, value 3);\n"), + ("models/marts/_sqlbuild/_audits/generic/is_true.sql", _AUDIT), + ("models/marts/_sqlbuild/_schemas/order_shape.sql", _SCHEMA), + ("models/marts/_sqlbuild/hooks/sql/record_refresh.sql", _HOOK), + ("audits/generic/is_true.sql", _AUDIT), + ("schemas/order_shape.sql", _SCHEMA), + ("hooks/sql/_private.sql", _HOOK), + ("functions/sql/_macros/helpers.py", _MACRO), + ("seeds/channels.csv", b"id\n1\n"), + ), + ), + EngineSwitchParityTestCase( + description="a nested declaration root fails with Python's error", + files=(("models/marts/_enums/macros/money.py", _MACRO),), + ), + EngineSwitchParityTestCase( + description="an unsupported audit role entry fails with Python's error", + files=(("models/marts/orders.sql", _MODEL), ("audits/is_true.sql", _AUDIT)), + ), + EngineSwitchParityTestCase( + description="an unreadable declaration file fails with Python's read error", + files=(("enums/status.sql", b"\xff"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_layout_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedLayoutParityTestCase( + description="seeded layouts over every declaration role", + seed=20261007, + count=400, + expected_minimum_valid=60, + expected_minimum_invalid=60, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_layouts_when_validating_natively_then_facts_and_failures_match_python( + test_case: GeneratedLayoutParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + project_dirs: list[Path] = [tmp_path / f"project_{index}" for index in range(test_case.count)] + for project_dir in project_dirs: + write_project(project_dir=project_dir, files=generated_layout_files(rng=rng)) + expected: list[tuple[object, object]] = [ + python_layout_outcome(project_dir=path) for path in project_dirs + ] + + actual: list[tuple[object, object]] = [ + native_layout_outcome(project_dir=path) for path in project_dirs + ] + + valid: int = sum(None not in outcome for outcome in expected) + inputs: list[object] = [path.name for path in project_dirs] + expected_outcomes: list[object] = [*expected] + actual_outcomes: list[object] = [*actual] + assert ( + mismatches(inputs=inputs, expected=expected_outcomes, actual=actual_outcomes), + valid >= test_case.expected_minimum_valid, + test_case.count - valid >= test_case.expected_minimum_invalid, + ) == (list(test_case.expected_mismatches), True, True), test_case.description From 4a3221c012718acab3dd94da06fcb37764f602f7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:27:13 +0100 Subject: [PATCH 2/3] feat(native): discover SQL unit-test and scenario files natively With the native engine, Rust selects, reads and splits SQL test files into TEST blocks, parses each header and rejects unsupported keys, and cleans bodies like inspect.cleandoc; scenario files get the same header handling. Python keeps the block and scenario semantics, including parameter value normalisation, so the first failure is unchanged. --- .../src/text/_helpers/cleandoc.rs | 34 ++++ crates/sqlbuild-core/src/text/_helpers/mod.rs | 1 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + .../src/text/main/python_cleandoc.rs | 34 ++++ .../{model_files => }/_helpers/header_keys.rs | 4 +- crates/sqlbuild-discovery/src/_helpers/mod.rs | 2 + .../src/_helpers/statement_headers.rs | 66 +++++++ crates/sqlbuild-discovery/src/constants.rs | 2 + crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/model_files/_helpers/mod.rs | 1 - .../src/model_files/_helpers/parsing.rs | 3 +- crates/sqlbuild-discovery/src/models.rs | 6 + .../src/sql_tests/_helpers/mod.rs | 3 + .../sql_tests/_helpers/scenario_parsing.rs | 45 +++++ .../src/sql_tests/_helpers/selection.rs | 39 ++++ .../src/sql_tests/_helpers/test_blocks.rs | 114 ++++++++++++ .../sql_tests/main/discover_scenario_files.rs | 19 ++ .../sql_tests/main/discover_sql_test_files.rs | 22 +++ .../src/sql_tests/main/mod.rs | 2 + .../sqlbuild-discovery/src/sql_tests/mod.rs | 6 + .../src/sql_tests/models.rs | 35 ++++ .../sqlbuild-discovery/src/sql_tests/tests.rs | 6 + .../src/sql_tests/tests/helpers.rs | 78 ++++++++ .../sql_tests/tests/test_sql_test_files.rs | 74 ++++++++ .../src/sql_tests/tests/test_types.rs | 7 + .../src/bindings/_helpers/discovery.rs | 121 +++++++++++-- .../src/bindings/_helpers/oracles.rs | 7 + crates/sqlbuild-python/src/bindings/models.rs | 10 ++ .../_helpers/model_headers/matching.rs | 15 +- .../sqlbuild-sqltext/src/compiler/main/mod.rs | 1 + .../main/statement_header_matching.rs | 10 ++ src/sqlbuild/_native.pyi | 7 + .../_helpers/filesystem/aggregation.py | 16 +- .../discovery/_helpers/native/payloads.py | 15 +- .../_helpers/native/sql_test_files.py | 169 ++++++++++++++++++ .../discovery/_helpers/sql/scenarios.py | 44 +++-- .../compiler/discovery/_helpers/sql/tests.py | 52 +++--- src/sqlbuild/compiler/discovery/constants.py | 4 + .../compiler/discovery/main/discover.py | 34 ++-- .../src/sqlbuild/compiler/_test_types.py | 10 ++ .../compiler/discovery/_test_types.py | 12 ++ .../sqlbuild/compiler/discovery/helpers.py | 88 +++++++++ .../test_native_sql_test_discovery.py | 117 ++++++++++++ .../src/sqlbuild/compiler/helpers.py | 10 ++ .../test_native_python_compat_oracle.py | 30 ++++ 45 files changed, 1295 insertions(+), 82 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/_helpers/cleandoc.rs create mode 100644 crates/sqlbuild-core/src/text/main/python_cleandoc.rs rename crates/sqlbuild-discovery/src/{model_files => }/_helpers/header_keys.rs (98%) create mode 100644 crates/sqlbuild-discovery/src/_helpers/statement_headers.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/mod.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/models.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/tests.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs create mode 100644 crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs create mode 100644 crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py diff --git a/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs b/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs new file mode 100644 index 0000000000..29c84702d6 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/cleandoc.rs @@ -0,0 +1,34 @@ +//! Line helpers behind the `inspect.cleandoc` port. + +const TAB_SIZE: usize = 8; + +/// `str.expandtabs()`: a tab advances to the next multiple of eight columns. +pub(crate) fn expand_tabs(text: &str) -> String { + let mut expanded: String = String::with_capacity(text.len()); + let mut column: usize = 0; + for character in text.chars() { + match character { + '\t' => { + let width = TAB_SIZE - column % TAB_SIZE; + expanded.extend(std::iter::repeat_n(' ', width)); + column += width; + } + '\n' | '\r' => { + expanded.push(character); + column = 0; + } + _ => { + expanded.push(character); + column += 1; + } + } + } + expanded +} + +/// `line[count:]` in code points. +pub(crate) fn skip_chars(line: &str, count: usize) -> &str { + line.char_indices() + .nth(count) + .map_or("", |(index, _)| &line[index..]) +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 54370118d3..579d227f76 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod alnum_ranges; +pub(crate) mod cleandoc; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index c78964b1c4..1c4f886f29 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -4,4 +4,5 @@ pub mod is_python_alnum; pub mod is_python_space; pub mod is_python_word; pub mod python_alnum_unicode_version; +pub mod python_cleandoc; pub mod python_strip; diff --git a/crates/sqlbuild-core/src/text/main/python_cleandoc.rs b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs new file mode 100644 index 0000000000..0619ba36cb --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs @@ -0,0 +1,34 @@ +//! Python's `inspect.cleandoc` (3.12), which SQL test and scenario bodies go through. + +use crate::text::_helpers::cleandoc::{expand_tabs, skip_chars}; +use crate::text::main::is_python_space::is_python_space; + +/// `inspect.cleandoc(text)`: expand tabs, dedent later lines and drop outer empty lines. +pub fn python_cleandoc(text: &str) -> String { + let expanded: String = expand_tabs(text); + let mut lines: Vec<&str> = expanded.split('\n').collect(); + let margin: Option = lines[1..] + .iter() + .filter_map(|line| { + let content = line.trim_start_matches(is_python_space); + (!content.is_empty()).then(|| line.chars().count() - content.chars().count()) + }) + .min(); + lines[0] = lines[0].trim_start_matches(is_python_space); + let mut cleaned: Vec<&str> = Vec::with_capacity(lines.len()); + cleaned.push(lines[0]); + cleaned.extend( + lines[1..] + .iter() + .map(|line| margin.map_or(*line, |margin| skip_chars(line, margin))), + ); + let last: usize = cleaned + .iter() + .rposition(|line| !line.is_empty()) + .map_or(0, |index| index + 1); + let first: usize = cleaned[..last] + .iter() + .position(|line| !line.is_empty()) + .unwrap_or(last); + cleaned[first..last].join("\n") +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs similarity index 98% rename from crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs rename to crates/sqlbuild-discovery/src/_helpers/header_keys.rs index 4623434102..93eefd20f1 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs +++ b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs @@ -7,6 +7,7 @@ use sqlbuild_core::text::main::is_python_word::is_python_word; /// The facts of one header that declares keys outside its supported set. pub(crate) struct UnsupportedKeys<'a> { + pub(crate) kind: FailureKind, pub(crate) statement: &'a str, pub(crate) file_path: &'a str, pub(crate) header: &'a str, @@ -18,6 +19,7 @@ pub(crate) struct UnsupportedKeys<'a> { pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> DiscoveryFailure { let UnsupportedKeys { + kind, statement, file_path, header, @@ -33,7 +35,7 @@ pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> Discovery .filter(|character| **character == '\n') .count(); DiscoveryFailure { - kind: FailureKind::ModelSql, + kind: *kind, message: format!( "{statement} in '{file_path}:{line}' has unsupported keys: {}", keys.join(", ") diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs index d1c99c888e..d7ec3c7a5c 100644 --- a/crates/sqlbuild-discovery/src/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -1,4 +1,6 @@ +pub(crate) mod header_keys; pub(crate) mod pool; pub(crate) mod reading; pub(crate) mod scoped_paths; +pub(crate) mod statement_headers; pub(crate) mod suggestions; diff --git a/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs new file mode 100644 index 0000000000..e19619d021 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs @@ -0,0 +1,66 @@ +//! Parse a statement header and reject its unsupported keys, as Python's discovery does. + +use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// One statement kind's header contract. +pub(crate) struct StatementHeader<'a> { + pub(crate) kind: FailureKind, + /// The name in syntax errors, such as `TEST`. + pub(crate) statement_name: &'a str, + /// The name in unsupported-key errors, such as `TEST()`. + pub(crate) statement: &'a str, + pub(crate) file_path: &'a str, + pub(crate) supported_keys: &'a [String], +} + +/// `parse_header_values` then `reject_unsupported_header_keys`; `header_line` is one-based. +pub(crate) fn parse_statement_header( + contract: &StatementHeader<'_>, + header: &str, + header_line: usize, +) -> Result, DiscoveryFailure> { + let values: Vec<(String, AuthoredValue)> = match parse_one(header) { + (_, _, Some(error)) => return Err(syntax_failure(contract, &error)), + (Some(AuthoredValue::Map(values)), _, None) => values, + _ => { + return Err(syntax_failure( + contract, + "Native MODEL header parser returned neither values nor an error", + )); + } + }; + let unsupported: Vec<&str> = values + .iter() + .map(|(key, _)| key.as_str()) + .filter(|key| !is_supported(contract.supported_keys, key)) + .collect(); + if unsupported.is_empty() { + return Ok(values); + } + Err(unsupported_keys_failure(&UnsupportedKeys { + kind: contract.kind, + statement: contract.statement, + file_path: contract.file_path, + header, + header_line, + keys: &unsupported, + supported_keys: contract.supported_keys, + })) +} + +fn syntax_failure(contract: &StatementHeader<'_>, error: &str) -> DiscoveryFailure { + DiscoveryFailure::new( + contract.kind, + format!( + "{}(...) in '{}' contains invalid SQLBuild header syntax: {error}", + contract.statement_name, contract.file_path + ), + ) +} + +fn is_supported(supported_keys: &[String], key: &str) -> bool { + supported_keys.iter().any(|supported| supported == key) +} diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index c6d8b50f71..32730b15f6 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -25,6 +25,8 @@ pub const CANONICAL_AUTHORED_ROOTS: [&[&str]; 5] = [ ]; /// The SQL test root, whose `macros/` SQL files are tests rather than declarations. pub const SQL_TESTS_ROOT: &[&str] = &["tests", "unit"]; +/// The SQL scenario root. +pub const SQL_SCENARIOS_ROOT: &str = "tests/scenarios"; /// The directory of macro tests below the SQL test root. pub const MACRO_TESTS_DIRECTORY: &str = "macros"; /// Inherited declaration directories, which may also be project-wide global roots. diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index a24a60344a..b93c9f48cc 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -7,4 +7,5 @@ pub mod constants; pub mod declarations; pub mod model_files; pub mod models; +pub mod sql_tests; pub mod tree; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index de1cf65fb0..c9e10edbd5 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod header_keys; pub(crate) mod locations; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index c23cd28716..c402ba4090 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -1,6 +1,6 @@ //! Parse one model file with the checks and messages of Python's `parse_matched_model_sql`. -use crate::model_files::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; +use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::model_files::_helpers::locations::header_column_locations; use crate::model_files::_helpers::output_columns::output_column_locations; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; @@ -54,6 +54,7 @@ pub(crate) fn parse_model_file( .collect(); if !unsupported.is_empty() { return Err(unsupported_keys_failure(&UnsupportedKeys { + kind: FailureKind::ModelSql, statement: MODEL_STATEMENT, file_path, header, diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index a8a4f167e5..6e2ff16226 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -10,6 +10,10 @@ pub enum FailureKind { ModelSql, /// `DeclarationParseError` (D013). Declaration, + /// `SqlTestParseError` (D003). + SqlTest, + /// `SqlScenarioParseError` (D009). + SqlScenario, } impl FailureKind { @@ -18,6 +22,8 @@ impl FailureKind { match self { Self::ModelSql => "model_sql", Self::Declaration => "declaration", + Self::SqlTest => "sql_test", + Self::SqlScenario => "sql_scenario", } } } diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs new file mode 100644 index 0000000000..bee80b3bef --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod scenario_parsing; +pub(crate) mod selection; +pub(crate) mod test_blocks; diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs new file mode 100644 index 0000000000..428efa510c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs @@ -0,0 +1,45 @@ +//! The header half of Python's `parse_sql_scenario_file`. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::sql_tests::models::DiscoveredScenarioFile; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +const SCENARIO_KEYWORD: &str = "SCENARIO"; +const SCENARIO_STATEMENT: &str = "SCENARIO()"; + +pub(crate) fn parse_scenario_file( + file_path: &str, + contents: String, + supported_keys: &[String], +) -> Result { + let Some((header_start, header_end, sql_start)) = + match_statement_header(&contents, 0, SCENARIO_KEYWORD) + else { + return Err(DiscoveryFailure::new( + FailureKind::SqlScenario, + format!( + "SQL scenario '{file_path}' must start with a SCENARIO() header as the first \ + non-whitespace content" + ), + )); + }; + let header_values = parse_statement_header( + &StatementHeader { + kind: FailureKind::SqlScenario, + statement_name: SCENARIO_KEYWORD, + statement: SCENARIO_STATEMENT, + file_path, + supported_keys, + }, + &contents[header_start..header_end], + contents[..header_start].matches('\n').count() + 1, + )?; + let sql_body: String = python_cleandoc(&contents[sql_start..]); + Ok(DiscoveredScenarioFile { + contents, + header_values, + sql_body, + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs new file mode 100644 index 0000000000..6641b2950c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs @@ -0,0 +1,39 @@ +//! Read and parse the unscoped SQL files below one root in parallel, in Python's order. + +use crate::_helpers::pool::discovery_pool; +use crate::_helpers::reading::read_authored_text; +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::constants::SQL_FILE_SUFFIX; +use crate::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, ProjectRoot, StageDeferral}; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +/// Every unscoped `*.sql` file below `directory`, read and handed to `parse`. +pub(crate) fn discover_sql_files( + root: &ProjectRoot, + tree: &ProjectTree, + directory: &str, + parse: impl Fn(&str, String) -> Result + Sync, +) -> Result>, StageDeferral> { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + pool.install(|| { + let paths: Vec = rglob(tree, directory, |entry| { + entry.name.ends_with(SQL_FILE_SUFFIX) + })?; + Ok(paths + .into_par_iter() + .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .map(|relative_path| DiscoveredFile { + outcome: match read_authored_text(&tree.absolute(&relative_path)) { + Ok(contents) => match parse(&root.display_path(&relative_path), contents) { + Ok(parsed) => FileOutcome::Parsed(parsed), + Err(failure) => FileOutcome::Failed(failure), + }, + Err(_unreadable) => FileOutcome::Unreadable, + }, + relative_path, + }) + .collect()) + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs new file mode 100644 index 0000000000..28b285846c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs @@ -0,0 +1,114 @@ +//! Python's `_split_sql_test_blocks` and the header half of `_parse_single_sql_test_block`. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestBlock}; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +const TEST_KEYWORD: &str = "TEST"; +const TEST_STATEMENT: &str = "TEST()"; + +/// One `^\s*TEST\s*\(header\)\s*;\s*` match: start, header start, header end and match end. +struct BlockMatch { + start: usize, + header_start: usize, + header_end: usize, + end: usize, +} + +pub(crate) fn parse_sql_test_file( + file_path: &str, + contents: String, + supported_keys: &[String], +) -> Result { + let matches: Vec = block_matches(&contents); + let Some(first) = matches.first() else { + return Err(missing_header_failure(file_path)); + }; + if !python_strip(&contents[..first.start]).is_empty() { + return Err(missing_header_failure(file_path)); + } + let contract = StatementHeader { + kind: FailureKind::SqlTest, + statement_name: TEST_KEYWORD, + statement: TEST_STATEMENT, + file_path, + supported_keys, + }; + let mut blocks: Vec = Vec::with_capacity(matches.len()); + let mut failure: Option = None; + for (index, block) in matches.iter().enumerate() { + let next_start: usize = matches + .get(index + 1) + .map_or(contents.len(), |next| next.start); + let header: &str = &contents[block.header_start..block.header_end]; + let header_line: usize = contents[..block.header_start].matches('\n').count() + 1; + match parse_statement_header(&contract, header, header_line) { + Ok(header_values) => blocks.push(SqlTestBlock { + header_values, + sql_body: python_cleandoc( + contents[block.end.min(next_start)..next_start] + .trim_end_matches(is_python_space), + ), + }), + Err(block_failure) => { + failure = Some(block_failure); + break; + } + } + } + Ok(DiscoveredSqlTestFile { + contents, + blocks, + failure, + }) +} + +fn missing_header_failure(file_path: &str) -> DiscoveryFailure { + DiscoveryFailure::new( + FailureKind::SqlTest, + format!( + "SQL test '{file_path}' must start with a TEST() header as the first non-whitespace \ + content" + ), + ) +} + +/// `_TEST_HEADER_ONLY_PATTERN.finditer(contents)` (MULTILINE): attempts start at line starts. +fn block_matches(contents: &str) -> Vec { + let mut matches: Vec = Vec::new(); + let mut position: usize = 0; + while position <= contents.len() { + let candidate: usize = next_line_start(contents, position); + if candidate > contents.len() { + break; + } + match match_statement_header(contents, candidate, TEST_KEYWORD) { + Some((header_start, header_end, end)) => { + matches.push(BlockMatch { + start: candidate, + header_start, + header_end, + end, + }); + position = end; + } + None => position = candidate + 1, + } + } + matches +} + +/// The first position at or after `position` where MULTILINE `^` matches, or past the end. +fn next_line_start(contents: &str, position: usize) -> usize { + if position == 0 || contents.as_bytes().get(position - 1) == Some(&b'\n') { + return position; + } + contents.as_bytes()[position..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(contents.len() + 1, |offset| position + offset + 1) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs new file mode 100644 index 0000000000..967eb618f3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -0,0 +1,19 @@ +//! Discover, read and header-parse the SQL scenario files under `tests/scenarios/`. + +use crate::constants::SQL_SCENARIOS_ROOT; +use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; +use crate::sql_tests::_helpers::selection::discover_sql_files; +use crate::sql_tests::models::{DiscoveredScenarioFile, SqlTestFileOptions}; +use crate::tree::models::ProjectTree; + +/// Every scenario file in Python's order, header-parsed or failing. +pub fn discover_scenario_files( + root: &ProjectRoot, + tree: &ProjectTree, + options: &SqlTestFileOptions, +) -> Result>, StageDeferral> { + discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { + parse_scenario_file(file_path, contents, &options.scenario_keys) + }) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs new file mode 100644 index 0000000000..41c74c256c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -0,0 +1,22 @@ +//! Discover, read and split the SQL unit-test files under `tests/unit/`. + +use crate::constants::SQL_TESTS_ROOT; +use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::sql_tests::_helpers::selection::discover_sql_files; +use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; +use crate::tree::models::ProjectTree; + +/// Every SQL test file in Python's order, split into header-parsed blocks or failing. +pub fn discover_sql_test_files( + root: &ProjectRoot, + tree: &ProjectTree, + options: &SqlTestFileOptions, +) -> Result>, StageDeferral> { + discover_sql_files( + root, + tree, + &SQL_TESTS_ROOT.join("/"), + |file_path, contents| parse_sql_test_file(file_path, contents, &options.test_keys), + ) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs new file mode 100644 index 0000000000..f79f30314c --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs @@ -0,0 +1,2 @@ +pub mod discover_scenario_files; +pub mod discover_sql_test_files; diff --git a/crates/sqlbuild-discovery/src/sql_tests/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/sql_tests/models.rs b/crates/sqlbuild-discovery/src/sql_tests/models.rs new file mode 100644 index 0000000000..9e04eae633 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/models.rs @@ -0,0 +1,35 @@ +//! Native results for SQL unit-test and scenario files. + +use crate::models::DiscoveryFailure; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// The keys each statement header supports, passed from Python's constants. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestFileOptions { + pub test_keys: Vec, + pub scenario_keys: Vec, +} + +/// One `TEST(...)` block whose header parsed with only supported keys. +#[derive(Clone, Debug, PartialEq)] +pub struct SqlTestBlock { + pub header_values: Vec<(String, AuthoredValue)>, + /// `inspect.cleandoc` of the SQL after the header. + pub sql_body: String, +} + +/// A test file split into blocks; parsing stops at the first block whose header fails. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredSqlTestFile { + pub contents: String, + pub blocks: Vec, + pub failure: Option, +} + +/// A scenario file whose header parsed with only supported keys. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredScenarioFile { + pub contents: String, + pub header_values: Vec<(String, AuthoredValue)>, + pub sql_body: String, +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests.rs b/crates/sqlbuild-discovery/src/sql_tests/tests.rs new file mode 100644 index 0000000000..158445fb36 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_sql_test_files.rs"] +mod sql_test_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs new file mode 100644 index 0000000000..1dd21e6c21 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -0,0 +1,78 @@ +use crate::models::DiscoveryFailure; +use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; +use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::models::SqlTestBlock; +use crate::sql_tests::tests::test_types::StatementFileTestCase; + +pub(super) type BlockRow = (Vec, String); +pub(super) const FILE_PATH: &str = "project/tests/unit/orders.sql"; + +fn keys(test_keys: &[&str]) -> Vec { + test_keys.iter().map(|key| (*key).to_owned()).collect() +} + +fn block_row(block: &SqlTestBlock) -> BlockRow { + ( + block + .header_values + .iter() + .map(|(key, _)| key.clone()) + .collect(), + block.sql_body.clone(), + ) +} + +fn message(failure: DiscoveryFailure) -> String { + failure.message +} + +fn failed_rows(failure: DiscoveryFailure) -> (Vec, Option) { + (Vec::new(), Some(failure.message)) +} + +/// The test file's parsed block rows and its failure message. +pub(super) fn test_file_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { + parse_sql_test_file( + FILE_PATH, + test_case.contents.to_owned(), + &keys(&["name", "mode", "parameters", "cases"]), + ) + .map_or_else(failed_rows, |file| { + ( + file.blocks.iter().map(block_row).collect(), + file.failure.map(message), + ) + }) +} + +/// The scenario file's block row and its failure message. +pub(super) fn scenario_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { + parse_scenario_file( + FILE_PATH, + test_case.contents.to_owned(), + &keys(&["description", "tags"]), + ) + .map_or_else(failed_rows, |file| { + ( + vec![block_row(&SqlTestBlock { + header_values: file.header_values, + sql_body: file.sql_body, + })], + None, + ) + }) +} + +/// The expected rows in the shape the parsers' rows take. +pub(super) fn expected_rows( + expected_blocks: &[(&[&str], &str)], + expected_failure: Option<&str>, +) -> (Vec, Option) { + ( + expected_blocks + .iter() + .map(|(block_keys, body)| (keys(block_keys), (*body).to_owned())) + .collect(), + expected_failure.map(str::to_owned), + ) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs new file mode 100644 index 0000000000..96e92835bd --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/test_sql_test_files.rs @@ -0,0 +1,74 @@ +use crate::sql_tests::tests::helpers::{expected_rows, scenario_rows, test_file_rows}; +use crate::sql_tests::tests::test_types::StatementFileTestCase; + +#[test] +fn given_sql_test_files_when_splitting_then_blocks_and_failures_match_python() { + let test_cases = [ + StatementFileTestCase { + description: "blocks start at line starts and bodies are cleaned", + contents: "\n TEST (name \"a\");\n SELECT 1\n FROM t\n\n TEST(name \"b\") ;\n\tSELECT 2\n", + expected_blocks: &[(&["name"], "SELECT 1\nFROM t"), (&["name"], "SELECT 2")], + expected_failure: None, + }, + StatementFileTestCase { + description: "TEST not at a line start stays inside the previous body", + contents: "TEST (name \"a\"); SELECT 'TEST (x);'", + expected_blocks: &[(&["name"], "SELECT 'TEST (x);'")], + expected_failure: None, + }, + StatementFileTestCase { + description: "content before the first header fails the file", + contents: "-- lead\nTEST ();\nSELECT 1", + expected_blocks: &[], + expected_failure: Some( + "SQL test 'project/tests/unit/orders.sql' must start with a TEST() header as the \ + first non-whitespace content", + ), + }, + StatementFileTestCase { + description: "an unsupported key stops at its block, at the key's line", + contents: "TEST (name \"a\");\nSELECT 1\nTEST (\n name \"b\",\n nme 1\n);\nSELECT 2", + expected_blocks: &[(&["name"], "SELECT 1")], + expected_failure: Some( + "TEST() in 'project/tests/unit/orders.sql:5' has unsupported keys: nme", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + test_file_rows(&test_case), + expected_rows(test_case.expected_blocks, test_case.expected_failure), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenario_files_when_parsing_then_headers_and_failures_match_python() { + let test_cases = [ + StatementFileTestCase { + description: "the body keeps trailing indentation as cleandoc does", + contents: "SCENARIO (description \"Orders\");\n\n SELECT 1\n ", + expected_blocks: &[(&["description"], "SELECT 1\n ")], + expected_failure: None, + }, + StatementFileTestCase { + description: "a missing header fails", + contents: "SELECT 1", + expected_blocks: &[], + expected_failure: Some( + "SQL scenario 'project/tests/unit/orders.sql' must start with a SCENARIO() \ + header as the first non-whitespace content", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + scenario_rows(&test_case), + expected_rows(test_case.expected_blocks, test_case.expected_failure), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs new file mode 100644 index 0000000000..f45e83d054 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(super) struct StatementFileTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + /// Each parsed block's header keys and SQL body, then the failure message if one stopped it. + pub(super) expected_blocks: &'static [(&'static [&'static str], &'static str)], + pub(super) expected_failure: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index b356bab861..3a6e71b43b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -13,6 +13,11 @@ use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOpti use sqlbuild_discovery::models::{ DiscoveredFile, DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, }; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::models::{ + DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, +}; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; use std::collections::HashSet; @@ -20,7 +25,7 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::bindings::_helpers::functions::map_to_python; -use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree}; +use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree, SqlTestDiscoveryRequest}; use crate::bindings::types::CompilerDetach; type PyObject = Py; @@ -49,15 +54,7 @@ fn outcome_object( match outcome { FileOutcome::Parsed(value) => parsed(py, value), FileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), - FileOutcome::Failed(failure) => tuple_object( - py, - vec![ - object(py, "error")?, - object(py, failure.kind.as_str())?, - object(py, failure.message)?, - object(py, failure.help)?, - ], - ), + FileOutcome::Failed(failure) => failure_object(py, failure), } } @@ -198,7 +195,111 @@ fn discover_declaration_layout( } } +fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { + tuple_object( + py, + vec![ + object(py, "error")?, + object(py, failure.kind.as_str())?, + object(py, failure.message)?, + object(py, failure.help)?, + ], + ) +} + +fn test_block_object(py: Python<'_>, block: SqlTestBlock) -> PyResult { + tuple_object( + py, + vec![ + map_to_python(py, block.header_values)?, + object(py, block.sql_body)?, + ], + ) +} + +fn test_file_object(py: Python<'_>, file: DiscoveredSqlTestFile) -> PyResult { + let blocks: Vec = file + .blocks + .into_iter() + .map(|block| test_block_object(py, block)) + .collect::>()?; + let failure: Option = file + .failure + .map(|failure| failure_object(py, failure)) + .transpose()?; + tuple_object( + py, + vec![ + object(py, "ok")?, + object(py, file.contents)?, + object(py, blocks)?, + object(py, failure)?, + ], + ) +} + +fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult { + tuple_object( + py, + vec![ + object(py, "ok")?, + object(py, file.contents)?, + map_to_python(py, file.header_values)?, + object(py, file.sql_body)?, + ], + ) +} + +fn sql_test_inputs(request: SqlTestDiscoveryRequest) -> (ProjectRoot, SqlTestFileOptions) { + ( + ProjectRoot { + directory: PathBuf::from(request.project_dir), + display_prefix: request.display_prefix, + }, + SqlTestFileOptions { + test_keys: request.test_keys, + scenario_keys: request.scenario_keys, + }, + ) +} + +/// Discover and split the SQL test files natively, or return `None` when Python must run. +#[pyfunction] +fn discover_sql_test_files( + py: Python<'_>, + request: SqlTestDiscoveryRequest, + tree: &NativeProjectTree, +) -> PyResult>> { + let (root, options) = sql_test_inputs(request); + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match discovered { + Ok(files) => Ok(Some(files_object(py, files, test_file_object)?)), + Err(_deferral) => Ok(None), + } +} + +/// Discover and header-parse the scenario files natively, or return `None` when Python must run. +#[pyfunction] +fn discover_scenario_files( + py: Python<'_>, + request: SqlTestDiscoveryRequest, + tree: &NativeProjectTree, +) -> PyResult>> { + let (root, options) = sql_test_inputs(request); + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match discovered { + Ok(files) => Ok(Some(files_object(py, files, scenario_object)?)), + Err(_deferral) => Ok(None), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index 05a6f267c3..8697cff25d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -18,6 +18,7 @@ use sqlbuild_core::text::main::close_matches::close_matches; use sqlbuild_core::text::main::decode_python_text::decode_python_text; use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::models::LineIndex; use std::path::Path; @@ -251,8 +252,14 @@ fn _oracle_close_matches( close_matches(word, &candidates, count, cutoff) } +#[pyfunction] +fn _oracle_cleandoc(texts: Vec) -> Vec { + texts.iter().map(|text| python_cleandoc(text)).collect() +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_python_alnum, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_cleandoc, module)?)?; module.add_function(wrap_pyfunction!(_oracle_close_matches, module)?)?; module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 72ab04cfc7..800836b3df 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -117,3 +117,13 @@ pub(crate) struct ModelDiscoveryRequest { pub(crate) extract_implicit_alias_columns: bool, pub(crate) extract_output_column_locations: bool, } + +/// One native SQL test or scenario discovery request from the Python discovery facade. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct SqlTestDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) test_keys: Vec, + pub(crate) scenario_keys: Vec, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs index cd07d750a3..6b913e3e02 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs @@ -15,12 +15,21 @@ pub(crate) fn match_one(text: &str) -> Option { /// UTF-8 byte offsets of one matched header: header start, header end, and SQL start. pub(crate) fn match_one_bytes(text: &str) -> Option { + match_statement_bytes(text, 0, MODEL_KEYWORD) +} + +/// `\s*KEYWORD\s*\((?P
...)\)\s*;\s*` matched at byte `start`, as UTF-8 byte offsets. +pub(crate) fn match_statement_bytes( + text: &str, + start: usize, + keyword: &[u8], +) -> Option { let bytes = text.as_bytes(); - let mut index = skip_whitespace(text, 0); - if !bytes[index..].starts_with(MODEL_KEYWORD) { + let mut index = skip_whitespace(text, start); + if !bytes[index..].starts_with(keyword) { return None; } - index = skip_whitespace(text, index + MODEL_KEYWORD.len()); + index = skip_whitespace(text, index + keyword.len()); if bytes.get(index) != Some(&b'(') { return None; } diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index e4d9f28ab3..0050ddac66 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -5,3 +5,4 @@ pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod sql_interpolation; pub mod sql_references; +pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs b/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs new file mode 100644 index 0000000000..807bbd0117 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/statement_header_matching.rs @@ -0,0 +1,10 @@ +use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; + +/// Byte offsets (header start, header end, match end) of `\s*KEYWORD\s*\(header\)\s*;\s*`. +pub fn match_statement_header( + text: &str, + start: usize, + keyword: &str, +) -> Option { + matching::match_statement_bytes(text, start, keyword.as_bytes()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8e89ea555a..94d9629367 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -53,6 +53,12 @@ class NativeProjectTree: def discover_model_files( request: dict[str, object], tree: NativeProjectTree ) -> list[tuple[str, tuple[object, ...]]] | None: ... +def discover_sql_test_files( + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... +def discover_scenario_files( + request: dict[str, object], tree: NativeProjectTree +) -> list[tuple[str, tuple[object, ...]]] | None: ... def discover_declaration_layout( tree: NativeProjectTree, ) -> ( @@ -78,6 +84,7 @@ def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... def _oracle_project_config(project_dir: str) -> str: ... def _oracle_python_alnum(code_points: list[int]) -> tuple[str, list[bool]]: ... +def _oracle_cleandoc(texts: list[str]) -> list[str]: ... def _oracle_close_matches( word: str, possibilities: list[str], count: int, cutoff: float ) -> list[str]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 07844fe457..75e395f6c6 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -40,6 +40,10 @@ from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, + discover_native_test_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( @@ -239,8 +243,16 @@ def _discover_declaration_files( python_function_files=discover_python_function_files(project_dir=project_dir), schema_files=discover_schema_files(project_dir=project_dir), seed_files=discover_seed_files(project_dir=project_dir), - test_files=discover_test_files(project_dir=project_dir, fact_cache=fact_cache), - scenario_files=discover_scenario_files(project_dir=project_dir), + test_files=( + discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache) + if native + else discover_test_files(project_dir=project_dir, fact_cache=fact_cache) + ), + scenario_files=( + discover_native_scenario_files(project_dir=project_dir) + if native + else discover_scenario_files(project_dir=project_dir) + ), audit_files=discover_audit_files(project_dir=project_dir), macro_files=discover_macro_files(project_dir=project_dir), adapter_file=discover_adapter_file(project_dir=project_dir), diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index f37989a615..01bd2ff86f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -8,7 +8,13 @@ import sqlbuild._native as _native from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError +from sqlbuild.compiler.discovery.exceptions import ( + DeclarationParseError, + DiscoveryError, + ModelSqlParseError, + SqlScenarioParseError, + SqlTestParseError, +) from sqlbuild.compiler.discovery.types import ( DirectorySnapshotEntry, NativeListing, @@ -20,7 +26,12 @@ _WINDOWS_OS_NAME: str = "nt" _PATH_ENCODING: str = "utf-8" _NATIVE_TREE_MEMO_KEY: str = "native_project_tree" -_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = {"model_sql": ModelSqlParseError} +_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = { + "model_sql": ModelSqlParseError, + "declaration": DeclarationParseError, + "sql_test": SqlTestParseError, + "sql_scenario": SqlScenarioParseError, +} def native_display_prefix(project_dir: Path) -> str: diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py new file mode 100644 index 0000000000..eb10063887 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py @@ -0,0 +1,169 @@ +"""SQL unit-test and scenario files discovered by the native engine, as Python records.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_scenario_files, + discover_test_files, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_failure, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values +from sqlbuild.compiler.discovery._helpers.sql.scenarios import ( + build_sql_scenario_file, + parse_sql_scenario_file, +) +from sqlbuild.compiler.discovery._helpers.sql.tests import ( + build_sql_test_block, + parse_sql_test_file, + validate_sql_test_names, +) +from sqlbuild.compiler.discovery.constants import ( + NATIVE_FAILED_TAG, + NATIVE_UNREADABLE_TAG, + SQL_SCENARIO_HEADER_KEYS, + SQL_TEST_HEADER_KEYS, + SQL_TESTS_OWNERSHIP_ROOT, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, +) +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore + +type _NativeFiles = list[tuple[str, tuple[object, ...]]] +type _NativeDiscovery = Callable[ + [dict[str, object], _native.NativeProjectTree], _NativeFiles | None +] + + +def discover_native_test_files( + *, project_dir: Path, fact_cache: FactCacheStore | None = None +) -> tuple[DiscoveredSqlTestFile, ...]: + """Discover SQL test files natively; Python, with its fact cache, runs where native cannot.""" + + files: _NativeFiles | None = _native_files( + project_dir=project_dir, discover=_native.discover_sql_test_files + ) + if files is None: + return discover_test_files(project_dir=project_dir, fact_cache=fact_cache) + return tuple( + _test_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) + for relative_path, payload in files + ) + + +def discover_native_scenario_files(*, project_dir: Path) -> tuple[DiscoveredSqlScenarioFile, ...]: + """Discover scenario files natively; Python runs when native discovery cannot match it.""" + + files: _NativeFiles | None = _native_files( + project_dir=project_dir, discover=_native.discover_scenario_files + ) + if files is None: + return discover_scenario_files(project_dir=project_dir) + return tuple( + _scenario_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) + for relative_path, payload in files + ) + + +def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> _NativeFiles | None: + display_prefix: str = native_display_prefix(project_dir) + if not native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix): + return None + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: _NativeFiles | None = discover( + { + "project_dir": str(project_dir), + "display_prefix": display_prefix, + "test_keys": sorted(SQL_TEST_HEADER_KEYS), + "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + }, + tree, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return files + + +def _test_file( + *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] +) -> DiscoveredSqlTestFile: + file_path: Path = project_dir / relative_path + tag: object = payload[0] + if tag == NATIVE_FAILED_TAG: + raise native_failure(payload) + contents: str + blocks: tuple[DiscoveredSqlTestBlock, ...] + if tag == NATIVE_UNREADABLE_TAG: + contents = file_path.read_text(encoding="utf-8") + blocks = parse_sql_test_file(contents=contents, file_path=file_path) + else: + _tag, native_contents, native_blocks, block_failure = payload + contents = str(native_contents) + blocks = _test_blocks( + file_path=file_path, + native_blocks=cast(list[tuple[dict[str, object], str]], native_blocks), + block_failure=cast(tuple[object, ...] | None, block_failure), + ) + return DiscoveredSqlTestFile( + file_path=file_path, + relative_path=relative_path, + contents=contents, + blocks=blocks, + ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), + ) + + +def _test_blocks( + *, + file_path: Path, + native_blocks: list[tuple[dict[str, object], str]], + block_failure: tuple[object, ...] | None, +) -> tuple[DiscoveredSqlTestBlock, ...]: + blocks: tuple[DiscoveredSqlTestBlock, ...] = tuple( + build_sql_test_block( + header_values=project_native_header_values(values), + sql_body=sql_body, + file_path=file_path, + test_index=test_index, + ) + for test_index, (values, sql_body) in enumerate(native_blocks, start=1) + ) + if block_failure is not None: + raise native_failure(block_failure) + validate_sql_test_names(file_path=file_path, blocks=blocks) + return blocks + + +def _scenario_file( + *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] +) -> DiscoveredSqlScenarioFile: + file_path: Path = project_dir / relative_path + tag: object = payload[0] + if tag == NATIVE_FAILED_TAG: + raise native_failure(payload) + if tag == NATIVE_UNREADABLE_TAG: + return parse_sql_scenario_file( + contents=file_path.read_text(encoding="utf-8"), + file_path=file_path, + relative_path=relative_path, + ) + _tag, contents, values, sql_body = payload + return build_sql_scenario_file( + header_values=project_native_header_values(cast(dict[str, object], values)), + sql_body=str(sql_body), + contents=str(contents), + file_path=file_path, + relative_path=relative_path, + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py index 431ad0b816..5ac51932c9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values from sqlbuild.compiler.discovery.constants import ( + SQL_SCENARIO_HEADER_KEYS, SQL_SCENARIOS_OWNERSHIP_ROOT, STATEMENT_HEADER_BODY_PATTERN, ) @@ -21,9 +22,6 @@ ) _SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" _SCENARIO_TAGS_HEADER_KEY: str = "tags" -_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset( - {_SCENARIO_DESCRIPTION_HEADER_KEY, _SCENARIO_TAGS_HEADER_KEY} -) def parse_sql_scenario_file( @@ -43,7 +41,33 @@ def parse_sql_scenario_file( header_line=contents.count("\n", 0, header_match.start("header")) + 1, file_path=file_path, ) - sql_body: str = cleandoc(header_match.group("sql")) + return build_sql_scenario_file( + header_values=header_values, + sql_body=cleandoc(header_match.group("sql")), + contents=contents, + file_path=file_path, + relative_path=relative_path, + ) + + +def build_sql_scenario_file( + *, + header_values: dict[str, object], + sql_body: str, + contents: str, + file_path: Path, + relative_path: Path, +) -> DiscoveredSqlScenarioFile: + """Validate a header-parsed SCENARIO file with only supported keys and build its record.""" + + description_value: object | None = header_values.get(_SCENARIO_DESCRIPTION_HEADER_KEY) + if _SCENARIO_DESCRIPTION_HEADER_KEY in header_values and not isinstance(description_value, str): + raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") + tags_value: object | None = header_values.get(_SCENARIO_TAGS_HEADER_KEY) + if _SCENARIO_TAGS_HEADER_KEY in header_values and ( + not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) + ): + raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") if not sql_body: raise SqlScenarioParseError( f"SQL scenario '{file_path}' must define SQL after SCENARIO(...)" @@ -70,21 +94,11 @@ def _parse_scenario_header(*, header: str, header_line: int, file_path: Path) -> reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_SCENARIO_HEADER_KEYS, + supported_keys=SQL_SCENARIO_HEADER_KEYS, statement="SCENARIO()", header=header, header_line=header_line, file_path=file_path, error_class=SqlScenarioParseError, ) - - description_value: object | None = parsed_header.get(_SCENARIO_DESCRIPTION_HEADER_KEY) - if _SCENARIO_DESCRIPTION_HEADER_KEY in parsed_header and not isinstance(description_value, str): - raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") - tags_value: object | None = parsed_header.get(_SCENARIO_TAGS_HEADER_KEY) - if _SCENARIO_TAGS_HEADER_KEY in parsed_header and ( - not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) - ): - raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") - return parsed_header diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index 3cd3e72c19..a8dac94e45 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -17,6 +17,7 @@ ) from sqlbuild.compiler.discovery.constants import ( SQL_ANALYSIS_CONFIG_KEY, + SQL_TEST_HEADER_KEYS, STATEMENT_HEADER_BODY_PATTERN, ) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError @@ -44,17 +45,6 @@ _TEST_CASES_HEADER_KEY: str = "cases" _TEST_CURSOR_START_HEADER_KEY: str = "cursor_start" _TEST_CURSOR_END_HEADER_KEY: str = "cursor_end" -_TEST_HEADER_KEYS: frozenset[str] = frozenset( - { - _TEST_NAME_HEADER_KEY, - _TEST_MODE_HEADER_KEY, - _TEST_PARAMETERS_HEADER_KEY, - _TEST_CASES_HEADER_KEY, - _TEST_CURSOR_START_HEADER_KEY, - _TEST_CURSOR_END_HEADER_KEY, - SQL_ANALYSIS_CONFIG_KEY, - } -) _PARAMETER_TYPES: tuple[SqlValueKind, ...] = ( SqlValueKind.STRING, SqlValueKind.INTEGER, @@ -102,7 +92,7 @@ def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSq ) ) - _validate_test_names(file_path=file_path, blocks=tuple(discovered_blocks)) + validate_sql_test_names(file_path=file_path, blocks=tuple(discovered_blocks)) return tuple(discovered_blocks) @@ -148,7 +138,27 @@ def _parse_single_sql_test_block( header_line=block_line + raw_test_block.count("\n", 0, header_match.start("header")), file_path=file_path, ) - sql_body: str = cleandoc(header_match.group("sql")) + return build_sql_test_block( + header_values=header_values, + sql_body=cleandoc(header_match.group("sql")), + file_path=file_path, + test_index=test_index, + ) + + +def build_sql_test_block( + *, header_values: dict[str, object], sql_body: str, file_path: Path, test_index: int +) -> DiscoveredSqlTestBlock: + """Validate a header-parsed TEST block with only supported keys and build its record.""" + + if _TEST_NAME_HEADER_KEY in header_values: + _validate_test_name(name_value=header_values[_TEST_NAME_HEADER_KEY], file_path=file_path) + if _TEST_MODE_HEADER_KEY in header_values: + _validate_test_mode(mode_value=header_values[_TEST_MODE_HEADER_KEY], file_path=file_path) + if SQL_ANALYSIS_CONFIG_KEY in header_values and not isinstance( + header_values[SQL_ANALYSIS_CONFIG_KEY], bool + ): + raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") if not sql_body: raise SqlTestParseError(f"SQL test '{file_path}' must define SQL after TEST(...)") @@ -211,23 +221,13 @@ def _parse_test_header(*, header: str, header_line: int, file_path: Path) -> dic reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_TEST_HEADER_KEYS, + supported_keys=SQL_TEST_HEADER_KEYS, statement="TEST()", header=header, header_line=header_line, file_path=file_path, error_class=SqlTestParseError, ) - - if _TEST_NAME_HEADER_KEY in parsed_header: - _validate_test_name(name_value=parsed_header[_TEST_NAME_HEADER_KEY], file_path=file_path) - if _TEST_MODE_HEADER_KEY in parsed_header: - _validate_test_mode(mode_value=parsed_header[_TEST_MODE_HEADER_KEY], file_path=file_path) - if SQL_ANALYSIS_CONFIG_KEY in parsed_header and not isinstance( - parsed_header[SQL_ANALYSIS_CONFIG_KEY], bool - ): - raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") - return parsed_header @@ -391,7 +391,9 @@ def _validate_test_mode(*, mode_value: object, file_path: Path) -> None: raise SqlTestParseError(f"TEST() mode in '{file_path}' must be one of: {allowed_modes}") -def _validate_test_names(*, file_path: Path, blocks: tuple[DiscoveredSqlTestBlock, ...]) -> None: +def validate_sql_test_names(*, file_path: Path, blocks: tuple[DiscoveredSqlTestBlock, ...]) -> None: + """Reject multi-block files whose blocks lack unique names.""" + if len(blocks) <= 1: return diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 451aeaea91..ca42bc8b71 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -121,6 +121,10 @@ CONFIG_CONCURRENCY_KEY: str = "concurrency" LEGACY_CONFIG_CONCURRENCY_KEY: str = "max_concurrency" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" +SQL_TEST_HEADER_KEYS: frozenset[str] = frozenset( + {"name", "mode", "parameters", "cases", "cursor_start", "cursor_end", SQL_ANALYSIS_CONFIG_KEY} +) +SQL_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset({"description", "tags"}) LEGACY_SQL_VALIDATION_CONFIG_KEY: str = "sql_validation" SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" REQUIRE_SQL_ANALYSIS_SETTING_KEY: str = "require_sql_analysis" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index c603cfce2f..af91ecf729 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -114,36 +114,26 @@ def _assemble_discovered_project_inputs( else project_config.settings.sql_analysis ) ) - if native: + with FactCacheStore( + root=discovery_cache_root( + project_dir=project_dir, + project_config=project_config, + local_config=local_config, + cache_request=cache_request, + ), + namespace=DISCOVERY_FACT_CACHE_NAMESPACE, + algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, + ) as fact_cache: discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( project_dir=project_dir, project_config=project_config, local_config=local_config, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, + fact_cache=fact_cache, declaration_reuse=declaration_reuse, - native=True, + native=native, ) - else: - with FactCacheStore( - root=discovery_cache_root( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - cache_request=cache_request, - ), - namespace=DISCOVERY_FACT_CACHE_NAMESPACE, - algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, - ) as fact_cache: - discovered_inputs = build_discovered_project_inputs( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - fact_cache=fact_cache, - declaration_reuse=declaration_reuse, - ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( configure_discovered_event_exporters, diff --git a/tests/integration/src/sqlbuild/compiler/_test_types.py b/tests/integration/src/sqlbuild/compiler/_test_types.py index 4c549cca28..6bf8097f5b 100644 --- a/tests/integration/src/sqlbuild/compiler/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/_test_types.py @@ -51,6 +51,16 @@ class CloseMatchesOracleTestCase: expected_mismatches: tuple[tuple[object, object, object], ...] = () +@dataclass(frozen=True) +class CleandocOracleTestCase: + """Seeded SQL bodies compared between the native port and `inspect.cleandoc`.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + @dataclass(frozen=True) class CharacterClassOracleTestCase: """A code point range classified natively and by Python's `str` methods.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index a2229f4dc0..faafd4e6a0 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -106,3 +106,15 @@ class GeneratedLayoutParityTestCase: expected_mismatches: tuple[str, ...] = () expected_minimum_valid: int = 0 expected_minimum_invalid: int = 0 + + +@dataclass(frozen=True) +class GeneratedSqlTestParityTestCase: + """Seeded SQL test and scenario files discovered by both compiler engines.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + expected_minimum_parsed: int = 0 + expected_minimum_failed: int = 0 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 39e05eb34f..7154c12f48 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -23,6 +23,8 @@ discover_enum_files, discover_macro_files, discover_model_files, + discover_scenario_files, + discover_test_files, ) from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( named_declaration_roots, @@ -31,6 +33,10 @@ from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, + discover_native_test_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.exceptions import DeclarationParseError @@ -658,6 +664,88 @@ def harmful_mismatches( SEPARATORS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", " ", "\u3000", "\n\n") +TEST_HEADER_ENTRIES: tuple[str, ...] = ( + 'name "keeps_orders"', "mode model", 'name "second"', "name ''", "name 1", "mode query", + "mode audit", "mode bogus", "sql_analysis false", "sql_analysis 1", "nme 1", + "parameters (status string), cases (placed (status 'placed'))", + "parameters (amount (type integer, nullable true)), cases (empty (amount null))", + "parameters (amount integer), cases (bad (amount 'x'))", "parameters (amount integer)", + "cursor_start '2024-01-01'", "cursor_end 3", "materialized table", "description (", + "tags [a, b]", "'unterminated", +) # fmt: skip +SCENARIO_HEADER_ENTRIES: tuple[str, ...] = ( + 'description "Orders world"', "tags [north, south]", "description 1", "tags [1]", + "tags north", "name x", "description (", '"unterminated', +) # fmt: skip +STATEMENT_BODIES: tuple[str, ...] = ( + "SELECT 1", "SELECT 2 AS amount", "\n SELECT 3\n FROM orders\n", "", " ", + "\n SELECT 1\n FROM t\n", "\tSELECT 1\n\tFROM\tt", + "SELECT 'TEST (x);' AS label", "SELECT 1\n ", "-- only a comment", "SELECT ')' AS c;", + "\u3000SELECT 1", +) # fmt: skip +STATEMENT_PREFIXES: tuple[str, ...] = ("",) * 10 + (" \n", "\ufeff", "-- lead\n", "\n\n ") + + +def generated_statement_text(*, rng: random.Random, keyword: str, entries: tuple[str, ...]) -> str: + """Return one statement header with random entries, separators and a body.""" + + separator: str = rng.choice(SEPARATORS) + chosen: list[str] = rng.choices( + (rng.sample(entries, k=rng.choice((0, 1, 1, 1, 2))), rng.sample(entries[:2], k=1)), + weights=(3, 2), + )[0] + header: str = ("," + separator).join(chosen) + closing: str = rng.choice((");",) * 8 + (") ;", ")\n;", ")", "));", ");;")) + return ( + f"{rng.choice(('', ' ', ' '))}{keyword}{rng.choice(('', ' '))}({separator}{header}" + f"{separator}{closing}{rng.choice(SEPARATORS)}{rng.choice(STATEMENT_BODIES)}" + ) + + +def generated_test_bytes(*, rng: random.Random) -> bytes: + """Return one SQL test file with one to three blocks, joined at or inside lines.""" + + blocks: list[str] = [ + generated_statement_text(rng=rng, keyword="TEST", entries=TEST_HEADER_ENTRIES) + for _ in range(rng.choice((1, 1, 1, 1, 2, 3))) + ] + joined: str = "".join(rng.choice(("\n", "\n\n", " ", "\r\n")) + block for block in blocks) + return (rng.choice(STATEMENT_PREFIXES) + joined.lstrip("\n\r ")).encode("utf-8") + + +def generated_scenario_bytes(*, rng: random.Random) -> bytes: + """Return one SQL scenario file.""" + + text: str = generated_statement_text( + rng=rng, keyword="SCENARIO", entries=SCENARIO_HEADER_ENTRIES + ) + return (rng.choice(STATEMENT_PREFIXES) + text).encode("utf-8") + + +def sql_test_discovery_outcome(*, project_dir: Path, native: bool) -> object: + """Return the rendered test and scenario files, or the error type, message and help.""" + + discover: Callable[[], tuple[object, object]] = { + True: lambda: ( + discover_native_test_files(project_dir=project_dir), + discover_native_scenario_files(project_dir=project_dir), + ), + False: lambda: ( + discover_test_files(project_dir=project_dir), + discover_scenario_files(project_dir=project_dir), + ), + }[native] + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with DirectorySnapshot.scope(project_dir=project_dir), capture: + rendered[0] = render_stage_capture(discover()) + failure: BaseException | None = capture.failure + return { + True: rendered[0], + False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), + }[failure is None] + + class FailureCapture: """Swallow and keep the discovery failure a block raises.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py new file mode 100644 index 0000000000..d4c478137f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py @@ -0,0 +1,117 @@ +"""Engine parity for SQL unit-test and scenario files discovered natively.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + GeneratedSqlTestParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + generated_scenario_bytes, + generated_test_bytes, + sql_test_discovery_outcome, + stage_outcome, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_TEST: bytes = b'TEST (name "keeps_orders");\n\nSELECT 1\n' +_SCENARIO: bytes = b'SCENARIO (description "Orders world");\n\nSELECT 1\n' + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedSqlTestParityTestCase( + description="seeded test blocks and scenario headers", + seed=71, + case_count=1000, + expected_minimum_parsed=80, + expected_minimum_failed=150, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_test_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: GeneratedSqlTestParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[tuple[tuple[str, bytes], ...]] = [ + ( + ("tests/unit/orders.sql", generated_test_bytes(rng=rng)), + ("tests/scenarios/orders.sql", generated_scenario_bytes(rng=rng)), + ) + for _ in range(test_case.case_count) + ] + project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] + for project_dir, project_files in zip(project_dirs, files, strict=True): + write_project(project_dir=project_dir, files=project_files) + expected: list[object] = [ + sql_test_discovery_outcome(project_dir=project_dir, native=False) + for project_dir in project_dirs + ] + + actual: list[object] = [ + sql_test_discovery_outcome(project_dir=project_dir, native=True) + for project_dir in project_dirs + ] + + parsed: int = sum(isinstance(outcome, str) for outcome in expected) + assert ( + mismatches(inputs=list(files), expected=expected, actual=actual), + parsed >= test_case.expected_minimum_parsed, + len(expected) - parsed >= test_case.expected_minimum_failed, + ) == (list(test_case.expected_mismatches), True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="test and scenario files in nested, macro and scoped folders", + files=( + ("models/orders.sql", b'MODEL (description "Orders");\nSELECT 1 AS order_id'), + ("tests/unit/orders.sql", _TEST), + ("tests/unit/marts/b.sql", _TEST), + ("tests/unit/macros/money.sql", _TEST), + ("tests/unit/marts/_macros/helpers.py", b"def cents(value):\n return value\n"), + ("tests/unit/marts/_sqlbuild/enums/status.sql", b"ENUM (name s, members [A]);\n"), + ("tests/scenarios/north/orders.sql", _SCENARIO), + ("tests/scenarios/south.sql", _SCENARIO), + ), + ), + EngineSwitchParityTestCase( + description="the first failing test file in path order wins", + files=( + ("tests/unit/a.sql", _TEST), + ("tests/unit/b.sql", b"\xff"), + ("tests/unit/c.sql", b"TEST (nme 1);\nSELECT 1"), + ), + ), + EngineSwitchParityTestCase( + description="a semantic failure in an earlier block beats a later syntax failure", + files=( + ("tests/unit/a.sql", b"TEST (mode bogus);\nSELECT 1\nTEST (name ();\nSELECT 2"), + ), + ), + EngineSwitchParityTestCase( + description="an unreadable scenario fails with Python's read error", + files=(("tests/scenarios/a.sql", b"SCENARIO ();\n\xfe"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_test_files_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py index 0d34d92e5f..0a0e398415 100644 --- a/tests/integration/src/sqlbuild/compiler/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -302,6 +302,10 @@ def python_read_text(*, path: Path, data: bytes) -> str: TABLE_PYTHON_VERSION: tuple[int, int] = (3, 12) +CLEANDOC_FRAGMENTS: tuple[str, ...] = ( + "", " ", " ", "\t", " \t", "\u3000", "\x0b", "\x1c", "\xa0", "\r", "\n", "\n", "\n\n", + "SELECT 1", "a", "é", "x\ty", "--c", +) # fmt: skip MUTATION_ALPHABET: str = "abcdefghijklmnopqrstuvwxyz_0é" @@ -321,6 +325,12 @@ def _substitute(*, rng: random.Random, word: list[str], position: int) -> None: MUTATIONS: tuple[Callable[..., None], ...] = (_insert, _delete, _substitute) +def cleandoc_text(*, rng: random.Random) -> str: + """Return a body mixing indentation, tabs, Unicode spaces and blank lines.""" + + return "".join(rng.choices(CLEANDOC_FRAGMENTS, k=rng.randint(0, 24))) + + def mutated_word(*, rng: random.Random, candidates: list[str]) -> str: """Return a candidate with a few random insertions, deletions and substitutions.""" diff --git a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py index b8f59494e3..0a3f83f3e8 100644 --- a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py @@ -3,6 +3,7 @@ from __future__ import annotations import difflib +import inspect import random import sys import unicodedata @@ -13,10 +14,12 @@ from sqlbuild.compiler.discovery.constants import SQL_MODEL_HEADER_KEYS from tests.integration.src.sqlbuild.compiler._test_types import ( CharacterClassOracleTestCase, + CleandocOracleTestCase, CloseMatchesOracleTestCase, ) from tests.integration.src.sqlbuild.compiler.helpers import ( TABLE_PYTHON_VERSION, + cleandoc_text, mismatches, mutated_word, ) @@ -86,3 +89,30 @@ def test_given_words_when_finding_close_matches_natively_then_difflib_agrees( assert mismatches(inputs=list(words), expected=expected, actual=actual) == list( test_case.expected_mismatches ) + + +@pytest.mark.skipif( + sys.version_info[:2] != TABLE_PYTHON_VERSION, + reason="native discovery runs only on the CI Python, whose cleandoc the port reproduces", +) +@pytest.mark.parametrize( + "test_case", + [ + CleandocOracleTestCase( + description="indented bodies with tabs and blank lines", seed=53, case_count=3000 + ) + ], + ids=lambda case: case.description, +) +def test_given_bodies_when_cleaning_natively_then_inspect_cleandoc_agrees( + test_case: CleandocOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + texts: list[str] = [cleandoc_text(rng=rng) for _ in range(test_case.case_count)] + + expected: list[object] = [inspect.cleandoc(text) for text in texts] + actual: list[object] = list(_native._oracle_cleandoc(texts)) + + assert mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) From dd182348dbf5b3c2adc05b279c3670341e2428ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:49:24 +0100 Subject: [PATCH 3/3] feat(native): load source and schema YAML files natively With the native engine, Rust reads the source, model schema and seed declaration files Python selects and loads each with the PyYAML-compatible loader. A file the native loader cannot reproduce is loaded by Python, and an unreadable file is re-read by Python for its error. Validation of the loaded documents stays in Python. --- Cargo.lock | 1 + .../src/yaml/_helpers/composer.rs | 29 ++- crates/sqlbuild-config/src/yaml/constants.rs | 2 + crates/sqlbuild-discovery/Cargo.toml | 1 + crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/yaml_files/main/load_yaml_files.rs | 31 +++ .../src/yaml_files/main/mod.rs | 1 + .../sqlbuild-discovery/src/yaml_files/mod.rs | 5 + .../src/yaml_files/models.rs | 16 ++ .../src/yaml_files/tests.rs | 6 + .../src/yaml_files/tests/helpers.rs | 8 + .../yaml_files/tests/test_load_yaml_files.rs | 49 ++++ .../src/yaml_files/tests/test_types.rs | 8 + .../src/bindings/_helpers/config_values.rs | 108 +++++++++ .../src/bindings/_helpers/discovery.rs | 41 ++++ .../src/bindings/_helpers/mod.rs | 1 + src/sqlbuild/_native.pyi | 1 + .../_helpers/filesystem/aggregation.py | 16 +- .../discovery/_helpers/filesystem/core.py | 37 +-- .../discovery/_helpers/native/payloads.py | 6 + .../discovery/_helpers/native/yaml_files.py | 172 ++++++++++++++ .../discovery/_helpers/yml/file_paths.py | 48 ++++ .../compiler/discovery/_helpers/yml/schema.py | 20 +- .../discovery/_helpers/yml/sources.py | 18 +- src/sqlbuild/compiler/discovery/constants.py | 2 + .../compiler/discovery/_test_types.py | 31 +++ .../sqlbuild/compiler/discovery/helpers.py | 213 +++++++++++++++++ .../test_native_yaml_file_discovery.py | 219 ++++++++++++++++++ .../discovery/test_native_yaml_oracle.py | 7 + 29 files changed, 1052 insertions(+), 46 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/mod.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/models.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/tests.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/config_values.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py create mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py diff --git a/Cargo.lock b/Cargo.lock index c87be16302..9686b7a315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,6 +665,7 @@ name = "sqlbuild-discovery" version = "0.138.0" dependencies = [ "rayon", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-sqltext", "tempfile", diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs index 7d04aaf9d9..fee4e4b92e 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -7,8 +7,8 @@ use crate::yaml::_helpers::anchors::{ use crate::yaml::_helpers::block_scalars::{column_of, scan_block_scalar}; use crate::yaml::_helpers::resolver::resolve_scalar; use crate::yaml::constants::{ - FLOW_KEY_INDICATOR, MAP_TAG, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG, - PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, + FLOW_KEY_INDICATOR, MAP_TAG, MAX_NATIVE_IMPLICIT_KEY_CHARACTERS, MAX_NESTING_DEPTH, + NON_SPECIFIC_TAG, PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, }; use crate::yaml::models::{ComposedDocument, Node, NodeContent}; use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag}; @@ -142,8 +142,8 @@ impl<'input> Composer<'input> { self.chars.get(index).copied() } - /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. - fn spans_lines(&self, span: Span) -> bool { + /// An implicit key's extent from its first property to its `:` indicator; `None` if explicit. + fn implicit_key_extent(&self, span: Span) -> Option<(usize, usize)> { let key_end = span.end.index().min(self.chars.len()); let indicator = (key_end..self.chars.len()) .find(|index| !matches!(self.chars[*index], ' ' | '\t' | '\n' | '\r')) @@ -166,10 +166,22 @@ impl<'input> Composer<'input> { self.char_at(start + 1), None | Some(' ' | '\t' | '\n' | '\r') ); - !explicit - && self.chars[start.min(end)..end] + (!explicit).then_some((start.min(end), end)) + } + + /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. + fn spans_lines(&self, span: Span) -> bool { + self.implicit_key_extent(span).is_some_and(|(start, end)| { + self.chars[start..end] .iter() .any(|character| matches!(character, '\n' | '\r')) + }) + } + + /// Whether an implicit key nears PyYAML's 1024-character simple-key limit. + fn is_long_implicit_key(&self, span: Span) -> bool { + self.implicit_key_extent(span) + .is_some_and(|(start, end)| end - start > MAX_NATIVE_IMPLICIT_KEY_CHARACTERS) } /// Whether the collection starting at `span` is written in flow style. @@ -327,6 +339,11 @@ impl<'input> Composer<'input> { "a plain scalar in a flow collection cannot start with ':'", )); } + if placement.key && self.is_long_implicit_key(span) { + return Err(unsupported( + "implicit keys near PyYAML's 1024-character limit are left to Python", + )); + } if placement.key && placement.flow && self.spans_lines(span) { return Err(ConfigError::new( ConfigErrorKind::Syntax, diff --git a/crates/sqlbuild-config/src/yaml/constants.rs b/crates/sqlbuild-config/src/yaml/constants.rs index 2a5b16581e..bd4f2ec34a 100644 --- a/crates/sqlbuild-config/src/yaml/constants.rs +++ b/crates/sqlbuild-config/src/yaml/constants.rs @@ -60,3 +60,5 @@ pub(crate) const STRIP_INDICATOR: char = '-'; pub(crate) const SEQUENCE_ENTRY_TOKEN: &str = "-"; pub(crate) const DIRECTIVE_INDICATOR: char = '%'; pub(crate) const PLAIN_FORBIDDEN_FIRST_CHARACTERS: [char; 5] = ['|', '>', '%', '@', '`']; +/// Longer implicit keys are left to Python, which rejects simple keys beyond 1024 characters. +pub(crate) const MAX_NATIVE_IMPLICIT_KEY_CHARACTERS: usize = 1000; diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 44344eb15b..cb2d6a4aed 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index b93c9f48cc..1858da47b2 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -9,3 +9,4 @@ pub mod model_files; pub mod models; pub mod sql_tests; pub mod tree; +pub mod yaml_files; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs new file mode 100644 index 0000000000..85bd8a1291 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs @@ -0,0 +1,31 @@ +//! Read and load the given YAML files in parallel, deferring each file Python must load. + +use crate::_helpers::pool::discovery_pool; +use crate::_helpers::reading::read_authored_text; +use crate::models::StageDeferral; +use crate::tree::models::ProjectTree; +use crate::yaml_files::models::YamlFileOutcome; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use sqlbuild_config::yaml::main::safe_load::safe_load; + +/// Each file's outcome, in the order of `relative_paths`. +pub fn load_yaml_files( + tree: &ProjectTree, + relative_paths: &[String], +) -> Result, StageDeferral> { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + Ok(pool.install(|| { + relative_paths + .par_iter() + .map( + |relative_path| match read_authored_text(&tree.absolute(relative_path)) { + Ok(contents) => match safe_load(&contents) { + Ok(value) => YamlFileOutcome::Loaded { contents, value }, + Err(_deferred) => YamlFileOutcome::LoadInPython { contents }, + }, + Err(_unreadable) => YamlFileOutcome::Unreadable, + }, + ) + .collect() + })) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs new file mode 100644 index 0000000000..d96b3a0c0b --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -0,0 +1 @@ +pub mod load_yaml_files; diff --git a/crates/sqlbuild-discovery/src/yaml_files/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/mod.rs new file mode 100644 index 0000000000..1a70a48981 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/mod.rs @@ -0,0 +1,5 @@ +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/yaml_files/models.rs b/crates/sqlbuild-discovery/src/yaml_files/models.rs new file mode 100644 index 0000000000..7aa974d0d5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/models.rs @@ -0,0 +1,16 @@ +//! Native outcomes of reading and loading authored YAML files. + +use sqlbuild_config::models::ConfigValue; + +/// One YAML file read and loaded as PyYAML `safe_load` would, or what Python must redo. +#[derive(Clone, Debug, PartialEq)] +pub enum YamlFileOutcome { + Loaded { + contents: String, + value: ConfigValue, + }, + /// Read, but the native loader cannot reproduce PyYAML here; Python loads the contents. + LoadInPython { contents: String }, + /// The bytes could not be read or decoded; Python re-reads the file to raise its error. + Unreadable, +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests.rs b/crates/sqlbuild-discovery/src/yaml_files/tests.rs new file mode 100644 index 0000000000..929b7b00fa --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_yaml_files.rs"] +mod load_yaml_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs new file mode 100644 index 0000000000..28a1ca493a --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs @@ -0,0 +1,8 @@ +use crate::yaml_files::models::YamlFileOutcome; + +/// An expected "Python loads the contents" outcome; only its variant is compared. +pub(super) fn python() -> YamlFileOutcome { + YamlFileOutcome::LoadInPython { + contents: String::new(), + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs new file mode 100644 index 0000000000..9f019458a0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs @@ -0,0 +1,49 @@ +use crate::tree::models::ProjectTree; +use crate::yaml_files::main::load_yaml_files::load_yaml_files; +use crate::yaml_files::models::YamlFileOutcome; +use crate::yaml_files::tests::test_types::YamlFileTestCase; + +use crate::yaml_files::tests::helpers::python; +use sqlbuild_config::models::ConfigValue; +use std::mem::discriminant; + +#[test] +fn given_yaml_files_when_loading_then_each_file_loads_or_defers() { + let test_cases = [ + YamlFileTestCase { + description: "a plain mapping loads natively", + contents: b"sources:\n - name: orders\n", + expected_outcome: YamlFileOutcome::Loaded { + contents: String::new(), + value: ConfigValue::Null, + }, + }, + YamlFileTestCase { + description: "invalid YAML is left to Python for its exact error", + contents: b"sources: [\n", + expected_outcome: python(), + }, + YamlFileTestCase { + description: "a Python-only tag is left to Python", + contents: b"value: !!set {a: null}\n", + expected_outcome: python(), + }, + YamlFileTestCase { + description: "undecodable bytes are re-read by Python", + contents: b"name: \xff\n", + expected_outcome: YamlFileOutcome::Unreadable, + }, + ]; + for test_case in test_cases { + let project = tempfile::tempdir().expect("temporary project"); + std::fs::write(project.path().join("file.yml"), test_case.contents).expect("file"); + let outcomes = load_yaml_files(&ProjectTree::new(project.path()), &["file.yml".to_owned()]) + .expect("no deferral"); + assert_eq!( + outcomes.iter().map(discriminant).collect::>(), + vec![discriminant(&test_case.expected_outcome)], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs new file mode 100644 index 0000000000..99eba3e5a5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::yaml_files::models::YamlFileOutcome; + +pub(super) struct YamlFileTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static [u8], + /// Only the variant is compared. + pub(super) expected_outcome: YamlFileOutcome, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs new file mode 100644 index 0000000000..140497c9f4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs @@ -0,0 +1,108 @@ +//! Loaded configuration values as the Python objects PyYAML and `tomllib` build. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +type PyObject = Py; + +/// Decimal digits per `int(str)` call, well below Python's default 4300-digit limit. +const BIG_INTEGER_CHUNK_DIGITS: usize = 1000; + +fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { + value.into_py_any(py) +} + +fn date(datetime: &Bound<'_, PyModule>, value: ConfigDate) -> PyResult { + Ok(datetime + .getattr("date")? + .call1((value.year, value.month, value.day))? + .unbind()) +} + +fn time(datetime: &Bound<'_, PyModule>, value: ConfigTime) -> PyResult { + Ok(datetime + .getattr("time")? + .call1((value.hour, value.minute, value.second, value.microsecond))? + .unbind()) +} + +fn date_time(datetime: &Bound<'_, PyModule>, value: ConfigDateTime) -> PyResult { + let tzinfo: Option> = match value.utc_offset_seconds { + Some(seconds) => { + let delta = datetime.getattr("timedelta")?.call( + (), + Some(&[("seconds", seconds)].into_py_dict(datetime.py())?), + )?; + Some(datetime.getattr("timezone")?.call1((delta,))?) + } + None => None, + }; + let (date, time) = (value.date, value.time); + Ok(datetime + .getattr("datetime")? + .call1(( + date.year, + date.month, + date.day, + time.hour, + time.minute, + time.second, + time.microsecond, + tzinfo, + ))? + .unbind()) +} + +/// Python's `int` of a decimal text of any length, built from chunks below the conversion limit. +fn big_integer(py: Python<'_>, text: &str) -> PyResult { + let (negative, digits) = match text.strip_prefix('-') { + Some(digits) => (true, digits), + None => (false, text), + }; + let int = py.import("builtins")?.getattr("int")?; + let mut value: Bound<'_, PyAny> = int.call1((0,))?; + for chunk in digits.as_bytes().chunks(BIG_INTEGER_CHUNK_DIGITS) { + let chunk_text = std::str::from_utf8(chunk) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let scale = int.call1((10,))?.pow(chunk.len(), py.None())?; + value = value.mul(scale)?.add(int.call1((chunk_text,))?)?; + } + if negative { + value = value.neg()?; + } + Ok(value.unbind()) +} + +/// Build the Python object for one loaded value; mapping entries keep their document order. +pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyResult { + match value { + ConfigValue::Null => Ok(py.None()), + ConfigValue::Bool(flag) => object(py, flag), + ConfigValue::Integer(number) => object(py, number), + ConfigValue::BigInteger(text) => big_integer(py, &text), + ConfigValue::Float(number) => object(py, number), + ConfigValue::String(text) => object(py, text), + ConfigValue::Date(value) => date(&py.import("datetime")?, value), + ConfigValue::DateTime(value) => date_time(&py.import("datetime")?, value), + ConfigValue::Time(value) => time(&py.import("datetime")?, value), + ConfigValue::List(items) => { + let list = PyList::empty(py); + for item in items { + list.append(config_value_to_python(py, item)?)?; + } + Ok(list.unbind().into_any()) + } + ConfigValue::Map(entries) => { + let mapping = PyDict::new(py); + for (key, item) in entries { + mapping.set_item( + config_value_to_python(py, key)?, + config_value_to_python(py, item)?, + )?; + } + Ok(mapping.unbind().into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index 3a6e71b43b..52261b6dff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -20,10 +20,13 @@ use sqlbuild_discovery::sql_tests::models::{ }; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; +use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Mutex; +use crate::bindings::_helpers::config_values::config_value_to_python; use crate::bindings::_helpers::functions::map_to_python; use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree, SqlTestDiscoveryRequest}; use crate::bindings::types::CompilerDetach; @@ -297,7 +300,45 @@ fn discover_scenario_files( } } +/// One file's payload; a value Python cannot hold natively makes Python load that file. +fn yaml_file_object(py: Python<'_>, outcome: YamlFileOutcome) -> PyResult { + match outcome { + YamlFileOutcome::Loaded { contents, value } => match config_value_to_python(py, value) { + Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), + Err(_unconvertible) => { + tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) + } + }, + YamlFileOutcome::LoadInPython { contents } => { + tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) + } + YamlFileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), + } +} + +/// Read and load YAML files natively, one payload per path, or `None` when Python must run. +#[pyfunction] +fn load_yaml_files( + py: Python<'_>, + relative_paths: Vec, + tree: &NativeProjectTree, +) -> PyResult>> { + let loaded: Result, StageDeferral> = py + .compiler_detach(|| Ok(load_yaml(&tree.inner, &relative_paths))) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match loaded { + Ok(outcomes) => Ok(Some( + outcomes + .into_iter() + .map(|outcome| yaml_file_object(py, outcome)) + .collect::>()?, + )), + Err(_deferral) => Ok(None), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; module.add_class::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 5e8b1378a6..8c2763913b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,5 +1,6 @@ mod allocator; mod catalog; +mod config_values; pub(crate) mod conversions; mod discovery; pub(crate) mod functions; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 94d9629367..2026ce2ab1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -59,6 +59,7 @@ def discover_sql_test_files( def discover_scenario_files( request: dict[str, object], tree: NativeProjectTree ) -> list[tuple[str, tuple[object, ...]]] | None: ... +def load_yaml_files(relative_paths: list[str], tree: NativeProjectTree) -> list[object] | None: ... def discover_declaration_layout( tree: NativeProjectTree, ) -> ( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 75e395f6c6..c8e4494128 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -44,6 +44,10 @@ discover_native_scenario_files, discover_native_test_files, ) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( + discover_native_schema_files, + discover_native_source_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( @@ -226,8 +230,10 @@ def _discover_declaration_files( native: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): - source_files: tuple[DiscoveredSourceFile, ...] = discover_source_files( - project_dir=project_dir, fact_cache=fact_cache + source_files: tuple[DiscoveredSourceFile, ...] = ( + discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache) + if native + else discover_source_files(project_dir=project_dir, fact_cache=fact_cache) ) model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() if native: @@ -241,7 +247,11 @@ def _discover_declaration_files( sql_function_files=discover_sql_function_files(project_dir=project_dir), sql_hook_files=discover_sql_hook_files(project_dir=project_dir), python_function_files=discover_python_function_files(project_dir=project_dir), - schema_files=discover_schema_files(project_dir=project_dir), + schema_files=( + discover_native_schema_files(project_dir=project_dir) + if native + else discover_schema_files(project_dir=project_dir) + ), seed_files=discover_seed_files(project_dir=project_dir), test_files=( discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index a93247e3a1..6b74f2b4b1 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -43,15 +43,17 @@ prepare_matched_model_file_headers, ) from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file +from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( + schema_file_paths, + source_file_paths, +) from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - SCHEMA_FILE_NAME, SEED_FILE_SUFFIX, - YAML_FILE_SUFFIXES, ) from sqlbuild.compiler.discovery.exceptions import ( DeclarationParseError, @@ -59,7 +61,6 @@ ModelSqlParseError, ProviderDiscoveryError, PythonNodeDiscoveryError, - SchemaParseError, ) from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, @@ -683,27 +684,7 @@ def discover_python_function_files( def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: """Discover model schema.yml files and seed declaration .yml files.""" - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - schema_paths: list[Path] = [] - models_root: Path = project_dir / "models" - seeds_root: Path = project_dir / "seeds" - - if models_root.is_dir(): - schema_paths.extend( - path - for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) - if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) - ) - if seeds_root.is_dir(): - yaml_path: Path - for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): - raise SchemaParseError( - f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " - ".yaml is not supported" - ) - schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) - - deduped_paths: tuple[Path, ...] = tuple(dict.fromkeys(schema_paths)) + deduped_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) discovered_schema_files: list[DiscoveredSchemaFile] = [] file_path: Path for file_path in deduped_paths: @@ -731,13 +712,7 @@ def discover_source_files( ) -> tuple[DiscoveredSourceFile, ...]: """Discover source declaration YAML files under sources/.""" - sources_root: Path = project_dir / "sources" - if not sources_root.is_dir(): - return () - - yaml_paths: tuple[Path, ...] = tuple( - sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) - ) + yaml_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) def parse(file_path: Path) -> DiscoveredSourceFile: return parse_source_file_with_cache( diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 01bd2ff86f..aeff70119d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -106,6 +106,12 @@ def _snapshot_entries( ) +def native_path_text_supported(text: str) -> bool: + """Whether a path's text crosses to the native engine unchanged (no surrogate escapes).""" + + return _is_utf8_text(text) + + def _is_utf8_text(text: str) -> bool: try: _ = text.encode(_PATH_ENCODING) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py new file mode 100644 index 0000000000..a374d1b213 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py @@ -0,0 +1,172 @@ +"""Source and schema YAML files loaded by the native engine, with per-file Python loading.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( + parse_source_file_with_cache, +) +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_source_files +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import project_relative_path +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, + native_display_prefix, + native_path_text_supported, + native_project_tree, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( + schema_file_paths, + source_file_paths, +) +from sqlbuild.compiler.discovery._helpers.yml.schema import ( + parse_loaded_schema_yml, + parse_schema_yml, +) +from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery.constants import ( + NATIVE_LOADED_TAG, + NATIVE_UNREADABLE_TAG, + NATIVE_YAML_BATCH_BYTES, +) +from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSourceFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry + +type _Payload = tuple[object, ...] + + +def discover_native_source_files( + *, project_dir: Path, fact_cache: FactCacheStore | None = None +) -> tuple[DiscoveredSourceFile, ...]: + """Discover source YAML files natively; Python, with its fact cache, loads the rest.""" + + file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) + if not _native_supported(project_dir=project_dir, file_paths=file_paths): + return discover_source_files(project_dir=project_dir, fact_cache=fact_cache) + discovered: list[DiscoveredSourceFile] = [] + for file_path, payload in _native_payloads(project_dir=project_dir, file_paths=file_paths): + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) + if payload[0] == NATIVE_LOADED_TAG: + _tag, contents, loaded = payload + discovered.append( + DiscoveredSourceFile( + file_path=file_path, + relative_path=relative_path, + contents=str(contents), + source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), + ) + ) + else: + discovered.append( + parse_source_file_with_cache( + project_dir=project_dir, + file_path=file_path, + relative_path=relative_path, + fact_cache=fact_cache, + ) + ) + return tuple(discovered) + + +def discover_native_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: + """Discover schema YAML files natively; Python loads any file native cannot reproduce.""" + + file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) + payloads: Iterator[tuple[Path, _Payload]] = ( + _native_payloads(project_dir=project_dir, file_paths=file_paths) + if _native_supported(project_dir=project_dir, file_paths=file_paths) + else ((file_path, (NATIVE_UNREADABLE_TAG,)) for file_path in file_paths) + ) + discovered: list[DiscoveredSchemaFile] = [] + for file_path, payload in payloads: + contents: str + entries: tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]] + if payload[0] == NATIVE_LOADED_TAG: + _tag, native_contents, loaded = payload + contents = str(native_contents) + entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) + else: + contents = _python_contents(file_path=file_path, payload=payload) + entries = parse_schema_yml(contents=contents, file_path=file_path) + discovered.append( + DiscoveredSchemaFile( + file_path=file_path, + relative_path=project_relative_path(path=file_path, project_dir=project_dir), + contents=contents, + model_entries=entries[0], + seed_entries=entries[1], + ) + ) + return tuple(discovered) + + +def _python_contents(*, file_path: Path, payload: _Payload) -> str: + read: Callable[[], str] = { + True: lambda: file_path.read_text(encoding="utf-8"), + False: lambda: str(payload[1]), + }[payload[0] == NATIVE_UNREADABLE_TAG] + return read() + + +def _native_supported(*, project_dir: Path, file_paths: tuple[Path, ...]) -> bool: + return ( + bool(file_paths) + and native_discovery_supported( + project_dir=project_dir, display_prefix=native_display_prefix(project_dir) + ) + and all( + native_path_text_supported(_relative_text(project_dir=project_dir, file_path=file_path)) + for file_path in file_paths + ) + ) + + +def _relative_text(*, project_dir: Path, file_path: Path) -> str: + return project_relative_path(path=file_path, project_dir=project_dir).as_posix() + + +def _native_payloads( + *, project_dir: Path, file_paths: tuple[Path, ...] +) -> Iterator[tuple[Path, _Payload]]: + """Load the files natively in batches bounded by size, releasing each file once used.""" + + tree: _native.NativeProjectTree = native_project_tree(project_dir) + for batch in _batches(file_paths=file_paths): + payloads: list[object] | None = _native.load_yaml_files( + [_relative_text(project_dir=project_dir, file_path=path) for path in batch], tree + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + pending: list[_Payload] = ( + [(NATIVE_UNREADABLE_TAG,)] * len(batch) + if payloads is None + else [cast(_Payload, payload) for payload in reversed(payloads)] + ) + del payloads + for file_path in batch: + yield file_path, pending.pop() + + +def _batches(*, file_paths: tuple[Path, ...]) -> Iterator[tuple[Path, ...]]: + batch: list[Path] = [] + batch_bytes: int = 0 + for file_path in file_paths: + size: int = _file_size(file_path) + if batch and batch_bytes + size > NATIVE_YAML_BATCH_BYTES: + yield tuple(batch) + batch, batch_bytes = [], 0 + batch.append(file_path) + batch_bytes += size + if batch: + yield tuple(batch) + + +def _file_size(file_path: Path) -> int: + try: + return file_path.stat().st_size + except OSError: + return 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py new file mode 100644 index 0000000000..39c9289c7a --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py @@ -0,0 +1,48 @@ +"""The source and schema YAML files discovery reads, in Python's discovery order.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( + is_in_scoped_declaration_tree, +) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.constants import SCHEMA_FILE_NAME, YAML_FILE_SUFFIXES +from sqlbuild.compiler.discovery.exceptions import SchemaParseError + + +def source_file_paths(*, project_dir: Path) -> tuple[Path, ...]: + """Return the YAML files directly below sources/, in path order.""" + + sources_root: Path = project_dir / "sources" + if not sources_root.is_dir(): + return () + return tuple( + sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) + ) + + +def schema_file_paths(*, project_dir: Path) -> tuple[Path, ...]: + """Return the unscoped model schema.yml files and seed declaration files, in Python's order.""" + + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + schema_paths: list[Path] = [] + models_root: Path = project_dir / "models" + seeds_root: Path = project_dir / "seeds" + + if models_root.is_dir(): + schema_paths.extend( + path + for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) + if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) + ) + if seeds_root.is_dir(): + yaml_path: Path + for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): + raise SchemaParseError( + f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " + ".yaml is not supported" + ) + schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) + return tuple(dict.fromkeys(schema_paths)) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py index 5feeaf44ff..2ae6db08d8 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py @@ -76,19 +76,33 @@ def parse_schema_yml( ) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: """Parse one schema.yml file into raw model and seed metadata.""" - payload: dict[str, object] = _load_schema_payload(contents=contents, file_path=file_path) + return parse_loaded_schema_yml( + loaded=_load_schema_yaml(contents=contents, file_path=file_path), file_path=file_path + ) + + +def parse_loaded_schema_yml( + *, loaded: object, file_path: Path +) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: + """Parse the document one schema.yml file loads to into raw model and seed metadata.""" + + payload: dict[str, object] = _schema_mapping(loaded=loaded, file_path=file_path) return ( _parse_model_entries(payload=payload, file_path=file_path), _parse_seed_entries(payload=payload, file_path=file_path), ) -def _load_schema_payload(*, contents: str, file_path: Path) -> dict[str, object]: +def _load_schema_yaml(*, contents: str, file_path: Path) -> object: try: loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - payload: object = yaml.load(contents, Loader=loader) + return yaml.load(contents, Loader=loader) except YAMLError as error: raise SchemaParseError(f"{file_path} contains invalid YAML: {error}") from error + + +def _schema_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: + payload: object = loaded if payload is None: return {} if not isinstance(payload, dict): diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py index 0813d61a05..2a6bcd2c68 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py @@ -92,7 +92,15 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: """Parse one sources/*.yml file into raw source declarations.""" - payload: dict[str, object] = _load_sources_payload(contents=contents, file_path=file_path) + return parse_loaded_sources_yml( + loaded=_load_sources_yaml(contents=contents, file_path=file_path), file_path=file_path + ) + + +def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[SourceEntry, ...]: + """Parse the document one sources/*.yml file loads to into raw source declarations.""" + + payload: dict[str, object] = _sources_mapping(loaded=loaded, file_path=file_path) _reject_unknown_source_keys( mapping=payload, allowed=_SOURCE_FILE_KEYS, file_path=file_path, label="sources file" ) @@ -112,12 +120,16 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, . return tuple(parsed_sources) -def _load_sources_payload(*, contents: str, file_path: Path) -> dict[str, object]: +def _load_sources_yaml(*, contents: str, file_path: Path) -> object: try: loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - payload: object = yaml.load(contents, Loader=loader) + return yaml.load(contents, Loader=loader) except YAMLError as error: raise SourceParseError(f"{file_path} contains invalid YAML: {error}") from error + + +def _sources_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: + payload: object = loaded if payload is None: return {} if not isinstance(payload, dict): diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index ca42bc8b71..2b02d9f48b 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -186,3 +186,5 @@ DISCOVERY_SQL_TEST_FACT_KIND: str = "sql_test" NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" +NATIVE_LOADED_TAG: str = "ok" +NATIVE_YAML_BATCH_BYTES: int = 4 * 1024 * 1024 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index faafd4e6a0..37de05fda3 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -118,3 +118,34 @@ class GeneratedSqlTestParityTestCase: expected_mismatches: tuple[tuple[object, object, object], ...] = () expected_minimum_parsed: int = 0 expected_minimum_failed: int = 0 + + +@dataclass(frozen=True) +class GeneratedYamlFileParityTestCase: + """Seeded source and seed declaration files discovered by both compiler engines.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + expected_minimum_parsed: int = 0 + expected_minimum_failed: int = 0 + + +@dataclass(frozen=True) +class NativeYamlLoadTestCase: + """Source files whose native load outcome and engine parity are both checked.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_native_tags: tuple[str, ...] + + +@dataclass(frozen=True) +class FactCacheFallbackTestCase: + """A runtime under which native source and test discovery must use the fact cache or not.""" + + description: str + unidata_version: str + expected_same_keys_as_python: bool + expected_native_keys: int diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 7154c12f48..c894609cb0 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -24,6 +24,8 @@ discover_macro_files, discover_model_files, discover_scenario_files, + discover_schema_files, + discover_source_files, discover_test_files, ) from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( @@ -37,6 +39,10 @@ discover_native_scenario_files, discover_native_test_files, ) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( + discover_native_schema_files, + discover_native_source_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.exceptions import DeclarationParseError @@ -48,6 +54,7 @@ DiscoveredSqlModelFile, ) from sqlbuild.compiler.discovery.types import NativeDeclarationFact +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.scopes.constants import NAMED_DECLARATION_KINDS @@ -278,7 +285,21 @@ def finished_document(*, rng: random.Random, text: str) -> str: return rng.choice(BYTE_ORDER_MARKS) + ending +LONG_KEY_CONTEXTS: tuple[str, ...] = ( + "{key}: 1\n", "{{{key}: 1}}\n", "[{key}: 1]\n", "a:\n {key} : 1\n", "'{key}': 1\n", + "&k {key}: 1\n", "? {key}\n: 1\n", "x: {{{key}: 1, b: 2}}\n", +) # fmt: skip + + +def long_key_document(*, rng: random.Random) -> str: + """Return a document whose key length straddles the 1024-character simple-key limit.""" + + key: str = rng.choice(("a", "0x", "é")) * rng.randint(990, 1040) + return rng.choice(LONG_KEY_CONTEXTS).format(key=key) + + YAML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "long keys": lambda rng: long_key_document(rng=rng), "scalars": lambda rng: finished_document(rng=rng, text=scalar_document(rng=rng)), "anchors": lambda rng: finished_document(rng=rng, text=anchored_document(rng=rng)), "dumped": lambda rng: finished_document(rng=rng, text=dumped_document(rng=rng)), @@ -746,6 +767,159 @@ def sql_test_discovery_outcome(*, project_dir: Path, native: bool) -> object: }[failure is None] +SOURCE_FIELD_CHOICES: dict[str, tuple[object, ...]] = { + "description": ("Orders feed.", "", 3, None), + "database": ("raw", ""), + "schema": ("shop", None), + "table": ("orders", 7), + "managed": (True, False, "yes"), + "write_strategy": ("append", "merge", "table", "bogus"), + "unique_key": ("order_id", ["order_id", "line"], [], ""), + "cursor_column": ("updated_at",), + "load_batch_size": (100, 0, True), + "type_enforcement": (True, "no"), + "contract": ("enforced", "none", "strict"), + "freshness": ( + {"strategy": "adapter"}, + {"strategy": "column", "column": "updated_at", "type": "timestamp", "lag_tolerance": "2h"}, + {"strategy": "sql", "query": "SELECT 1", "type": "timestamp"}, + {"strategy": "column"}, + ), + "audits": (["not_null"], [{"accepted_values": {"values": [1, 2]}}], "x"), + "freshnes": ("1d",), +} +COLUMN_FIELD_CHOICES: dict[str, tuple[object, ...]] = { + "type": ("INTEGER", "VARCHAR", ""), + "nullable": (True, False, "yes", None), + "description": ("Order id.", 5), + "audits": (["not_null"], ["unique"], []), +} +SCALAR_META_TEMPLATES: tuple[str, ...] = ( + "\n meta: {{flag: {plain}}}", "\n meta:\n seen: {plain}", + "\n meta: &shared{n} {{k: {plain}}}", "\n description: '{single}'", + "\n <<: {{description: '{single}'}}", "\n meta: {{k: {plain}, j: [{plain}]}}", +) # fmt: skip + + +def _random_fields( + *, rng: random.Random, choices: dict[str, tuple[object, ...]] +) -> dict[str, object]: + keys: list[str] = rng.sample(sorted(choices), k=rng.randint(0, 3)) + return { + key: rng.choices(choices[key], weights=(6, *(1,) * (len(choices[key]) - 1)))[0] + for key in keys + } + + +def _generated_column(*, rng: random.Random, names: tuple[str, ...]) -> dict[str, object]: + return {"name": rng.choice(names), **_random_fields(rng=rng, choices=COLUMN_FIELD_CHOICES)} + + +def _generated_entry( + *, + rng: random.Random, + names: tuple[str, ...], + column_names: tuple[str, ...], + fields: dict[str, object], +) -> dict[str, object]: + return { + "name": rng.choice(names), + **fields, + "columns": [ + _generated_column(rng=rng, names=column_names) for _ in range(rng.randint(0, 3)) + ], + "meta": {"owner": random_yaml_value(rng=rng, depth=2)}, + } + + +def generated_source_document(*, rng: random.Random) -> str: + """Return a sources file of random entries in a random dump style, with resolver scalars.""" + + sources: list[object] = [ + _generated_entry( + rng=rng, + names=("orders", "customers", "line_items", ""), + column_names=("order_id", "amount", "status"), + fields=_random_fields(rng=rng, choices=SOURCE_FIELD_CHOICES), + ) + for _ in range(rng.randint(0, 3)) + ] + flow_style: bool | None = rng.choice((None, False, False, True)) + dumped: str = yaml.safe_dump( + {"sources": sources}, + default_flow_style=flow_style, + default_style=rng.choice((None, None, None, '"', "'")), + width=rng.choice((40, 80, 4096)), + allow_unicode=rng.random() < 0.5, + sort_keys=False, + ) + scalar: str = random_scalar(rng=rng) + tail: str = rng.choice(SCALAR_META_TEMPLATES).format( + plain=scalar, single=scalar.replace("'", "''"), n=rng.randint(0, 9) + ) + extra: str = rng.choices((f"- name: extra{tail}\n", "other: [\n"), weights=(8, 1))[0] + appendable: bool = flow_style is False and bool(sources) + return rng.choice(BYTE_ORDER_MARKS) + dumped + {True: extra, False: ""}[appendable] + + +def generated_seed_declaration(*, rng: random.Random) -> str: + """Return a seed declaration file of random entries in a random dump style.""" + + seeds: list[object] = [ + _generated_entry( + rng=rng, + names=("channels", "regions", ""), + column_names=("id", "label"), + fields={"description": rng.choice(("Order channels.", 1))}, + ) + for _ in range(rng.randint(0, 2)) + ] + return yaml.safe_dump( + {"seeds": seeds}, + default_flow_style=rng.choice((None, False, True)), + default_style=rng.choice((None, None, '"')), + sort_keys=False, + ) + + +def native_yaml_tags_and_values( + *, project_dir: Path, relative_paths: list[str] +) -> tuple[list[object], bool]: + """Return each file's native tag and whether every natively loaded value equals LibYAML's.""" + + payloads: list[object] = ( + _native.load_yaml_files(relative_paths, _native.NativeProjectTree(str(project_dir))) or [] + ) + rows: list[tuple[object, ...]] = [cast(tuple[object, ...], payload) for payload in payloads] + loaded: list[tuple[object, ...]] = list(compress(rows, [row[0] == "ok" for row in rows])) + expected: list[object] = [yaml.load(str(row[1]), Loader=yaml.CSafeLoader) for row in loaded] + return [row[0] for row in rows], [row[2] for row in loaded] == expected + + +def yaml_discovery_outcome(*, project_dir: Path, native: bool) -> object: + """Return the rendered source and schema files, or the error type, message and help.""" + + discover: Callable[[], tuple[object, object]] = { + True: lambda: ( + discover_native_source_files(project_dir=project_dir), + discover_native_schema_files(project_dir=project_dir), + ), + False: lambda: ( + discover_source_files(project_dir=project_dir), + discover_schema_files(project_dir=project_dir), + ), + }[native] + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with DirectorySnapshot.scope(project_dir=project_dir), capture: + rendered[0] = render_stage_capture(discover()) + failure: BaseException | None = capture.failure + return { + True: rendered[0], + False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), + }[failure is None] + + class FailureCapture: """Swallow and keep the discovery failure a block raises.""" @@ -968,6 +1142,45 @@ def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyP return (rendered[0], type(capture.failure).__name__, str(capture.failure)) +class RecordingFactCache: + """A fact cache that records the fact keys discovery asks for and caches nothing.""" + + enabled: bool = True + + def __init__(self) -> None: + self.keys: list[tuple[str, ...]] = [] + + def key(self, *parts: str) -> str: + self.keys.append(parts) + return str(len(self.keys)) + + def read_many(self, entries: object) -> dict[str, object]: + return {} + + def stage(self, *, key: str, slot: str, value: object) -> None: + return None + + +def fact_cache_keys(*, project_dir: Path, native: bool) -> list[tuple[str, ...]]: + """Return the fact keys source and test discovery request under the chosen path.""" + + cache: RecordingFactCache = RecordingFactCache() + fact_cache: FactCacheStore = cast(FactCacheStore, cache) + discover: Callable[[], object] = { + True: lambda: ( + discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache), + discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache), + ), + False: lambda: ( + discover_source_files(project_dir=project_dir, fact_cache=fact_cache), + discover_test_files(project_dir=project_dir, fact_cache=fact_cache), + ), + }[native] + with DirectorySnapshot.scope(project_dir=project_dir): + _ = discover() + return cache.keys + + class CallCounter: """Count calls and delegate to the wrapped function.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py new file mode 100644 index 0000000000..6f6ce814e9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py @@ -0,0 +1,219 @@ +"""Engine parity for source and schema YAML files loaded natively with per-file deferral.""" + +from __future__ import annotations + +import random +import unicodedata +from pathlib import Path + +import pytest + +import sqlbuild._native as _native +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + FactCacheFallbackTestCase, + GeneratedYamlFileParityTestCase, + NativeYamlLoadTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + fact_cache_keys, + generated_seed_declaration, + generated_source_document, + native_yaml_tags_and_values, + stage_outcome, + write_project, + yaml_discovery_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_SOURCES: bytes = ( + b"sources:\n - name: raw_orders\n description: Orders feed.\n" + b" columns:\n - name: order_id\n type: INTEGER\n" +) +_BIG_HEX: str = "0x" + "f" * 4000 +_BIG_OCTAL: str = "0" + "7" * 4200 +_BIG_BINARY: str = "0b" + "1" * 4200 +_SEED: bytes = ( + b"seeds:\n - name: channels\n description: Channels.\n" + b" columns:\n - name: id\n type: INTEGER\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedYamlFileParityTestCase( + description="seeded source and seed declarations in random YAML styles", + seed=81, + case_count=800, + expected_minimum_parsed=60, + expected_minimum_failed=100, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_yaml_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: GeneratedYamlFileParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[tuple[tuple[str, bytes], ...]] = [ + ( + ("sources/orders.yml", generated_source_document(rng=rng).encode("utf-8")), + ("seeds/channels.yml", generated_seed_declaration(rng=rng).encode("utf-8")), + ("seeds/channels.csv", b"id,label\n1,web\n"), + ) + for _ in range(test_case.case_count) + ] + project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] + for project_dir, project_files in zip(project_dirs, files, strict=True): + write_project(project_dir=project_dir, files=project_files) + expected: list[object] = [ + yaml_discovery_outcome(project_dir=project_dir, native=False) + for project_dir in project_dirs + ] + + actual: list[object] = [ + yaml_discovery_outcome(project_dir=project_dir, native=True) for project_dir in project_dirs + ] + + parsed: int = sum(isinstance(outcome, str) for outcome in expected) + assert ( + mismatches(inputs=list(files), expected=expected, actual=actual), + parsed >= test_case.expected_minimum_parsed, + len(expected) - parsed >= test_case.expected_minimum_failed, + ) == (list(test_case.expected_mismatches), True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="sources, model schema and seed declarations", + files=( + ("sources/orders.yml", _SOURCES), + ("sources/more.yaml", _SOURCES.replace(b"raw_orders", b"raw_customers")), + ("sources/notes.txt", b"ignored"), + ("models/orders.sql", b'MODEL (description "Orders");\nSELECT 1 AS order_id'), + ("models/marts/schema.yml", b"models: []\n"), + ("models/marts/_sqlbuild/schemas/schema.yml", b"models: []\n"), + ("seeds/channels.yml", _SEED), + ("seeds/channels.csv", b"id\n1\n"), + ), + ), + EngineSwitchParityTestCase( + description="a Python-only tag and invalid YAML load in Python with its error", + files=( + ("sources/a.yml", b"sources: []\nextra: !!set {a: null}\n"), + ("sources/b.yml", b"sources: [\n"), + ), + ), + EngineSwitchParityTestCase( + description="an unreadable source file fails with Python's read error", + files=(("sources/a.yml", b"sources: []\n# \xff\n"),), + ), + EngineSwitchParityTestCase( + description="a directory named like a source file fails as Python reads it", + files=(("sources/folder.yml/inner.txt", b""),), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_files_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = stage_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + + native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) + + assert (native == python) is test_case.expected_identical, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + NativeYamlLoadTestCase( + description="big hexadecimal, octal and binary integers load as values and explicit keys", + files=( + ( + "sources/a.yml", + ( + f"sources:\n - name: orders\n meta:\n hex: {_BIG_HEX}\n" + f" octal: {_BIG_OCTAL}\n binary: {_BIG_BINARY}\n" + f" ? {_BIG_HEX}\n : key\n ? {_BIG_BINARY}\n : key\n" + ).encode(), + ), + ), + expected_native_tags=("ok",), + ), + NativeYamlLoadTestCase( + description="an implicit key beyond PyYAML's simple-key limit is left to Python", + files=(("sources/a.yml", f"sources: []\nmeta: {{{_BIG_HEX}: key}}\n".encode()),), + expected_native_tags=("load",), + ), + NativeYamlLoadTestCase( + description="an earlier invalid file still fails before a later big integer", + files=( + ("sources/a.yml", b"sources: [\n"), + ("sources/b.yml", f"sources: []\nmeta: {{v: {_BIG_HEX}}}\n".encode()), + ), + expected_native_tags=("load", "ok"), + ), + ], + ids=lambda case: case.description, +) +def test_given_big_integers_when_loading_natively_then_values_and_error_order_match_python( + test_case: NativeYamlLoadTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: object = yaml_discovery_outcome(project_dir=tmp_path, native=False) + + native: object = yaml_discovery_outcome(project_dir=tmp_path, native=True) + tags, values_match = native_yaml_tags_and_values( + project_dir=tmp_path, relative_paths=[path for path, _contents in test_case.files] + ) + + assert (native, tags, values_match) == ( + python, + list(test_case.expected_native_tags), + True, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + FactCacheFallbackTestCase( + description="when native defers, Python discovery uses the fact cache as usual", + unidata_version="0.0.0", + expected_same_keys_as_python=True, + expected_native_keys=2, + ), + FactCacheFallbackTestCase( + description="native discovery itself reads no cached facts", + unidata_version=_native.PYTHON_ALNUM_UNICODE_VERSION, + expected_same_keys_as_python=False, + expected_native_keys=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_native_deferral_when_discovering_sources_and_tests_then_fact_cache_is_used( + test_case: FactCacheFallbackTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project( + project_dir=tmp_path, + files=( + ("sources/orders.yml", _SOURCES), + ("tests/unit/orders.sql", b'TEST (name "keeps_orders");\nSELECT 1\n'), + ), + ) + python: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=False) + monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) + + native: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=True) + + assert ((native == python), len(native)) == ( + test_case.expected_same_keys_as_python, + test_case.expected_native_keys, + ), test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py index 132ce747ef..33913e49c1 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py @@ -47,6 +47,13 @@ case_count=1500, expected_maximum_deferred=20, ), + GeneratedDocumentOracleTestCase( + description="keys straddling the 1024-character simple-key limit", + generator="long keys", + seed=44, + case_count=600, + expected_maximum_deferred=400, + ), GeneratedDocumentOracleTestCase( description="documents written by PyYAML's safe dumper in random styles", generator="dumped",