From 905a785f223eb615c2a55c89cffbb1e99707c914 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:55:14 +0100 Subject: [PATCH 1/4] feat(native): parse declaration files natively in sqlbuild-discovery Add native listing, reading and parsing for enum, constant, model schema, audit, SQL hook, SQL function, seed and macro files, in Python's discovery order, with a deferred outcome for inputs only Python can judge and a session that retains each collection and its layout for later stages. --- .../{model_files => }/_helpers/locations.rs | 0 crates/sqlbuild-discovery/src/_helpers/mod.rs | 1 + .../_helpers/checks/identities.rs | 77 ++++ .../declaration_files/_helpers/checks/mod.rs | 3 + .../_helpers/checks/python_values.rs | 122 ++++++ .../_helpers/checks/stops.rs | 16 + .../_helpers/listing/collection_reads.rs | 127 ++++++ .../_helpers/listing/collections.rs | 272 ++++++++++++ .../_helpers/listing/layouts.rs | 36 ++ .../declaration_files/_helpers/listing/mod.rs | 3 + .../src/declaration_files/_helpers/mod.rs | 3 + .../_helpers/parsing/audit_files.rs | 406 ++++++++++++++++++ .../_helpers/parsing/audit_text.rs | 190 ++++++++ .../_helpers/parsing/constant_files.rs | 119 +++++ .../_helpers/parsing/declaration_headers.rs | 154 +++++++ .../_helpers/parsing/enum_files.rs | 148 +++++++ .../_helpers/parsing/function_files.rs | 52 +++ .../_helpers/parsing/hook_files.rs | 144 +++++++ .../declaration_files/_helpers/parsing/mod.rs | 8 + .../_helpers/parsing/schema_files.rs | 106 +++++ .../main/discover_declaration_collection.rs | 23 + .../src/declaration_files/main/mod.rs | 3 + .../main/parse_declaration_text.rs | 46 ++ .../main/retained_collection.rs | 34 ++ .../src/declaration_files/mod.rs | 7 + .../src/declaration_files/models.rs | 255 +++++++++++ .../src/declaration_files/tests.rs | 8 + .../src/declaration_files/tests/helpers.rs | 84 ++++ .../tests/test_discovery_session.rs | 89 ++++ .../tests/test_parse_declaration_text.rs | 215 ++++++++++ .../src/declaration_files/tests/test_types.rs | 20 + .../src/declaration_files/types.rs | 6 + crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/model_files/_helpers/mod.rs | 1 - .../model_files/_helpers/output_columns.rs | 2 +- .../src/model_files/_helpers/parsing.rs | 2 +- crates/sqlbuild-discovery/src/models.rs | 11 + .../_helpers/discovery/project_files.rs | 24 +- 38 files changed, 2807 insertions(+), 11 deletions(-) rename crates/sqlbuild-discovery/src/{model_files => }/_helpers/locations.rs (100%) create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/identities.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/layouts.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_text.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/declaration_headers.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/function_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/main/discover_declaration_collection.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/main/retained_collection.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/mod.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/models.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/tests.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/tests/test_discovery_session.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/tests/test_types.rs create mode 100644 crates/sqlbuild-discovery/src/declaration_files/types.rs diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs b/crates/sqlbuild-discovery/src/_helpers/locations.rs similarity index 100% rename from crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs rename to crates/sqlbuild-discovery/src/_helpers/locations.rs diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs index d7ec3c7a5c..e331bbba87 100644 --- a/crates/sqlbuild-discovery/src/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod header_keys; +pub(crate) mod locations; pub(crate) mod pool; pub(crate) mod reading; pub(crate) mod scoped_paths; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/identities.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/identities.rs new file mode 100644 index 0000000000..66b2b74795 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/identities.rs @@ -0,0 +1,77 @@ +//! Python's `validate_resource_identity` for declaration names that are already SQL identifiers. + +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::models::{DiscoveryFailure, FailureKind}; + +const IDENTITY_HELP: &str = "Rename the authored identity and update its references, selectors, \ + and integration keys. SQLBuild does not silently normalize resource \ + identities. Double underscores remain valid."; +const FALLBACK_IDENTITY: &str = "resource_name"; + +/// Reject a public identity unless it is lowercase snake_case; `name` matches `[A-Za-z_]\w*`. +pub(crate) fn validate_public_identity( + name: &str, + kind: &str, + file_path: &str, +) -> Result<(), ParseStop> { + if is_snake_case(name) { + return Ok(()); + } + Err(ParseStop::Failed(DiscoveryFailure { + kind: FailureKind::ResourceIdentity, + message: format!( + "Invalid public {kind} identity '{name}' in {file_path}; use snake_case '{}'", + suggested_identity(name) + ), + help: Some(IDENTITY_HELP.to_owned()), + })) +} + +/// `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`. +fn is_snake_case(name: &str) -> bool { + let bytes = name.as_bytes(); + let tail_ok = |byte: &u8| byte.is_ascii_lowercase() || byte.is_ascii_digit(); + match bytes { + [] => false, + [first] => first.is_ascii_lowercase(), + [first, middle @ .., last] => { + first.is_ascii_lowercase() + && middle.iter().all(|byte| tail_ok(byte) || *byte == b'_') + && tail_ok(last) + } + } +} + +/// `suggest_resource_identity` for an ASCII identifier: split acronyms and words, then lowercase. +fn suggested_identity(name: &str) -> String { + let characters: Vec = name.chars().collect(); + let mut acronyms: Vec = Vec::with_capacity(characters.len() * 2); + for (index, character) in characters.iter().enumerate() { + let splits = index > 0 + && characters[index - 1].is_ascii_uppercase() + && character.is_ascii_uppercase() + && characters + .get(index + 1) + .is_some_and(char::is_ascii_lowercase); + if splits { + acronyms.push('_'); + } + acronyms.push(*character); + } + let mut words: String = String::with_capacity(acronyms.len() * 2); + for (index, character) in acronyms.iter().enumerate() { + if index > 0 + && (acronyms[index - 1].is_ascii_lowercase() || acronyms[index - 1].is_ascii_digit()) + && character.is_ascii_uppercase() + { + words.push('_'); + } + words.push(character.to_ascii_lowercase()); + } + let corrected: &str = words.trim_matches('_'); + if corrected.is_empty() { + FALLBACK_IDENTITY.to_owned() + } else { + corrected.to_owned() + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/mod.rs new file mode 100644 index 0000000000..3017d6f4cc --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod identities; +pub(crate) mod python_values; +pub(crate) mod stops; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs new file mode 100644 index 0000000000..f59b447d76 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs @@ -0,0 +1,122 @@ +//! The Python type of a projected header value, as the discovery parsers' `isinstance` checks see it. + +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// The Python type a header value projects to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PythonType { + None, + Bool, + Int, + Float, + Str, + List, + Dict, + /// `AuthoredSqlSet`, a tuple, a `constant(...)` call or a hook entry. + Other, +} + +/// The projected type; a bare word with non-ASCII text may be a Unicode number, so it defers. +pub(crate) fn python_type(value: &AuthoredValue) -> Result { + Ok(match value { + AuthoredValue::Null => PythonType::None, + AuthoredValue::Boolean(_) => PythonType::Bool, + AuthoredValue::BareWord(word) => word_type(word)?, + AuthoredValue::String(_) => PythonType::Str, + AuthoredValue::List(_) => PythonType::List, + AuthoredValue::Map(_) => PythonType::Dict, + AuthoredValue::Set(_) + | AuthoredValue::Tuple(_) + | AuthoredValue::TypedConstant(_) + | AuthoredValue::InlineSqlHook(_) + | AuthoredValue::NamedSqlHook(..) + | AuthoredValue::PythonHook(..) => PythonType::Other, + }) +} + +/// The text of a value that projects to `str`, or `None` for any other type. +pub(crate) fn python_str(value: &AuthoredValue) -> Result, ParseStop> { + Ok(match value { + AuthoredValue::String(text) => Some(text), + AuthoredValue::BareWord(word) if word_type(word)? == PythonType::Str => Some(word), + _ => None, + }) +} + +/// `isinstance(value, str) and value.strip()`. +pub(crate) fn non_empty_str(value: &AuthoredValue) -> Result, ParseStop> { + Ok(python_str(value)?.filter(|text| !python_strip(text).is_empty())) +} + +/// Python's `dict.get(key)`. +pub(crate) fn get<'a>( + values: &'a [(String, AuthoredValue)], + key: &str, +) -> Option<&'a AuthoredValue> { + values + .iter() + .find(|(candidate, _)| candidate == key) + .map(|(_, value)| value) +} + +/// The keys outside `allowed`, sorted as Python's `sorted(set(values) - allowed)`. +pub(crate) fn unknown_keys(values: &[(String, AuthoredValue)], allowed: &[&str]) -> Vec { + let mut unknown: Vec = values + .iter() + .map(|(key, _)| key.clone()) + .filter(|key| !allowed.contains(&key.as_str())) + .collect(); + unknown.sort_unstable(); + unknown.dedup(); + unknown +} + +/// `re.fullmatch(r"^[A-Za-z_][A-Za-z0-9_]*$", text)`. +pub(crate) fn is_identifier(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// A failure of `kind` with `message` and no help. +pub(crate) fn failure(kind: FailureKind, message: String) -> ParseStop { + ParseStop::Failed(DiscoveryFailure::new(kind, message)) +} + +/// `_parse_word_value`: `true`, `false` and `null` are parsed as values; numbers become numbers. +fn word_type(word: &str) -> Result { + if !word.is_ascii() { + return Err(ParseStop::Deferred); + } + Ok(match word { + "true" | "false" => PythonType::Bool, + "null" => PythonType::None, + _ if is_integer(word) => PythonType::Int, + _ if is_float(word) => PythonType::Float, + _ => PythonType::Str, + }) +} + +fn unsigned(word: &str) -> &str { + word.strip_prefix(['+', '-']).unwrap_or(word) +} + +/// `^[+-]?\d+$` over ASCII text. +fn is_integer(word: &str) -> bool { + let digits = unsigned(word); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// `^[+-]?(?:\d+\.\d*|\d*\.\d+)$` over ASCII text. +fn is_float(word: &str) -> bool { + let Some((whole, fraction)) = unsigned(word).split_once('.') else { + return false; + }; + let digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit()); + digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty()) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs new file mode 100644 index 0000000000..3ce19f339b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs @@ -0,0 +1,16 @@ +//! Why native parsing of one declaration file stopped. + +use crate::models::DiscoveryFailure; + +/// The first failure Python raises for the file, or a file only the Python parser reproduces. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum ParseStop { + Failed(DiscoveryFailure), + Deferred, +} + +impl From for ParseStop { + fn from(failure: DiscoveryFailure) -> Self { + Self::Failed(failure) + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs new file mode 100644 index 0000000000..0c6eceefd3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs @@ -0,0 +1,127 @@ +//! Read and parse the files of one collection into its retained native form. + +use crate::declaration_files::_helpers::listing::collections::{ + NamedKind, declaration_files, file_name, function_files, named_declaration_files, read_files, + seed_files, +}; +use crate::declaration_files::_helpers::parsing::audit_files::parse_audit_file; +use crate::declaration_files::_helpers::parsing::constant_files::parse_constant_file; +use crate::declaration_files::_helpers::parsing::enum_files::parse_enum_file; +use crate::declaration_files::_helpers::parsing::function_files::parse_function_file; +use crate::declaration_files::_helpers::parsing::hook_files::parse_hook_file; +use crate::declaration_files::_helpers::parsing::schema_files::parse_schema_file; +use crate::declaration_files::models::{ + CollectionFailure, CollectionKind, CollectionRequest, DeclarationCollection, + DeclarationFileOptions, DiscoverySession, MacroFile, ScopedFile, SeedFile, +}; +use crate::declarations::models::DeclarationKind; +use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageFailure}; +use crate::tree::models::ProjectTree; + +/// The project a collection is read from and the options its parsers use. +pub(crate) struct CollectionSource<'a> { + pub(crate) session: &'a DiscoverySession, + pub(crate) tree: &'a ProjectTree, +} + +/// Read and parse the collection `request` names. +pub(crate) fn read_collection( + source: &CollectionSource<'_>, + request: CollectionRequest, +) -> DeclarationCollection { + let tree: &ProjectTree = source.tree; + let root: &ProjectRoot = &source.session.root; + let options: &DeclarationFileOptions = &source.session.options; + let isolate_kind: bool = request.isolate_kind; + match request.kind { + CollectionKind::Enums => DeclarationCollection::Enums( + declaration_files(source, DeclarationKind::Enum, isolate_kind).map(|files| { + read_files(root, tree, files, |file_path, _, contents| { + parse_enum_file(file_path, contents) + }) + }), + ), + CollectionKind::Constants => DeclarationCollection::Constants( + declaration_files(source, DeclarationKind::Constant, isolate_kind).map(|files| { + read_files(root, tree, files, |file_path, _, contents| { + parse_constant_file(file_path, contents) + }) + }), + ), + CollectionKind::Macros => DeclarationCollection::Macros( + declaration_files(source, DeclarationKind::Macro, isolate_kind).map(|files| { + read_files(root, tree, files, |_, _, contents| { + Ok(MacroFile { contents }) + }) + }), + ), + CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas( + named_declaration_files(source, &[NamedKind::Schema], false).map(|files| { + read_files(root, tree, files, |file_path, _, contents| { + parse_schema_file(file_path, contents) + }) + }), + ), + CollectionKind::Audits => DeclarationCollection::Audits( + named_declaration_files(source, &[NamedKind::Audit, NamedKind::SingularAudit], false) + .map(|files| { + read_files(root, tree, files, |file_path, _, contents| { + parse_audit_file(file_path, contents, options) + }) + }), + ), + CollectionKind::SqlHooks => DeclarationCollection::SqlHooks( + named_declaration_files(source, &[NamedKind::SqlHook], true).map(|files| { + read_files(root, tree, files, |file_path, relative_path, contents| { + parse_hook_file(file_path, stem(file_name(relative_path)), contents, options) + }) + }), + ), + CollectionKind::SqlFunctions => { + DeclarationCollection::SqlFunctions(function_files(tree).map(|files| { + read_files(root, tree, files, |file_path, _, contents| { + parse_function_file(file_path, contents, options) + }) + })) + } + CollectionKind::Seeds => DeclarationCollection::Seeds( + seed_files(tree).map(|paths| paths.into_iter().map(seed_file).collect()), + ), + } +} + +fn seed_file(relative_path: String) -> ScopedFile { + ScopedFile { + file: DiscoveredFile { + relative_path, + outcome: FileOutcome::Parsed(SeedFile), + }, + scope: None, + } +} + +/// The collection `kind` failing as a whole. +pub(crate) fn failed_collection( + kind: CollectionKind, + failure: StageFailure, +) -> DeclarationCollection { + let failure = CollectionFailure::Stage(failure); + match kind { + CollectionKind::Enums => DeclarationCollection::Enums(Err(failure)), + CollectionKind::Constants => DeclarationCollection::Constants(Err(failure)), + CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas(Err(failure)), + CollectionKind::SqlFunctions => DeclarationCollection::SqlFunctions(Err(failure)), + CollectionKind::SqlHooks => DeclarationCollection::SqlHooks(Err(failure)), + CollectionKind::Audits => DeclarationCollection::Audits(Err(failure)), + CollectionKind::Seeds => DeclarationCollection::Seeds(Err(failure)), + CollectionKind::Macros => DeclarationCollection::Macros(Err(failure)), + } +} + +/// Python's `Path.stem` of a file name. +fn stem(file_name: &str) -> &str { + match file_name.rfind('.') { + Some(index) if index > 0 && index + 1 < file_name.len() => &file_name[..index], + _ => file_name, + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs new file mode 100644 index 0000000000..7d51f06e03 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs @@ -0,0 +1,272 @@ +//! The files of each collection in Python's discovery order, read and parsed in parallel. + +use crate::_helpers::reading::read_authored_text; +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::listing::collection_reads::CollectionSource; +use crate::declaration_files::_helpers::listing::layouts::session_layout; +use crate::declaration_files::models::{CollectionFailure, FileScope, ScopedFile}; +use crate::declarations::models::{ + DeclarationFileFact, DeclarationGroup, DeclarationKind, DeclarationLayout, +}; +use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageFailure}; +use crate::tree::main::path_order::compare_posix_text; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use std::sync::Arc; + +/// One file to read, with the scope facts of the role that holds it. +pub(crate) type ListedFile = (String, Option); + +/// A named declaration role: `GLOBAL_NAMED_DECLARATION_ROOTS` and the grouped roles. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum NamedKind { + Audit, + SingularAudit, + Schema, + SqlHook, +} + +impl NamedKind { + fn as_str(self) -> &'static str { + match self { + Self::Audit => "audit", + Self::SingularAudit => "singular_audit", + Self::Schema => "schema", + Self::SqlHook => "sql_hook", + } + } +} + +/// `GLOBAL_NAMED_DECLARATION_ROOTS` without Python hooks, in Python's dict order. +const GLOBAL_NAMED_ROOTS: [(&str, NamedKind); 4] = [ + ("audits/generic", NamedKind::Audit), + ("audits/singular", NamedKind::SingularAudit), + ("schemas", NamedKind::Schema), + ("hooks/sql", NamedKind::SqlHook), +]; + +/// `GROUPED_NAMED_DECLARATION_ROLES` without Python hooks, in Python's dict order. +const GROUPED_NAMED_ROLES: [(&str, NamedKind, &str); 7] = [ + ("audits/generic", NamedKind::Audit, "inherited"), + ("_audits/generic", NamedKind::Audit, "local"), + ("audits/singular", NamedKind::SingularAudit, "inherited"), + ("schemas", NamedKind::Schema, "inherited"), + ("_schemas", NamedKind::Schema, "local"), + ("hooks/sql", NamedKind::SqlHook, "inherited"), + ("_hooks/sql", NamedKind::SqlHook, "local"), +]; + +const FUNCTIONS_ROOT: &str = "functions/sql"; +const SEEDS_ROOT: &str = "seeds"; +const SEED_FILE_SUFFIX: &str = ".csv"; + +/// The macro, enum or constant files of `kind`, from the whole layout or `kind`'s alone. +pub(crate) fn declaration_files( + source: &CollectionSource<'_>, + kind: DeclarationKind, + isolate_kind: bool, +) -> Result, CollectionFailure> { + let layout: Arc> = + session_layout(source.session, source.tree, isolate_kind.then_some(kind)); + let layout: &DeclarationLayout = layout + .as_ref() + .as_ref() + .map_err(|failure| CollectionFailure::Stage(failure.clone()))?; + let facts: &[DeclarationFileFact] = layout + .file_facts + .as_deref() + .map_err(|failure| CollectionFailure::Layout(failure.clone()))?; + Ok(facts + .iter() + .filter(|fact| fact.kind == kind) + .map(|fact| { + let scope = FileScope { + declaration_kind: fact.kind.as_str(), + scope_kind: fact.scope_kind.as_str(), + ownership_root: Some(fact.ownership_root.clone()), + owning_path: fact.owning_path.clone(), + declaration_root: Some(fact.declaration_root.clone()), + }; + (fact.relative_path.clone(), Some(scope)) + }) + .collect()) +} + +/// Every `*.sql` file below the named roles of `kinds`, root by root as Python yields them. +pub(crate) fn named_declaration_files( + source: &CollectionSource<'_>, + kinds: &[NamedKind], + skip_underscored: bool, +) -> Result, CollectionFailure> { + let tree: &ProjectTree = source.tree; + let mut roots: Vec<(String, FileScope)> = Vec::new(); + for (role, kind) in GLOBAL_NAMED_ROOTS { + if kinds.contains(&kind) && tree.is_dir(role) { + let scope = named_scope(&NamedRole { + kind, + scope_kind: "global", + ownership_root: None, + owning_path: None, + declaration_root: role, + }); + roots.push((role.to_owned(), scope)); + } + } + let layout: Arc> = + session_layout(source.session, tree, None); + let layout: &DeclarationLayout = layout + .as_ref() + .as_ref() + .map_err(|failure| CollectionFailure::Stage(failure.clone()))?; + let groups: &[DeclarationGroup] = layout + .named_groups + .as_deref() + .map_err(|failure| CollectionFailure::Layout(failure.clone()))?; + for group in groups { + let owning_path: &str = group + .directory + .rsplit_once('/') + .map_or("", |(parent, _)| parent); + for (role, kind, scope_kind) in GROUPED_NAMED_ROLES { + let directory: String = format!("{}/{role}", group.directory); + if kinds.contains(&kind) && tree.is_dir(&directory) { + let scope = named_scope(&NamedRole { + kind, + scope_kind, + ownership_root: Some(&group.root), + owning_path: Some(owning_path), + declaration_root: &directory, + }); + roots.push((directory, scope)); + } + } + } + roots.sort_by(|left, right| compare_posix_text(&left.0, &right.0)); + let mut files: Vec = Vec::new(); + for (directory, scope) in roots { + for relative_path in rglob(tree, &directory, |entry| matches_sql_glob(&entry.name)) + .map_err(CollectionFailure::Stage)? + { + if skip_underscored && file_name(&relative_path).starts_with('_') { + continue; + } + files.push((relative_path, Some(scope.clone()))); + } + } + Ok(files) +} + +/// One named declaration role directory and who owns it. +struct NamedRole<'a> { + kind: NamedKind, + scope_kind: &'static str, + ownership_root: Option<&'a str>, + owning_path: Option<&'a str>, + declaration_root: &'a str, +} + +fn named_scope(role: &NamedRole<'_>) -> FileScope { + FileScope { + declaration_kind: role.kind.as_str(), + scope_kind: role.scope_kind, + ownership_root: role.ownership_root.map(str::to_owned), + owning_path: role.owning_path.map(str::to_owned), + declaration_root: Some(role.declaration_root.to_owned()), + } +} + +/// The unscoped `*.sql` files below `functions/sql/`. +pub(crate) fn function_files(tree: &ProjectTree) -> Result, CollectionFailure> { + if !tree.is_dir(FUNCTIONS_ROOT) { + return Ok(Vec::new()); + } + Ok( + rglob(tree, FUNCTIONS_ROOT, |entry| matches_sql_glob(&entry.name)) + .map_err(CollectionFailure::Stage)? + .into_iter() + .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .map(|relative_path| (relative_path, None)) + .collect(), + ) +} + +/// The regular `*.csv` files below `seeds/`, following links as `Path.is_file()` does. +pub(crate) fn seed_files(tree: &ProjectTree) -> Result, CollectionFailure> { + if !tree.is_dir(SEEDS_ROOT) { + return Ok(Vec::new()); + } + let paths: Vec = rglob(tree, SEEDS_ROOT, |entry: &TreeEntry| { + has_seed_suffix(&entry.name) + }) + .map_err(CollectionFailure::Stage)?; + Ok(paths + .into_par_iter() + .filter(|relative_path| { + std::fs::metadata(tree.absolute(relative_path)).is_ok_and(|metadata| metadata.is_file()) + }) + .collect()) +} + +/// Python's `Path.suffix == ".csv"`. +fn has_seed_suffix(name: &str) -> bool { + name.len() > SEED_FILE_SUFFIX.len() && name.ends_with(SEED_FILE_SUFFIX) +} + +/// `fnmatch("*.sql")` with the platform's case rule: Windows ignores case as `re.IGNORECASE` does. +fn matches_sql_glob(name: &str) -> bool { + if !cfg!(windows) { + return name.ends_with(".sql"); + } + let tail: Vec = name.chars().rev().take(4).collect(); + matches!( + tail.as_slice(), + ['l' | 'L', 'q' | 'Q', 's' | 'S' | '\u{17f}', '.'] + ) +} + +/// The final component of a `/`-separated relative path. +pub(crate) fn file_name(relative_path: &str) -> &str { + relative_path + .rsplit_once('/') + .map_or(relative_path, |(_, name)| name) +} + +/// Read and parse each listed file in parallel; a name that is not UTF-8 defers to Python. +pub(crate) fn read_files( + root: &ProjectRoot, + tree: &ProjectTree, + files: Vec, + parse: impl Fn(&str, &str, String) -> Result + Sync, +) -> Vec> { + files + .into_par_iter() + .map(|(relative_path, scope)| ScopedFile { + file: DiscoveredFile { + outcome: read_one(root, tree, &relative_path, &parse), + relative_path, + }, + scope, + }) + .collect() +} + +fn read_one( + root: &ProjectRoot, + tree: &ProjectTree, + relative_path: &str, + parse: &impl Fn(&str, &str, String) -> Result, +) -> FileOutcome { + if tree.is_undecodable(relative_path) { + return FileOutcome::Deferred; + } + match read_authored_text(&tree.absolute(relative_path)) { + Ok(contents) => match parse(&root.display_path(relative_path), relative_path, contents) { + Ok(parsed) => FileOutcome::Parsed(parsed), + Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), + Err(ParseStop::Deferred) => FileOutcome::Deferred, + }, + Err(failure) => FileOutcome::Unreadable(failure), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/layouts.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/layouts.rs new file mode 100644 index 0000000000..a2e6888bff --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/layouts.rs @@ -0,0 +1,36 @@ +//! The declaration layout of a discovery session, walked once per kind as Python memoizes it. + +use crate::declaration_files::models::{DiscoverySession, RetainedLayout}; +use crate::declarations::main::declaration_layout::declaration_layout; +use crate::declarations::models::{DeclarationKind, DeclarationLayout}; +use crate::models::StageFailure; +use crate::tree::models::ProjectTree; +use std::sync::{Arc, MutexGuard, PoisonError}; + +/// The layout of `kind` (every kind with `None`), walked the first time the session asks for it. +pub(crate) fn session_layout( + session: &DiscoverySession, + tree: &ProjectTree, + kind: Option, +) -> Arc> { + if let Some(retained) = layouts(session) + .iter() + .find(|retained| retained.kind == kind) + { + return Arc::clone(&retained.layout); + } + let layout: Arc> = + Arc::new(declaration_layout(tree, kind)); + layouts(session).push(RetainedLayout { + kind, + layout: Arc::clone(&layout), + }); + layout +} + +fn layouts(session: &DiscoverySession) -> MutexGuard<'_, Vec> { + session + .layouts + .lock() + .unwrap_or_else(PoisonError::into_inner) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/mod.rs new file mode 100644 index 0000000000..16776e88f1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod collection_reads; +pub(crate) mod collections; +pub(crate) mod layouts; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/mod.rs new file mode 100644 index 0000000000..7e6b88226a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod checks; +pub(crate) mod listing; +pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs new file mode 100644 index 0000000000..ad089ae598 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs @@ -0,0 +1,406 @@ +//! Python's `parse_sql_audit_file`: split `AUDIT(...)` blocks and check each header and body. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::declaration_files::_helpers::checks::python_values::{ + PythonType, failure, get, non_empty_str, python_str, python_type, +}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::parsing::audit_text::{AuditText, chars, text}; +use crate::declaration_files::models::{AuditBlock, AuditFile, DeclarationFileOptions}; +use crate::models::FailureKind; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const AUDIT_KEYWORD: &str = "AUDIT"; +const STRING_KEYS: [&str; 7] = [ + "name", + "severity", + "run_scope", + "evaluation", + "value", + "sample_count", + "sample_unit", +]; +const BOOLEAN_KEYS: [&str; 2] = ["always_run", "sql_analysis"]; +const MEASUREMENT_ONLY_KEYS: [&str; 5] = [ + "value", + "sample_count", + "sample_unit", + "thresholds", + "minimum_samples", +]; +const VIOLATIONS: &str = "violations"; +const MEASUREMENT: &str = "measurement"; + +/// The file, its parsing options and the code points of the text being parsed. +struct AuditParse<'a> { + file_path: &'a str, + options: &'a DeclarationFileOptions, +} + +fn audit_failure(message: String) -> ParseStop { + failure(FailureKind::SqlAudit, message) +} + +pub(crate) fn parse_audit_file( + file_path: &str, + contents: String, + options: &DeclarationFileOptions, +) -> Result { + let parse = AuditParse { file_path, options }; + let content_chars: Vec = chars(&contents); + let content_text = AuditText { + chars: &content_chars, + python: options.python, + }; + let starts: Vec = content_text.top_level_keywords(AUDIT_KEYWORD); + let leading: String = starts + .first() + .map_or_else(String::new, |first| text(&content_chars[..*first])); + if starts.is_empty() || !python_strip(&leading).is_empty() { + return Err(audit_failure(format!( + "SQL audit '{file_path}' must start with an AUDIT() header as the first \ + non-whitespace content" + ))); + } + let mut blocks: Vec = Vec::with_capacity(starts.len()); + for (offset, start) in starts.iter().enumerate() { + let end: usize = starts + .get(offset + 1) + .copied() + .unwrap_or(content_chars.len()); + let raw_block: String = python_strip(&text(&content_chars[*start..end])).to_owned(); + let block_line: usize = content_chars[..*start] + .iter() + .filter(|character| **character == '\n') + .count() + + 1; + blocks.push(parse.block(&raw_block, block_line, offset + 1)?); + } + validate_audit_names(file_path, &blocks)?; + Ok(AuditFile { contents, blocks }) +} + +impl AuditParse<'_> { + fn text<'a>(&self, chars: &'a [char]) -> AuditText<'a> { + AuditText { + chars, + python: self.options.python, + } + } + + fn block( + &self, + raw_block: &str, + block_line: usize, + audit_index: usize, + ) -> Result { + let file_path: &str = self.file_path; + let block_chars: Vec = chars(raw_block); + let block = self.text(&block_chars); + let open_index: usize = self.keyword_open_paren(&block, 0, AUDIT_KEYWORD)?; + let close_index: usize = block + .matching_parenthesis(open_index) + .ok_or_else(|| audit_failure(format!("Unclosed AUDIT(...) block in '{file_path}'")))?; + let body_start: usize = + self.consume_delimiter(&block, close_index + 1, "AUDIT(...) header")?; + let header: String = text(&block_chars[open_index + 1..close_index]); + let header_line: usize = block_line + + block_chars[..=open_index] + .iter() + .filter(|character| **character == '\n') + .count(); + let header_values: Vec<(String, AuthoredValue)> = self.header(&header, header_line)?; + let evaluation_mode: &'static str = evaluation_mode(&header_values, file_path)?; + let body: &[char] = &block_chars[body_start..]; + let mut measure_sql: Option = None; + let mut evidence_sql: Option = None; + let sql_body: String = if evaluation_mode == MEASUREMENT { + let (measure, evidence) = self.measurement_body(body)?; + measure_sql = Some(measure.clone()); + evidence_sql = evidence; + measure + } else { + let sql_body: String = python_cleandoc(self.options.python, &text(body)); + if sql_body.is_empty() { + return Err(audit_failure(format!( + "SQL audit '{file_path}' must define SQL after AUDIT(...)" + ))); + } + let sql_chars: Vec = chars(&sql_body); + let sql = self.text(&sql_chars); + let code_start: usize = sql.skip_space_and_comments(0); + if sql.top_level_keywords("MEASURE").contains(&code_start) + || sql.top_level_keywords("EVIDENCE").contains(&code_start) + { + return Err(audit_failure(format!( + "Violation audit '{file_path}' must use a bare SELECT body, not \ + MEASURE/EVIDENCE" + ))); + } + sql_body + }; + let name: Option = match get(&header_values, "name") { + Some(value) => python_str(value)?.map(str::to_owned), + None => None, + }; + Ok(AuditBlock { + audit_index, + header_values, + sql_body, + name, + evaluation_mode, + measure_sql, + evidence_sql, + }) + } + + fn header( + &self, + header: &str, + header_line: usize, + ) -> Result, ParseStop> { + let file_path: &str = self.file_path; + let values: Vec<(String, AuthoredValue)> = parse_statement_header( + &StatementHeader { + kind: FailureKind::SqlAudit, + statement_name: AUDIT_KEYWORD, + statement: "AUDIT()", + file_path, + supported_keys: &self.options.audit_keys, + python: self.options.python, + }, + header, + header_line, + )?; + for key in STRING_KEYS { + if let Some(value) = get(&values, key) + && non_empty_str(value)?.is_none() + { + return Err(audit_failure(format!( + "AUDIT() {key} in '{file_path}' must be a non-empty string" + ))); + } + } + for key in BOOLEAN_KEYS { + if let Some(value) = get(&values, key) + && python_type(value)? != PythonType::Bool + { + return Err(audit_failure(format!( + "AUDIT() {key} in '{file_path}' must be a boolean" + ))); + } + } + Ok(values) + } + + /// `_keyword_open_paren` for the keyword at the first code after `start`. + fn keyword_open_paren( + &self, + text: &AuditText<'_>, + start: usize, + keyword: &str, + ) -> Result { + let missing = || { + audit_failure(format!( + "SQL audit '{}' must start with {keyword}(...)", + self.file_path + )) + }; + let keyword_start: usize = text.skip_space_and_comments(start); + if !keyword_at_slice_start(text, keyword, keyword_start, start) { + return Err(missing()); + } + let position: usize = text.skip_space_and_comments(keyword_start + keyword.len()); + if text.chars.get(position) != Some(&'(') { + return Err(missing()); + } + Ok(position) + } + + /// `_consume_delimiter`: the position after the `;` that must follow `start`. + fn consume_delimiter( + &self, + text: &AuditText<'_>, + start: usize, + label: &str, + ) -> Result { + let position: usize = text.skip_space_and_comments(start); + if text.chars.get(position) != Some(&';') { + return Err(audit_failure(format!( + "{label} in '{}' must end with `);`", + self.file_path + ))); + } + Ok(position + 1) + } + + /// `_parse_measurement_body`: one `MEASURE(...)` block and an optional `EVIDENCE(...)` block. + fn measurement_body(&self, body: &[char]) -> Result<(String, Option), ParseStop> { + let file_path: &str = self.file_path; + let text = self.text(body); + let mut position: usize = text.skip_space_and_comments(0); + if !text.keyword_at("MEASURE", position) { + return Err(audit_failure(format!( + "Measurement audit '{file_path}' must define exactly one MEASURE(...) block" + ))); + } + let (measure, end) = self.delimited_query(&text, position, "MEASURE")?; + position = text.skip_space_and_comments(end); + let mut evidence: Option = None; + if text.keyword_at("EVIDENCE", position) { + let (query, end) = self.delimited_query(&text, position, "EVIDENCE")?; + evidence = Some(query); + position = text.skip_space_and_comments(end); + } + if position != text.len() { + let label: &str = + if text.keyword_at("MEASURE", position) || text.keyword_at("EVIDENCE", position) { + "duplicate MEASURE/EVIDENCE block" + } else { + "bare SQL outside MEASURE/EVIDENCE blocks" + }; + return Err(audit_failure(format!( + "Measurement audit '{file_path}' has {label}" + ))); + } + Ok((measure, evidence)) + } + + /// `_extract_delimited_query` for a keyword already found at `position`. + fn delimited_query( + &self, + scan: &AuditText<'_>, + position: usize, + keyword: &str, + ) -> Result<(String, usize), ParseStop> { + let file_path: &str = self.file_path; + let open_index: usize = self.keyword_open_paren(scan, position, keyword)?; + let close_index: usize = scan.matching_parenthesis(open_index).ok_or_else(|| { + audit_failure(format!("Unclosed {keyword}(...) block in '{file_path}'")) + })?; + let query: String = python_cleandoc( + self.options.python, + &text(&scan.chars[open_index + 1..close_index]), + ); + self.validate_single_query(&query, keyword)?; + let end: usize = + self.consume_delimiter(scan, close_index + 1, &format!("{keyword}(...) block"))?; + Ok((query, end)) + } + + /// `_validate_single_query`: one `SELECT` or `WITH` query, optionally ending with `;`. + fn validate_single_query(&self, query: &str, label: &str) -> Result<(), ParseStop> { + let file_path: &str = self.file_path; + let one_select = || { + audit_failure(format!( + "{label}(...) in '{file_path}' must contain one SELECT query" + )) + }; + if query.is_empty() { + return Err(one_select()); + } + let query_chars: Vec = chars(query); + let query_text = self.text(&query_chars); + let start: usize = query_text.skip_space_and_comments(0); + if !(query_text.keyword_at("SELECT", start) || query_text.keyword_at("WITH", start)) { + return Err(one_select()); + } + let semicolons: Vec = query_text.top_level_semicolons(); + if let Some(first) = semicolons.first() + && (semicolons.len() > 1 + || query_chars[first + 1..] + .iter() + .any(|character| !is_python_space(*character))) + { + return Err(audit_failure(format!( + "{label}(...) in '{file_path}' must contain exactly one query" + ))); + } + Ok(()) + } +} + +/// `_keyword_at(text[start:], keyword, position - start)`: the slice hides what precedes it. +fn keyword_at_slice_start( + text: &AuditText<'_>, + keyword: &str, + position: usize, + slice_start: usize, +) -> bool { + if position == slice_start { + let slice = AuditText { + chars: &text.chars[slice_start..], + python: text.python, + }; + return slice.keyword_at(keyword, 0); + } + text.keyword_at(keyword, position) +} + +fn evaluation_mode( + header_values: &[(String, AuthoredValue)], + file_path: &str, +) -> Result<&'static str, ParseStop> { + let mode: &str = match get(header_values, "evaluation") { + Some(value) => python_str(value)?.unwrap_or_default(), + None => VIOLATIONS, + }; + let mode: &'static str = match mode { + VIOLATIONS => VIOLATIONS, + MEASUREMENT => MEASUREMENT, + _ => { + return Err(audit_failure(format!( + "AUDIT() evaluation in '{file_path}' must be one of: violations, measurement" + ))); + } + }; + if mode == VIOLATIONS { + let invalid: Vec<&str> = MEASUREMENT_ONLY_KEYS + .into_iter() + .filter(|key| get(header_values, key).is_some()) + .collect(); + if !invalid.is_empty() { + return Err(audit_failure(format!( + "Violation audit '{file_path}' must not define measurement keys: {}", + invalid.join(", ") + ))); + } + } else if get(header_values, "value").is_none() { + return Err(audit_failure(format!( + "Measurement audit '{file_path}' must define `value`" + ))); + } + Ok(mode) +} + +fn validate_audit_names(file_path: &str, blocks: &[AuditBlock]) -> Result<(), ParseStop> { + if blocks.len() <= 1 { + return Ok(()); + } + let unnamed: Vec = blocks + .iter() + .filter(|block| block.name.is_none()) + .map(|block| block.audit_index.to_string()) + .collect(); + if !unnamed.is_empty() { + return Err(audit_failure(format!( + "SQL audit '{file_path}' contains multiple AUDIT blocks; every block must define a \ + unique `name`. Missing names for blocks: {}", + unnamed.join(", ") + ))); + } + let mut seen: Vec<&str> = Vec::with_capacity(blocks.len()); + for block in blocks { + let name: &str = block.name.as_deref().unwrap_or_default(); + if seen.contains(&name) { + return Err(audit_failure(format!( + "SQL audit '{file_path}' defines duplicate AUDIT() name '{name}'" + ))); + } + seen.push(name); + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_text.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_text.rs new file mode 100644 index 0000000000..4142dede22 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_text.rs @@ -0,0 +1,190 @@ +//! The code-point scanning helpers Python's audit parser uses: comments, quotes and keywords. + +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; + +const SQL_QUOTES: [char; 3] = ['\'', '"', '`']; + +/// Audit text as code points, with the Python semantics its keyword checks use. +pub(crate) struct AuditText<'a> { + pub(crate) chars: &'a [char], + pub(crate) python: PythonText, +} + +impl AuditText<'_> { + pub(crate) fn len(&self) -> usize { + self.chars.len() + } + + fn starts_with(&self, index: usize, prefix: &[char]) -> bool { + self.chars + .get(index..) + .is_some_and(|rest| rest.starts_with(prefix)) + } + + /// `text.find(needle, start)`. + fn find(&self, needle: &[char], start: usize) -> Option { + (start..self.len()).find(|index| self.starts_with(*index, needle)) + } + + /// `_code_characters`: positions outside quotes and comments, from `start`. + pub(crate) fn code_positions(&self, start: usize) -> Vec { + let mut positions: Vec = Vec::new(); + let length: usize = self.len(); + let mut index: usize = start; + while index < length { + let character: char = self.chars[index]; + if self.starts_with(index, &['-', '-']) { + index = self + .find(&['\n'], index + 2) + .map_or(length, |newline| newline + 1); + continue; + } + if self.starts_with(index, &['/', '*']) { + index = self + .find(&['*', '/'], index + 2) + .map_or(length, |close| close + 2); + continue; + } + if SQL_QUOTES.contains(&character) { + index += 1; + while index < length { + if self.chars[index] == character { + if self.chars.get(index + 1) == Some(&character) { + index += 2; + continue; + } + index += 1; + break; + } + index += if self.chars[index] == '\\' { 2 } else { 1 }; + } + continue; + } + positions.push(index); + index += 1; + } + positions + } + + /// `_skip_space_and_comments`. + pub(crate) fn skip_space_and_comments(&self, start: usize) -> usize { + let length: usize = self.len(); + let mut position: usize = start; + while position < length { + if is_python_space(self.chars[position]) { + position += 1; + } else if self.starts_with(position, &['-', '-']) { + position = self + .find(&['\n'], position + 2) + .map_or(length, |newline| newline + 1); + } else if self.starts_with(position, &['/', '*']) { + position = self + .find(&['*', '/'], position + 2) + .map_or(length, |close| close + 2); + } else { + break; + } + } + position + } + + /// `_keyword_at`: `text[position:end].upper() == keyword` between non-word characters. + pub(crate) fn keyword_at(&self, keyword: &str, position: usize) -> bool { + let keyword: Vec = keyword.chars().collect(); + let end: usize = position + keyword.len(); + let Some(slice) = self.chars.get(position..end.min(self.len())) else { + return false; + }; + if slice.len() != keyword.len() + || !slice + .iter() + .zip(&keyword) + .all(|(character, expected)| upper_is(*character, *expected)) + { + return false; + } + let word = |index: Option| { + index + .and_then(|index| self.chars.get(index)) + .is_some_and(|character| { + *character == '_' || is_python_alnum(self.python, *character) + }) + }; + !word(position.checked_sub(1)) && !word(Some(end)) + } + + /// `_find_top_level_keywords`: keyword positions at depth 0 followed by `(`. + pub(crate) fn top_level_keywords(&self, keyword: &str) -> Vec { + let first: char = keyword.chars().next().unwrap_or_default(); + let mut depth: usize = 0; + let mut found: Vec = Vec::new(); + for index in self.code_positions(0) { + let character: char = self.chars[index]; + if character == '(' { + depth += 1; + } else if character == ')' { + depth = depth.saturating_sub(1); + } else if depth == 0 && upper_is(character, first) && self.keyword_at(keyword, index) { + let after: usize = self.skip_space_and_comments(index + keyword.chars().count()); + if self.chars.get(after) == Some(&'(') { + found.push(index); + } + } + } + found + } + + /// `_top_level_semicolons`. + pub(crate) fn top_level_semicolons(&self) -> Vec { + let mut depth: usize = 0; + let mut found: Vec = Vec::new(); + for index in self.code_positions(0) { + match self.chars[index] { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ';' if depth == 0 => found.push(index), + _ => {} + } + } + found + } + + /// `_find_matching_parenthesis`: the closing parenthesis of the one at `open_index`. + pub(crate) fn matching_parenthesis(&self, open_index: usize) -> Option { + let mut depth: usize = 1; + for index in self.code_positions(open_index + 1) { + match self.chars[index] { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + return Some(index); + } + } + _ => {} + } + } + None + } +} + +/// Whether Python's `character.upper()` is exactly the ASCII uppercase letter `expected`. +fn upper_is(character: char, expected: char) -> bool { + match character { + '\u{131}' => expected == 'I', + '\u{17f}' => expected == 'S', + _ => character.to_ascii_uppercase() == expected && character.is_ascii(), + } +} + +/// Code points of `text`. +pub(crate) fn chars(text: &str) -> Vec { + text.chars().collect() +} + +/// A string from code points. +pub(crate) fn text(chars: &[char]) -> String { + chars.iter().collect() +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs new file mode 100644 index 0000000000..d6e61a5d28 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs @@ -0,0 +1,119 @@ +//! Python's `parse_constant_declaration_file` up to value normalisation, which Python performs. + +use crate::declaration_files::_helpers::checks::python_values::{ + failure, get, python_str, unknown_keys, +}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; +use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; +use crate::declaration_files::models::{ConstantDeclaration, ConstantFile}; +use crate::models::FailureKind; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const CONSTANT_KEYS: [&str; 4] = ["name", "value", "type", "render_as"]; +const WRAPPER_KEYS: [&str; 3] = ["value", "type", "render_as"]; +const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; + +pub(crate) fn parse_constant_file( + file_path: &str, + contents: String, +) -> Result { + let headers = declaration_headers(&contents, file_path, "CONSTANT")?; + let mut declarations: Vec = Vec::with_capacity(headers.len()); + let mut failure: Option<_> = None; + for header in headers { + match parse_constant(&header.values, file_path) { + Ok(declaration) => declarations.push(declaration), + Err(ParseStop::Failed(stopped)) => { + failure = Some(stopped); + break; + } + Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), + } + } + Ok(ConstantFile { + contents, + declarations, + failure, + }) +} + +fn declaration(message: String) -> ParseStop { + failure(FailureKind::Declaration, message) +} + +fn parse_constant( + values: &[(String, AuthoredValue)], + file_path: &str, +) -> Result { + let unknown: Vec = unknown_keys(values, &CONSTANT_KEYS); + if !unknown.is_empty() { + return Err(declaration(format!( + "{file_path} constant has unknown keys: {}", + unknown.join(", ") + ))); + } + let name: String = declaration_name(get(values, "name"), file_path, "constant")?; + let mut value: Option<&AuthoredValue> = get(values, "value"); + let mut explicit_type: Option<&AuthoredValue> = get(values, "type"); + let mut render_as: Option<&AuthoredValue> = get(values, "render_as"); + if let Some(AuthoredValue::TypedConstant(arguments)) = value { + if explicit_type.is_some() || render_as.is_some() { + return Err(declaration(format!( + "{file_path} constant '{name}' cannot combine wrapper and outer options" + ))); + } + let unknown: Vec = unknown_keys(arguments, &WRAPPER_KEYS); + if !unknown.is_empty() { + return Err(declaration(format!( + "{file_path} constant '{name}' wrapper has unknown keys: {}", + unknown.join(", ") + ))); + } + value = get(arguments, "value"); + explicit_type = get(arguments, "type"); + render_as = get(arguments, "render_as"); + } + let Some(value) = value else { + return Err(declaration(format!( + "{file_path} constant '{name}' is missing required value" + ))); + }; + let explicit_type: Option = + constant_option(explicit_type, file_path, &format!("constant '{name}' type"))?; + let render_as: Option = constant_option( + render_as, + file_path, + &format!("constant '{name}' render_as"), + )?; + if let Some(rendering) = &render_as + && !COLLECTION_RENDERINGS.contains(&rendering.as_str()) + { + return Err(declaration(format!( + "{file_path} constant '{name}' render_as must be value_list or array" + ))); + } + Ok(ConstantDeclaration { + name, + value: value.clone(), + explicit_type, + render_as, + }) +} + +/// `_parse_constant_option`: absent, or a non-empty string. +fn constant_option( + raw_value: Option<&AuthoredValue>, + file_path: &str, + label: &str, +) -> Result, ParseStop> { + let Some(raw_value) = raw_value else { + return Ok(None); + }; + match python_str(raw_value)? { + Some(text) if !text.is_empty() => Ok(Some(text.to_owned())), + _ => Err(declaration(format!( + "{file_path} {label} must be an identifier" + ))), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/declaration_headers.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/declaration_headers.rs new file mode 100644 index 0000000000..902475055d --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/declaration_headers.rs @@ -0,0 +1,154 @@ +//! Python's `_parse_declaration_headers`: split a file into `KIND(...);` headers and parse each. + +use crate::declaration_files::_helpers::checks::python_values::failure; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::models::FailureKind; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "SCHEMA"]; + +/// One parsed declaration header with its byte range in the file. +pub(crate) struct DeclarationHeader { + pub(crate) values: Vec<(String, AuthoredValue)>, + pub(crate) header_start: usize, + pub(crate) header_end: usize, + /// The tokenizer's column-key offsets, for schema column locations. + pub(crate) column_offsets: Vec<(String, usize, usize)>, +} + +/// Every `expected_kind(...)` header in `contents`, or the first failure Python raises. +pub(crate) fn declaration_headers( + contents: &str, + file_path: &str, + expected_kind: &str, +) -> Result, ParseStop> { + let declaration = |message: String| failure(FailureKind::Declaration, message); + let mut headers: Vec = Vec::new(); + let mut cursor: usize = 0; + while cursor < contents.len() { + cursor = skip_whitespace(contents, cursor); + if cursor == contents.len() { + break; + } + let Some((actual_kind, open_index)) = declaration_start(contents, cursor) else { + return Err(declaration(format!( + "{file_path} must contain only {expected_kind}(...) declarations" + ))); + }; + if actual_kind != expected_kind { + return Err(declaration(format!( + "{file_path} contains {actual_kind}(...) under the {}s root", + expected_kind.to_ascii_lowercase() + ))); + } + let Some(close_index) = closing_parenthesis(contents, open_index) else { + return Err(declaration(format!( + "{file_path} has an unterminated declaration header" + ))); + }; + cursor = skip_whitespace(contents, close_index + 1); + if contents.as_bytes().get(cursor) != Some(&b';') { + return Err(declaration(format!( + "{expected_kind}(...) in '{file_path}' must end with ';'" + ))); + } + let header_start: usize = open_index + 1; + let (values, column_offsets) = match parse_one(&contents[header_start..close_index]) { + (_, _, Some(error)) => { + return Err(declaration(syntax_message( + expected_kind, + file_path, + &error, + ))); + } + (Some(AuthoredValue::Map(values)), Some(offsets), None) => (values, offsets), + _ => { + return Err(declaration(syntax_message( + expected_kind, + file_path, + "Native MODEL header parser returned neither values nor an error", + ))); + } + }; + headers.push(DeclarationHeader { + values, + header_start, + header_end: close_index, + column_offsets, + }); + cursor += 1; + } + if headers.is_empty() { + return Err(declaration(format!( + "{file_path} contains no {expected_kind}(...) declarations" + ))); + } + Ok(headers) +} + +fn syntax_message(kind: &str, file_path: &str, error: &str) -> String { + format!("{kind}(...) in '{file_path}' contains invalid SQLBuild header syntax: {error}") +} + +/// `(ENUM|CONSTANT|SCHEMA)\s*\(` at `start`: the kind and the byte offset of the parenthesis. +fn declaration_start(contents: &str, start: usize) -> Option<(&'static str, usize)> { + let rest: &str = &contents[start..]; + let kind: &'static str = DECLARATION_KINDS + .into_iter() + .find(|kind| rest.starts_with(kind))?; + let open_index: usize = skip_whitespace(contents, start + kind.len()); + (contents.as_bytes().get(open_index) == Some(&b'(')).then_some((kind, open_index)) +} + +/// `_find_closing_parenthesis`: quotes skip backslash escapes; `None` when the header never closes. +fn closing_parenthesis(contents: &str, open_index: usize) -> Option { + let bytes: &[u8] = contents.as_bytes(); + let mut depth: usize = 1; + let mut quote: Option = None; + let mut index: usize = open_index + 1; + while index < bytes.len() { + let byte: u8 = bytes[index]; + if let Some(open_quote) = quote { + if byte == b'\\' { + index += 1 + next_char_len(contents, index + 1); + continue; + } + if byte == open_quote { + quote = None; + } + index += 1; + continue; + } + match byte { + b'\'' | b'"' => quote = Some(byte), + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(index); + } + } + _ => {} + } + index += 1; + } + None +} + +/// The UTF-8 length of the character at `index`, or 0 at the end. +fn next_char_len(contents: &str, index: usize) -> usize { + contents + .get(index..) + .and_then(|rest| rest.chars().next()) + .map_or(0, char::len_utf8) +} + +/// The byte offset of the first non-whitespace character at or after `start`. +fn skip_whitespace(contents: &str, start: usize) -> usize { + contents[start..] + .char_indices() + .find(|(_, character)| !is_python_space(*character)) + .map_or(contents.len(), |(offset, _)| start + offset) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs new file mode 100644 index 0000000000..f19e8e8e2a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs @@ -0,0 +1,148 @@ +//! Python's `parse_enum_declaration_file` for public enum files. + +use crate::declaration_files::_helpers::checks::identities::validate_public_identity; +use crate::declaration_files::_helpers::checks::python_values::{ + PythonType, failure, get, is_identifier, python_str, python_type, unknown_keys, +}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; +use crate::declaration_files::models::{EnumDeclaration, EnumFile}; +use crate::models::FailureKind; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const ENUM_KEYS: [&str; 2] = ["name", "members"]; +const VARCHAR_TYPE: &str = "VARCHAR"; +const INTEGER_TYPE: &str = "INTEGER"; + +pub(crate) fn parse_enum_file(file_path: &str, contents: String) -> Result { + let declarations: Vec = declaration_headers(&contents, file_path, "ENUM")? + .into_iter() + .map(|header| parse_enum(&header.values, file_path)) + .collect::>()?; + Ok(EnumFile { + contents, + declarations, + }) +} + +fn declaration(message: String) -> ParseStop { + failure(FailureKind::Declaration, message) +} + +fn parse_enum( + values: &[(String, AuthoredValue)], + file_path: &str, +) -> Result { + let unknown: Vec = unknown_keys(values, &ENUM_KEYS); + if !unknown.is_empty() { + return Err(declaration(format!( + "{file_path} enum has unknown keys: {}", + unknown.join(", ") + ))); + } + let name: String = declaration_name(get(values, "name"), file_path, "enum")?; + let members: Vec<(String, AuthoredValue)> = match get(values, "members") { + Some(AuthoredValue::List(items)) => shorthand_members(items, file_path, &name)?, + Some(AuthoredValue::Map(items)) => explicit_members(items, file_path, &name)?, + _ => { + return Err(declaration(format!( + "{file_path} enum '{name}' members must use [...] or (...)" + ))); + } + }; + if members.is_empty() { + return Err(declaration(format!( + "{file_path} enum '{name}' must declare at least one member" + ))); + } + if let Some((member, _)) = members + .iter() + .find(|(member, _)| *member != member.to_ascii_uppercase()) + { + return Err(declaration(format!( + "{file_path} enum '{name}' member identifiers must be uppercase: '{member}'" + ))); + } + let mut types: Vec = Vec::with_capacity(members.len()); + for (_, value) in &members { + types.push(python_type(value)?); + } + if types.iter().any(|value_type| *value_type != types[0]) { + return Err(declaration(format!( + "{file_path} enum '{name}' members must use one consistent scalar type" + ))); + } + Ok(EnumDeclaration { + name, + members, + scalar_type: if types[0] == PythonType::Str { + VARCHAR_TYPE + } else { + INTEGER_TYPE + }, + }) +} + +/// `_parse_declaration_name`: a SQL identifier in canonical public snake_case. +pub(crate) fn declaration_name( + raw_name: Option<&AuthoredValue>, + file_path: &str, + kind: &str, +) -> Result { + let name: Option<&str> = match raw_name { + Some(value) => python_str(value)?, + None => None, + }; + let Some(name) = name.filter(|name| is_identifier(name)) else { + return Err(declaration(format!( + "{file_path} {kind} name must be a SQL identifier" + ))); + }; + validate_public_identity(name, kind, file_path)?; + Ok(name.to_owned()) +} + +fn shorthand_members( + items: &[AuthoredValue], + file_path: &str, + enum_name: &str, +) -> Result, ParseStop> { + let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); + for item in items { + let Some(member) = python_str(item)?.filter(|member| is_identifier(member)) else { + return Err(declaration(format!( + "{file_path} enum '{enum_name}' shorthand members must be identifiers" + ))); + }; + if members.iter().any(|(seen, _)| seen == member) { + return Err(declaration(format!( + "{file_path} enum '{enum_name}' has duplicate member '{member}'" + ))); + } + members.push((member.to_owned(), AuthoredValue::String(member.to_owned()))); + } + Ok(members) +} + +fn explicit_members( + items: &[(String, AuthoredValue)], + file_path: &str, + enum_name: &str, +) -> Result, ParseStop> { + let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); + for (member, value) in items { + if !is_identifier(member) { + return Err(declaration(format!( + "{file_path} enum '{enum_name}' member name must be a SQL identifier" + ))); + } + if !matches!(python_type(value)?, PythonType::Str | PythonType::Int) { + return Err(declaration(format!( + "{file_path} enum '{enum_name}' member '{member}' value must be a string or \ + integer" + ))); + } + members.push((member.clone(), value.clone())); + } + Ok(members) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/function_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/function_files.rs new file mode 100644 index 0000000000..d52dcba930 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/function_files.rs @@ -0,0 +1,52 @@ +//! Python's `parse_function_sql`: a `FUNCTION(...);` header and the SQL body after it. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::models::{DeclarationFileOptions, FunctionFile}; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +const FUNCTION_KEYWORD: &str = "FUNCTION"; + +pub(crate) fn parse_function_file( + file_path: &str, + contents: String, + options: &DeclarationFileOptions, +) -> Result { + let Some((header_start, header_end, sql_start)) = + match_statement_header(&contents, 0, FUNCTION_KEYWORD) + else { + return Err(ParseStop::Failed(DiscoveryFailure::new( + FailureKind::ModelSql, + format!( + "SQL function '{file_path}' must start with a FUNCTION(...) header as the first \ + non-whitespace content" + ), + ))); + }; + let header_values = parse_statement_header( + &StatementHeader { + kind: FailureKind::ModelSql, + statement_name: FUNCTION_KEYWORD, + statement: "FUNCTION()", + file_path, + supported_keys: &options.function_keys, + python: options.python, + }, + &contents[header_start..header_end], + contents[..header_start].matches('\n').count() + 1, + )?; + let body_sql: String = python_strip(&contents[sql_start..]).to_owned(); + if body_sql.is_empty() { + return Err(ParseStop::Failed(DiscoveryFailure::new( + FailureKind::ModelSql, + format!("SQL function '{file_path}' must contain SQL after FUNCTION(...)"), + ))); + } + Ok(FunctionFile { + contents, + header_values, + body_sql, + }) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs new file mode 100644 index 0000000000..05b1c6dbc0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs @@ -0,0 +1,144 @@ +//! Python's `parse_sql_hook_file`: exactly one `HOOK(...);` header and the SQL after it. + +use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; +use crate::declaration_files::_helpers::checks::python_values::{ + failure, get, non_empty_str, python_str, +}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::models::{DeclarationFileOptions, HookFile}; +use crate::models::FailureKind; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const HOOK_KEYWORD: &str = "HOOK"; +const DESCRIPTION_KEY: &str = "description"; +const SQL_QUOTES: &[u8] = b"'\"`"; + +fn hook_failure(message: String) -> ParseStop { + failure(FailureKind::SqlHook, message) +} + +pub(crate) fn parse_hook_file( + file_path: &str, + name: &str, + contents: String, + options: &DeclarationFileOptions, +) -> Result { + let (header_start, header_end, body_start) = split_hook_file(&contents, file_path)?; + let header: &str = &contents[header_start..header_end]; + let header_values: Vec<(String, AuthoredValue)> = parse_statement_header( + &StatementHeader { + kind: FailureKind::SqlHook, + statement_name: HOOK_KEYWORD, + statement: "HOOK()", + file_path, + supported_keys: &options.hook_keys, + python: options.python, + }, + header, + contents[..header_start].matches('\n').count() + 1, + )?; + let description: Option = match get(&header_values, DESCRIPTION_KEY) { + Some(value) => { + if non_empty_str(value)?.is_none() { + return Err(hook_failure(format!( + "HOOK() description in '{file_path}' must be a non-empty string" + ))); + } + python_str(value)?.map(str::to_owned) + } + None => None, + }; + let sql_body: String = python_cleandoc(options.python, &contents[body_start..]); + if sql_body.is_empty() { + return Err(hook_failure(format!( + "SQL hook '{file_path}' must define SQL after HOOK(...)" + ))); + } + Ok(HookFile { + header_values, + sql_body, + name: name.to_owned(), + description, + contents, + }) +} + +/// `_split_hook_file`: byte offsets of the header start and end, and of the SQL after `;`. +fn split_hook_file(contents: &str, file_path: &str) -> Result<(usize, usize, usize), ParseStop> { + let bytes: &[u8] = contents.as_bytes(); + let mut index: usize = skip_whitespace(contents, 0); + if !contents[index..].starts_with(HOOK_KEYWORD) { + return Err(hook_failure(format!( + "SQL hook '{file_path}' must start with a HOOK() header as the first non-whitespace \ + content" + ))); + } + index = skip_whitespace(contents, index + HOOK_KEYWORD.len()); + if bytes.get(index) != Some(&b'(') { + return Err(hook_failure(format!( + "SQL hook '{file_path}' must define a HOOK(...) header" + ))); + } + let header_start: usize = index + 1; + let mut depth: usize = 1; + let mut quote: Option = None; + index = header_start; + while index < bytes.len() { + let byte: u8 = bytes[index]; + if let Some(open_quote) = quote { + if byte == b'\\' { + index += 1 + next_char_len(contents, index + 1); + continue; + } + if byte == open_quote { + if bytes.get(index + 1) == Some(&open_quote) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + if SQL_QUOTES.contains(&byte) { + quote = Some(byte); + } else if byte == b'(' { + depth += 1; + } else if byte == b')' { + depth -= 1; + if depth == 0 { + break; + } + } + index += 1; + } + if depth != 0 { + return Err(hook_failure(format!( + "SQL hook '{file_path}' has an unterminated HOOK(...) header" + ))); + } + let header_end: usize = index; + index = skip_whitespace(contents, index + 1); + if bytes.get(index) != Some(&b';') { + return Err(hook_failure(format!( + "SQL hook '{file_path}' HOOK(...) header must end with ';'" + ))); + } + Ok((header_start, header_end, index + 1)) +} + +fn next_char_len(contents: &str, index: usize) -> usize { + contents + .get(index..) + .and_then(|rest| rest.chars().next()) + .map_or(0, char::len_utf8) +} + +fn skip_whitespace(contents: &str, start: usize) -> usize { + contents[start..] + .char_indices() + .find(|(_, character)| !is_python_space(*character)) + .map_or(contents.len(), |(offset, _)| start + offset) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/mod.rs new file mode 100644 index 0000000000..8d78968b71 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod audit_files; +pub(crate) mod audit_text; +pub(crate) mod constant_files; +pub(crate) mod declaration_headers; +pub(crate) mod enum_files; +pub(crate) mod function_files; +pub(crate) mod hook_files; +pub(crate) mod schema_files; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs new file mode 100644 index 0000000000..8e1054486f --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs @@ -0,0 +1,106 @@ +//! Python's `parse_model_schema_declaration_file` up to the columns, which Python parses. + +use crate::_helpers::locations::header_column_locations; +use crate::declaration_files::_helpers::checks::python_values::{ + PythonType, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, +}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; +use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; +use crate::declaration_files::models::{SchemaDeclaration, SchemaFile}; +use crate::models::FailureKind; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const SCHEMA_KEYS: [&str; 4] = ["name", "description", "extends", "columns"]; + +pub(crate) fn parse_schema_file( + file_path: &str, + contents: String, +) -> Result { + let headers = declaration_headers(&contents, file_path, "SCHEMA")?; + let mut declarations: Vec = Vec::with_capacity(headers.len()); + let mut failure: Option<_> = None; + for header in headers { + match parse_schema(&header.values, file_path) { + Ok((name, description, extends)) => declarations.push(SchemaDeclaration { + name, + description, + extends, + columns: get(&header.values, "columns").cloned(), + column_locations: header_column_locations( + &contents, + (header.header_start, header.header_end), + &header.column_offsets, + ), + }), + Err(ParseStop::Failed(stopped)) => { + failure = Some(stopped); + break; + } + Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), + } + } + Ok(SchemaFile { + contents, + declarations, + failure, + }) +} + +fn declaration(message: String) -> ParseStop { + failure(FailureKind::Declaration, message) +} + +type SchemaFacts = (String, Option, Option); + +fn parse_schema( + values: &[(String, AuthoredValue)], + file_path: &str, +) -> Result { + let unknown: Vec = unknown_keys(values, &SCHEMA_KEYS); + if !unknown.is_empty() { + return Err(declaration(format!( + "{file_path} schema has unknown keys: {}", + unknown.join(", ") + ))); + } + let name: String = declaration_name(get(values, "name"), file_path, "schema")?; + let description: Option = optional_string( + get(values, "description"), + file_path, + &format!("schema '{name}' description"), + )?; + let extends: Option = optional_string( + get(values, "extends"), + file_path, + &format!("schema '{name}' extends"), + )?; + if let Some(parent) = &extends + && !is_identifier(parent) + { + return Err(declaration(format!( + "{file_path} schema '{name}' extends must be an identifier" + ))); + } + Ok((name, description, extends)) +} + +/// `_parse_optional_declaration_string`: absent or `null`, or a non-empty string. +fn optional_string( + raw_value: Option<&AuthoredValue>, + file_path: &str, + label: &str, +) -> Result, ParseStop> { + let Some(raw_value) = raw_value else { + return Ok(None); + }; + if python_type(raw_value)? == PythonType::None { + return Ok(None); + } + match non_empty_str(raw_value)? { + Some(text) => Ok(Some(text.to_owned())), + None => Err(declaration(format!( + "{file_path} {label} must be a non-empty string" + ))), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/discover_declaration_collection.rs b/crates/sqlbuild-discovery/src/declaration_files/main/discover_declaration_collection.rs new file mode 100644 index 0000000000..42bb2c258a --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/discover_declaration_collection.rs @@ -0,0 +1,23 @@ +//! Discover, read and parse one declaration collection in Python's discovery order. + +use crate::_helpers::pool::discovery_pool; +use crate::declaration_files::_helpers::listing::collection_reads::{ + CollectionSource, failed_collection, read_collection, +}; +use crate::declaration_files::models::{ + CollectionRequest, DeclarationCollection, DiscoverySession, +}; +use crate::models::StageFailure; +use crate::tree::models::ProjectTree; + +/// The collection `request` names, every file read and parsed on the discovery pool. +pub fn discover_declaration_collection( + session: &DiscoverySession, + tree: &ProjectTree, + request: CollectionRequest, +) -> DeclarationCollection { + match discovery_pool() { + Ok(pool) => pool.install(|| read_collection(&CollectionSource { session, tree }, request)), + Err(reason) => failed_collection(request.kind, StageFailure::Internal(reason)), + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs new file mode 100644 index 0000000000..f16e07d5e1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs @@ -0,0 +1,3 @@ +pub mod discover_declaration_collection; +pub mod parse_declaration_text; +pub mod retained_collection; diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs new file mode 100644 index 0000000000..b467ac5d1f --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs @@ -0,0 +1,46 @@ +//! Parse in-memory declaration file contents as discovery parses a file of one kind. + +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::declaration_files::_helpers::parsing::audit_files::parse_audit_file; +use crate::declaration_files::_helpers::parsing::constant_files::parse_constant_file; +use crate::declaration_files::_helpers::parsing::enum_files::parse_enum_file; +use crate::declaration_files::_helpers::parsing::function_files::parse_function_file; +use crate::declaration_files::_helpers::parsing::hook_files::parse_hook_file; +use crate::declaration_files::_helpers::parsing::schema_files::parse_schema_file; +use crate::declaration_files::models::{ + CollectionKind, DeclarationFileOptions, ParsedDeclarationText, +}; +use crate::models::FileOutcome; + +/// `contents` parsed as a `kind` file named `file_path`; `None` for seeds and macros, which have none. +pub fn parse_declaration_text( + kind: CollectionKind, + (file_path, hook_name): (&str, &str), + contents: String, + options: &DeclarationFileOptions, +) -> Option> { + let parsed: Result = match kind { + CollectionKind::Enums => { + parse_enum_file(file_path, contents).map(ParsedDeclarationText::Enum) + } + CollectionKind::Constants => { + parse_constant_file(file_path, contents).map(ParsedDeclarationText::Constant) + } + CollectionKind::ModelSchemas => { + parse_schema_file(file_path, contents).map(ParsedDeclarationText::ModelSchema) + } + CollectionKind::SqlFunctions => parse_function_file(file_path, contents, options) + .map(ParsedDeclarationText::SqlFunction), + CollectionKind::SqlHooks => parse_hook_file(file_path, hook_name, contents, options) + .map(ParsedDeclarationText::SqlHook), + CollectionKind::Audits => { + parse_audit_file(file_path, contents, options).map(ParsedDeclarationText::Audit) + } + CollectionKind::Seeds | CollectionKind::Macros => return None, + }; + Some(match parsed { + Ok(parsed) => FileOutcome::Parsed(parsed), + Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), + Err(ParseStop::Deferred) => FileOutcome::Deferred, + }) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/retained_collection.rs b/crates/sqlbuild-discovery/src/declaration_files/main/retained_collection.rs new file mode 100644 index 0000000000..dd95701836 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/retained_collection.rs @@ -0,0 +1,34 @@ +//! Read one declaration collection once per discovery session and keep it for later stages. + +use crate::declaration_files::models::{ + CollectionRequest, DeclarationCollection, DiscoverySession, RetainedCollection, +}; +use std::sync::{Arc, MutexGuard, PoisonError}; + +/// The collection `request` names, read with `discover` the first time the session asks for it. +pub fn retained_collection( + session: &DiscoverySession, + request: CollectionRequest, + discover: impl FnOnce() -> DeclarationCollection, +) -> Arc { + if let Some(retained) = collections(session) + .iter() + .find(|retained| retained.request == request) + { + return Arc::clone(&retained.collection); + } + let collection: Arc = Arc::new(discover()); + collections(session).push(RetainedCollection { + request, + collection: Arc::clone(&collection), + }); + collection +} + +/// The session's collections; a panic while another thread held them leaves them readable. +fn collections(session: &DiscoverySession) -> MutexGuard<'_, Vec> { + session + .collections + .lock() + .unwrap_or_else(PoisonError::into_inner) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/mod.rs new file mode 100644 index 0000000000..91fb358a0f --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/declaration_files/models.rs b/crates/sqlbuild-discovery/src/declaration_files/models.rs new file mode 100644 index 0000000000..b382daf982 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/models.rs @@ -0,0 +1,255 @@ +//! Native results for enum, constant, schema, audit, hook, SQL function, seed and macro files. + +use crate::declaration_files::types::Collection; +use crate::declarations::models::{DeclarationKind, DeclarationLayout}; +use crate::models::{DiscoveredFile, DiscoveryFailure, LineColumnSpan, ProjectRoot, StageFailure}; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::sync::{Arc, Mutex}; + +/// The header key sets Python owns and the Python semantics native parsing reproduces. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationFileOptions { + pub function_keys: Vec, + pub audit_keys: Vec, + pub hook_keys: Vec, + pub python: PythonText, +} + +/// One collection Python's discovery reads, in its discovery order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum CollectionKind { + Enums, + Constants, + ModelSchemas, + SqlFunctions, + SqlHooks, + Audits, + Seeds, + Macros, +} + +impl CollectionKind { + /// The stable name the facade requests a collection by. + pub fn as_str(self) -> &'static str { + match self { + Self::Enums => "enum", + Self::Constants => "constant", + Self::ModelSchemas => "model_schema", + Self::SqlFunctions => "sql_function", + Self::SqlHooks => "sql_hook", + Self::Audits => "audit", + Self::Seeds => "seed", + Self::Macros => "macro", + } + } + + /// The kind named `name`, if any. + pub fn from_name(name: &str) -> Option { + [ + Self::Enums, + Self::Constants, + Self::ModelSchemas, + Self::SqlFunctions, + Self::SqlHooks, + Self::Audits, + Self::Seeds, + Self::Macros, + ] + .into_iter() + .find(|kind| kind.as_str() == name) + } +} + +/// The scope facts Python attaches to a declaration file, with `/`-separated relative paths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileScope { + /// Python's `DeclarationKind` value of the file's role. + pub declaration_kind: &'static str, + /// Python's `ScopeKind` value. + pub scope_kind: &'static str, + pub ownership_root: Option, + pub owning_path: Option, + pub declaration_root: Option, +} + +/// One discovered file with the scope facts of the role that holds it, if any. +#[derive(Clone, Debug, PartialEq)] +pub struct ScopedFile { + pub file: DiscoveredFile, + pub scope: Option, +} + +/// Why a whole collection fails: an unreadable directory or an invalid declaration layout. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CollectionFailure { + Stage(StageFailure), + Layout(DiscoveryFailure), +} + +/// One public enum; member values are the authored values Python projects. +#[derive(Clone, Debug, PartialEq)] +pub struct EnumDeclaration { + pub name: String, + pub members: Vec<(String, AuthoredValue)>, + /// `VARCHAR` or `INTEGER`. + pub scalar_type: &'static str, +} + +/// An enum file whose declarations all parsed. +#[derive(Clone, Debug, PartialEq)] +pub struct EnumFile { + pub contents: String, + pub declarations: Vec, +} + +/// One public constant checked up to value normalisation, which Python performs. +#[derive(Clone, Debug, PartialEq)] +pub struct ConstantDeclaration { + pub name: String, + pub value: AuthoredValue, + pub explicit_type: Option, + pub render_as: Option, +} + +/// A constant file: the declarations before the first native failure, then that failure. +#[derive(Clone, Debug, PartialEq)] +pub struct ConstantFile { + pub contents: String, + pub declarations: Vec, + pub failure: Option, +} + +/// One reusable model schema checked up to its columns, which Python parses. +#[derive(Clone, Debug, PartialEq)] +pub struct SchemaDeclaration { + pub name: String, + pub description: Option, + pub extends: Option, + pub columns: Option, + pub column_locations: Vec<(String, LineColumnSpan)>, +} + +/// A schema file: the declarations before the first native failure, then that failure. +#[derive(Clone, Debug, PartialEq)] +pub struct SchemaFile { + pub contents: String, + pub declarations: Vec, + pub failure: Option, +} + +/// One `AUDIT(...)` block; values mirror Python's `DiscoveredAuditBlock`. +#[derive(Clone, Debug, PartialEq)] +pub struct AuditBlock { + pub audit_index: usize, + pub header_values: Vec<(String, AuthoredValue)>, + pub sql_body: String, + pub name: Option, + /// `violations` or `measurement`. + pub evaluation_mode: &'static str, + pub measure_sql: Option, + pub evidence_sql: Option, +} + +/// An audit file whose blocks all parsed. +#[derive(Clone, Debug, PartialEq)] +pub struct AuditFile { + pub contents: String, + pub blocks: Vec, +} + +/// A named SQL hook file; values mirror Python's `DiscoveredSqlHookFile`. +#[derive(Clone, Debug, PartialEq)] +pub struct HookFile { + pub contents: String, + pub header_values: Vec<(String, AuthoredValue)>, + pub sql_body: String, + pub name: String, + pub description: Option, +} + +/// A SQL function file; values mirror Python's `DiscoveredSqlFunctionFile`. +#[derive(Clone, Debug, PartialEq)] +pub struct FunctionFile { + pub contents: String, + pub header_values: Vec<(String, AuthoredValue)>, + pub body_sql: String, +} + +/// A macro file's text; Python analyses and imports it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroFile { + pub contents: String, +} + +/// A seed file; Python reads seeds when it loads them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SeedFile; + +/// One discovered collection, retained for later native stages. +#[derive(Clone, Debug, PartialEq)] +pub enum DeclarationCollection { + Enums(Collection), + Constants(Collection), + ModelSchemas(Collection), + SqlFunctions(Collection), + SqlHooks(Collection), + Audits(Collection), + Seeds(Collection), + Macros(Collection), +} + +/// One in-memory parse of `contents` as a `kind` file named `file_path`. +#[derive(Clone, Debug, PartialEq)] +pub enum ParsedDeclarationText { + Enum(EnumFile), + Constant(ConstantFile), + ModelSchema(SchemaFile), + SqlFunction(FunctionFile), + SqlHook(HookFile), + Audit(AuditFile), +} + +/// One collection a discovery pass asks for; macro, enum and constant layouts may be isolated. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct CollectionRequest { + pub kind: CollectionKind, + /// Read the layout of this declaration kind alone, as tolerant discovery does. + pub isolate_kind: bool, +} + +/// One collection a discovery pass read, kept for later native stages. +#[derive(Clone, Debug)] +pub struct RetainedCollection { + pub request: CollectionRequest, + pub collection: Arc, +} + +/// The declaration layout one discovery pass read for a kind, or for every kind with `None`. +#[derive(Clone, Debug)] +pub struct RetainedLayout { + pub kind: Option, + pub layout: Arc>, +} + +/// The declaration collections one discovery pass read, retained for later native stages. +#[derive(Debug)] +pub struct DiscoverySession { + pub root: ProjectRoot, + pub options: DeclarationFileOptions, + pub collections: Mutex>, + /// The layouts the collections were listed from, walked once per pass. + pub layouts: Mutex>, +} + +impl DiscoverySession { + /// A session of `root` that has read nothing yet. + pub fn new(root: ProjectRoot, options: DeclarationFileOptions) -> Self { + Self { + root, + options, + collections: Mutex::new(Vec::new()), + layouts: Mutex::new(Vec::new()), + } + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests.rs b/crates/sqlbuild-discovery/src/declaration_files/tests.rs new file mode 100644 index 0000000000..0dee9ad7f3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_discovery_session.rs"] +mod discovery_session; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_parse_declaration_text.rs"] +mod parse_declaration_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/helpers.rs new file mode 100644 index 0000000000..3521bc8517 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/helpers.rs @@ -0,0 +1,84 @@ +use crate::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use crate::declaration_files::main::parse_declaration_text::parse_declaration_text; +use crate::declaration_files::main::retained_collection::retained_collection; +use crate::declaration_files::models::{ + CollectionKind, CollectionRequest, DeclarationCollection, DeclarationFileOptions, + DiscoverySession, +}; +use crate::models::ProjectRoot; +use crate::tree::models::ProjectTree; +use sqlbuild_core::text::main::python_text::python_text; +use std::fs; +use std::sync::Arc; + +pub(super) const FILE_PATH: &str = "/project/declarations/orders.sql"; +const FUNCTION_KEYS: [&str; 3] = ["arguments", "description", "returns"]; +const AUDIT_KEYS: [&str; 7] = [ + "always_run", + "evaluation", + "name", + "sample_unit", + "severity", + "sql_analysis", + "value", +]; + +fn owned(keys: &[&str]) -> Vec { + keys.iter().map(|key| (*key).to_owned()).collect() +} + +fn options() -> DeclarationFileOptions { + DeclarationFileOptions { + function_keys: owned(&FUNCTION_KEYS), + audit_keys: owned(&AUDIT_KEYS), + hook_keys: owned(&["description"]), + python: python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"), + } +} + +/// The debug text of `contents` parsed as a `kind` file named `FILE_PATH` and hook `orders`. +pub(super) fn parsed_debug(kind: CollectionKind, contents: &str) -> String { + format!( + "{:?}", + parse_declaration_text(kind, (FILE_PATH, "orders"), contents.to_owned(), &options()) + ) +} + +/// Each collection's file count, the layouts walked and whether re-reads reuse the first read. +pub(super) fn session_reads( + files: &[(&str, &str)], + requests: &[CollectionRequest], +) -> (Vec, usize, bool) { + let project = tempfile::tempdir().expect("temporary project"); + for (relative_path, contents) in files { + let path = project.path().join(relative_path); + fs::create_dir_all(path.parent().expect("parent directory")).expect("directories"); + fs::write(path, contents).expect("file"); + } + let session = DiscoverySession::new( + ProjectRoot { + directory: project.path().to_path_buf(), + display_prefix: project.path().display().to_string(), + }, + options(), + ); + let tree = ProjectTree::new(project.path()); + let read = |request: CollectionRequest| { + retained_collection(&session, request, || { + discover_declaration_collection(&session, &tree, request) + }) + }; + let counts: Vec = requests + .iter() + .map(|request| collection_len(&read(*request))) + .collect(); + let reused: bool = requests + .iter() + .all(|request| Arc::ptr_eq(&read(*request), &read(*request))); + let layouts: usize = session.layouts.lock().expect("layouts").len(); + (counts, layouts, reused) +} + +fn collection_len(collection: &DeclarationCollection) -> usize { + format!("{collection:?}").matches("ScopedFile {").count() +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_discovery_session.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_discovery_session.rs new file mode 100644 index 0000000000..60588577b8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_discovery_session.rs @@ -0,0 +1,89 @@ +use crate::declaration_files::models::{CollectionKind, CollectionRequest}; +use crate::declaration_files::tests::helpers::session_reads; +use crate::declaration_files::tests::test_types::SessionReadTestCase; + +const FILES: &[(&str, &str)] = &[ + ( + "enums/status.sql", + "ENUM (name order_status, members [PLACED]);\n", + ), + ( + "constants/limits.sql", + "CONSTANT (name max_orders, value 10);\n", + ), + ("macros/cents.py", "def cents(value):\n return value\n"), + ("schemas/orders.sql", "SCHEMA (name orders_shape);\n"), + ( + "audits/generic/positive.sql", + "AUDIT (name positive);\nSELECT 1\n", + ), +]; + +const SHARED_REQUESTS: &[CollectionRequest] = &[ + CollectionRequest { + kind: CollectionKind::Enums, + isolate_kind: false, + }, + CollectionRequest { + kind: CollectionKind::Constants, + isolate_kind: false, + }, + CollectionRequest { + kind: CollectionKind::Macros, + isolate_kind: false, + }, + CollectionRequest { + kind: CollectionKind::ModelSchemas, + isolate_kind: false, + }, + CollectionRequest { + kind: CollectionKind::Audits, + isolate_kind: false, + }, +]; +const ISOLATED_REQUESTS: &[CollectionRequest] = &[ + CollectionRequest { + kind: CollectionKind::Enums, + isolate_kind: false, + }, + CollectionRequest { + kind: CollectionKind::Constants, + isolate_kind: true, + }, +]; + +#[test] +fn given_session_when_reading_collections_then_layout_is_walked_once_and_reads_are_retained() { + let test_cases = [ + SessionReadTestCase { + description: "every kind shares one layout", + files: FILES, + requests: SHARED_REQUESTS, + expected_counts: &[1, 1, 1, 1, 1], + expected_layouts: 1, + }, + SessionReadTestCase { + description: "an isolated kind walks its own layout", + files: FILES, + requests: ISOLATED_REQUESTS, + expected_counts: &[1, 1], + expected_layouts: 2, + }, + ]; + + for test_case in test_cases { + let (counts, layouts, reused) = session_reads(test_case.files, test_case.requests); + + assert_eq!( + counts, test_case.expected_counts, + "{}", + test_case.description + ); + assert_eq!( + layouts, test_case.expected_layouts, + "{}", + test_case.description + ); + assert!(reused, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs new file mode 100644 index 0000000000..0aab6e90ac --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs @@ -0,0 +1,215 @@ +use crate::declaration_files::models::CollectionKind; +use crate::declaration_files::tests::helpers::parsed_debug; +use crate::declaration_files::tests::test_types::DeclarationTextTestCase; + +#[test] +fn given_declaration_files_when_parsing_then_values_and_failures_match_python() { + let test_cases = [ + DeclarationTextTestCase { + description: "shorthand and explicit enums", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members [PLACED, SHIPPED]);\n\u{a0}ENUM(name rank, members (LOW 1, HIGH -2)) ;", + expected_fragments: &[ + "Parsed(Enum(", + "name: \"status\", members: [(\"PLACED\", String(\"PLACED\")), (\"SHIPPED\", String(\"SHIPPED\"))], scalar_type: \"VARCHAR\"", + "(\"HIGH\", BareWord(\"-2\"))], scalar_type: \"INTEGER\"", + ], + }, + DeclarationTextTestCase { + description: "lowercase enum members", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members [placed]);", + expected_fragments: &[ + "Failed(", + "Declaration", + "/project/declarations/orders.sql enum 'status' member identifiers must be uppercase: 'placed'", + ], + }, + DeclarationTextTestCase { + description: "mixed enum member types", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members (A 'x', B 1));", + expected_fragments: &[ + "/project/declarations/orders.sql enum 'status' members must use one consistent scalar type", + ], + }, + DeclarationTextTestCase { + description: "a float member value", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members (A 1.5));", + expected_fragments: &[ + "/project/declarations/orders.sql enum 'status' member 'A' value must be a string or integer", + ], + }, + DeclarationTextTestCase { + description: "a non-ASCII bare word may be a Unicode number", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members (A \u{661}));", + expected_fragments: &["Some(Deferred)"], + }, + DeclarationTextTestCase { + description: "a camel-case name suggests snake case", + kind: CollectionKind::Enums, + contents: "ENUM (name HTTPStatusCode, members [A]);", + expected_fragments: &[ + "ResourceIdentity", + "Invalid public enum identity 'HTTPStatusCode' in /project/declarations/orders.sql; use snake_case 'http_status_code'", + "Double underscores remain valid.", + ], + }, + DeclarationTextTestCase { + description: "another declaration kind under the enums root", + kind: CollectionKind::Enums, + contents: "ENUM (name a, members [A]);\nCONSTANT (name b, value 1);", + expected_fragments: &[ + "/project/declarations/orders.sql contains CONSTANT(...) under the enums root", + ], + }, + DeclarationTextTestCase { + description: "a quoted parenthesis does not close the header", + kind: CollectionKind::Enums, + contents: "ENUM (name a, members ['x)']", + expected_fragments: &[ + "/project/declarations/orders.sql has an unterminated declaration header", + ], + }, + DeclarationTextTestCase { + description: "constants keep their options and stop at the first failure", + kind: CollectionKind::Constants, + contents: "CONSTANT (name a, value [1], render_as array);\nCONSTANT (name b, value constant(value 1, type integer));\nCONSTANT (name c);", + expected_fragments: &[ + "name: \"a\", value: List([BareWord(\"1\")]), explicit_type: None, render_as: Some(\"array\")", + "name: \"b\", value: BareWord(\"1\"), explicit_type: Some(\"integer\"), render_as: None", + "/project/declarations/orders.sql constant 'c' is missing required value", + ], + }, + DeclarationTextTestCase { + description: "a wrapped constant with outer options", + kind: CollectionKind::Constants, + contents: "CONSTANT (name a, value constant(value 1), type integer);", + expected_fragments: &[ + "declarations: []", + "/project/declarations/orders.sql constant 'a' cannot combine wrapper and outer options", + ], + }, + DeclarationTextTestCase { + description: "an unknown render_as", + kind: CollectionKind::Constants, + contents: "CONSTANT (name a, value [1], render_as list);", + expected_fragments: &[ + "/project/declarations/orders.sql constant 'a' render_as must be value_list or array", + ], + }, + DeclarationTextTestCase { + description: "schema column locations over CRLF lines", + kind: CollectionKind::ModelSchemas, + contents: "SCHEMA (\r\n name shape,\r\n columns (\r\n id (type INT),\r\n ),\r\n);", + expected_fragments: &[ + "(\"id\", LineColumnSpan { line: 4, column: 5, end_line: 4, end_column: 7 })", + "failure: None", + ], + }, + DeclarationTextTestCase { + description: "a schema extending a non-identifier", + kind: CollectionKind::ModelSchemas, + contents: "SCHEMA (name shape, extends 'a b', columns (id (type INT)));", + expected_fragments: &[ + "/project/declarations/orders.sql schema 'shape' extends must be an identifier", + ], + }, + DeclarationTextTestCase { + description: "a SQL function", + kind: CollectionKind::SqlFunctions, + contents: " FUNCTION (returns INT);\n SELECT 1 \n", + expected_fragments: &["body_sql: \"SELECT 1\""], + }, + DeclarationTextTestCase { + description: "a SQL function with an unsupported key", + kind: CollectionKind::SqlFunctions, + contents: "FUNCTION (\n returns INT,\n retuns INT\n);\nSELECT 1", + expected_fragments: &[ + "ModelSql", + "FUNCTION() in '/project/declarations/orders.sql:3' has unsupported keys: retuns", + "did you mean 'returns'?", + ], + }, + DeclarationTextTestCase { + description: "a hook body is dedented", + kind: CollectionKind::SqlHooks, + contents: "HOOK (description 'Refresh');\n SELECT 1\n FROM t\n", + expected_fragments: &[ + "sql_body: \"SELECT 1\\nFROM t\", name: \"orders\", description: Some(\"Refresh\")", + ], + }, + DeclarationTextTestCase { + description: "a hook header without a terminator", + kind: CollectionKind::SqlHooks, + contents: "HOOK () SELECT 1", + expected_fragments: &[ + "SqlHook", + "SQL hook '/project/declarations/orders.sql' HOOK(...) header must end with ';'", + ], + }, + DeclarationTextTestCase { + description: "violation and measurement audit blocks", + kind: CollectionKind::Audits, + contents: "audit (name a);\nSELECT 1\nAUDIT (name b, evaluation measurement, value v);\nMEASURE (SELECT 1 AS v);\nEVIDENCE (SELECT 2);\n", + expected_fragments: &[ + "audit_index: 1", + "sql_body: \"SELECT 1\", name: Some(\"a\"), evaluation_mode: \"violations\"", + "evaluation_mode: \"measurement\", measure_sql: Some(\"SELECT 1 AS v\"), evidence_sql: Some(\"SELECT 2\")", + ], + }, + DeclarationTextTestCase { + description: "a dotless i spells AUDIT for Python's upper()", + kind: CollectionKind::Audits, + contents: "AUD\u{131}T ();\nSELECT 1", + expected_fragments: &["Parsed(Audit(", "sql_body: \"SELECT 1\""], + }, + DeclarationTextTestCase { + description: "unnamed blocks in a multi-block audit file", + kind: CollectionKind::Audits, + contents: "AUDIT ();\nSELECT 1\nAUDIT (name b);\nSELECT 2\n", + expected_fragments: &[ + "SqlAudit", + "contains multiple AUDIT blocks; every block must define a unique `name`. Missing names for blocks: 1", + ], + }, + DeclarationTextTestCase { + description: "a violation audit written as a measurement", + kind: CollectionKind::Audits, + contents: "AUDIT ();\n-- note\nMEASURE (SELECT 1);", + expected_fragments: &[ + "Violation audit '/project/declarations/orders.sql' must use a bare SELECT body, not MEASURE/EVIDENCE", + ], + }, + DeclarationTextTestCase { + description: "two statements in a MEASURE block", + kind: CollectionKind::Audits, + contents: "AUDIT (evaluation measurement, value v);\nMEASURE (SELECT 1; SELECT 2);", + expected_fragments: &[ + "MEASURE(...) in '/project/declarations/orders.sql' must contain exactly one query", + ], + }, + DeclarationTextTestCase { + description: "seeds have no authored text", + kind: CollectionKind::Seeds, + contents: "id\n1\n", + expected_fragments: &["None"], + }, + ]; + for test_case in test_cases { + let debug: String = parsed_debug(test_case.kind, test_case.contents); + let found: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| debug.contains(fragment)) + .collect(); + assert_eq!( + found, test_case.expected_fragments, + "{}: {debug}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_types.rs new file mode 100644 index 0000000000..f0cc45c79b --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_types.rs @@ -0,0 +1,20 @@ +use crate::declaration_files::models::{CollectionKind, CollectionRequest}; + +/// One in-memory declaration file and the outcome Python's parser produces for it. +pub(super) struct DeclarationTextTestCase { + pub(super) description: &'static str, + pub(super) kind: CollectionKind, + pub(super) contents: &'static str, + /// Fragments of the outcome's debug text: the variant, names, messages and bodies. + pub(super) expected_fragments: &'static [&'static str], +} + +/// A project read collection by collection through one discovery session. +pub(super) struct SessionReadTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) requests: &'static [CollectionRequest], + pub(super) expected_counts: &'static [usize], + /// The layouts walked: one shared by every kind, plus one per isolated kind. + pub(super) expected_layouts: usize, +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/types.rs b/crates/sqlbuild-discovery/src/declaration_files/types.rs new file mode 100644 index 0000000000..21dd01106f --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/types.rs @@ -0,0 +1,6 @@ +//! Type aliases of the declaration file collections. + +use crate::declaration_files::models::{CollectionFailure, ScopedFile}; + +/// Every file of one collection in Python's order, or the collection's failure. +pub type Collection = Result>, CollectionFailure>; diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 1858da47b2..bca84d474c 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -4,6 +4,7 @@ pub(crate) mod _helpers; pub mod constants; +pub mod declaration_files; pub mod declarations; pub mod model_files; pub mod models; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index c9e10edbd5..a39813b4a5 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,3 +1,2 @@ -pub(crate) mod locations; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs index a042bb5cb6..87056af636 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs @@ -1,6 +1,6 @@ //! Python's lexical scan of the top-level SELECT list for authored output column locations. -use crate::model_files::_helpers::locations::{absolute_span, line_starts}; +use crate::_helpers::locations::{absolute_span, line_starts}; use crate::models::LineColumnSpan; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::is_python_word::is_python_word; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index accf0a32ca..7ba71042ba 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -1,7 +1,7 @@ //! Parse one model file: header checks, model-local declaration checks and their messages. use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; -use crate::model_files::_helpers::locations::header_column_locations; +use crate::_helpers::locations::header_column_locations; use crate::model_files::_helpers::output_columns::output_column_locations; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use crate::models::{DiscoveryFailure, FailureKind}; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 7918049945..3f8be1e5de 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -20,6 +20,12 @@ pub enum FailureKind { Source, /// `ProjectPathError` (D016). ProjectPath, + /// `SqlHookParseError` (D014). + SqlHook, + /// `SqlAuditParseError` (D004). + SqlAudit, + /// `ResourceIdentityError` (D016). + ResourceIdentity, } impl FailureKind { @@ -33,6 +39,9 @@ impl FailureKind { Self::Schema => "schema", Self::Source => "source", Self::ProjectPath => "project_path", + Self::SqlHook => "sql_hook", + Self::SqlAudit => "sql_audit", + Self::ResourceIdentity => "resource_identity", } } } @@ -75,6 +84,8 @@ pub enum FileOutcome { Failed(DiscoveryFailure), /// The file could not be read or decoded as Python's `read_text` would. Unreadable(ReadFailure), + /// Native parsing cannot reproduce Python for this file; the Python parser reads it. + Deferred, } /// One discovered file, by `/`-separated path relative to the project directory. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 5de24615c9..5a21341886 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -42,7 +42,7 @@ use crate::bindings::models::{ }; use crate::bindings::types::CompilerDetach; -type PyObject = Py; +pub(crate) type PyObject = Py; type Located = (String, usize, usize, usize, usize); type Listing = (PyObject, Vec<(PyObject, bool, bool)>); @@ -53,16 +53,19 @@ fn located(locations: Vec<(String, LineColumnSpan)>) -> Vec { .collect() } -fn tuple_object<'py>(py: Python<'py>, items: Vec) -> PyResult { +pub(crate) fn tuple_object<'py>(py: Python<'py>, items: Vec) -> PyResult { Ok(PyTuple::new(py, items)?.unbind().into_any()) } -fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { +pub(crate) fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { value.into_py_any(py) } /// The Python semantics native parsing reproduces; an unknown Python is rejected by the facade. -fn python_semantics(python_version: (u8, u8), unicode_version: &str) -> PyResult { +pub(crate) fn python_semantics( + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult { python_text(python_version, unicode_version).ok_or_else(|| { PyValueError::new_err(format!( "native discovery does not support Python {}.{} with Unicode {unicode_version}", @@ -71,7 +74,7 @@ fn python_semantics(python_version: (u8, u8), unicode_version: &str) -> PyResult }) } -fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { +pub(crate) fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { tuple_object( py, vec![ @@ -84,7 +87,7 @@ fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult, failure: ReadFailure) -> PyResult { +pub(crate) fn read_object(py: Python<'_>, failure: ReadFailure) -> PyResult { match failure { ReadFailure::Io { errno, @@ -120,7 +123,7 @@ fn read_object(py: Python<'_>, failure: ReadFailure) -> PyResult { } /// A collection's failure payload; an unlistable directory is `("unlistable", path, read)`. -fn stage_failure_object( +pub(crate) fn stage_failure_object( py: Python<'_>, tree: &ProjectTree, failure: StageFailure, @@ -150,6 +153,7 @@ fn outcome_object( FileOutcome::Parsed(value) => parsed(py, value), FileOutcome::Unreadable(failure) => read_object(py, failure), FileOutcome::Failed(failure) => failure_object(py, failure), + FileOutcome::Deferred => tuple_object(py, vec![object(py, "defer")?]), } } @@ -190,7 +194,11 @@ fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult, tree: &ProjectTree, relative_path: String) -> PyResult { +pub(crate) fn path_object( + py: Python<'_>, + tree: &ProjectTree, + relative_path: String, +) -> PyResult { match tree.raw_relative(&relative_path) { Some(raw) => object(py, raw.into_os_string()), None => object(py, relative_path), From cdcd47859895473398fef9462823af4c1468c5fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:55:18 +0100 Subject: [PATCH 2/4] feat(native): discover declaration files through the native session behind the preview engine Add the preview-tier DECLARATION_FILES stage. Each declaration collection is read once per pass through a retained native discovery session; Python builds the records, normalises constant values and parses schema columns, and re-runs its own parser for any file native rejects or defers, raising NativeStageMismatchError when the outcomes differ. Default engines are unchanged. --- .../_helpers/discovery/declaration_files.rs | 409 +++++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../src/bindings/main/register.rs | 1 + crates/sqlbuild-python/src/bindings/models.rs | 32 + .../_helpers/coverage/discovery.py | 11 +- scripts/compiler_differential/constants.py | 2 +- src/sqlbuild/_native.pyi | 17 + .../compile/classes/render_reuse_session.py | 4 +- src/sqlbuild/compiler/compile/constants.py | 1 + .../_helpers/filesystem/aggregation.py | 5 + .../discovery/_helpers/filesystem/core.py | 121 +++- .../_helpers/native/declaration_files.py | 573 ++++++++++++++++++ .../discovery/_helpers/native/payloads.py | 6 + .../compiler/discovery/_helpers/sql/audits.py | 19 +- .../discovery/_helpers/sql/declarations.py | 26 +- .../compiler/discovery/_helpers/sql/hooks.py | 4 +- src/sqlbuild/compiler/discovery/constants.py | 18 + src/sqlbuild/compiler/discovery/models.py | 4 + src/sqlbuild/compiler/discovery/types.py | 12 +- .../frontier/classes/stage_capture_encoder.py | 6 +- src/sqlbuild/compiler/frontier/constants.py | 4 +- src/sqlbuild/compiler/frontier/types.py | 1 + .../compiler/discovery/_test_types.py | 67 ++ .../sqlbuild/compiler/discovery/helpers.py | 495 ++++++++++++++- .../test_native_declaration_files.py | 472 +++++++++++++++ .../_helpers/coverage/helpers.py | 12 +- .../frontier/test_native_stage_tiers.py | 18 + 27 files changed, 2303 insertions(+), 38 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs new file mode 100644 index 0000000000..5a961a34d8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs @@ -0,0 +1,409 @@ +//! Native declaration collections as the plain data the Python discovery facade materialises. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use sqlbuild_discovery::declaration_files::main::parse_declaration_text::parse_declaration_text; +use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; +use sqlbuild_discovery::declaration_files::models::{ + AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, + DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, + FunctionFile, HookFile, MacroFile, ParsedDeclarationText, SchemaFile, ScopedFile, SeedFile, +}; +use sqlbuild_discovery::declaration_files::types::Collection; +use sqlbuild_discovery::models::{DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot}; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::bindings::_helpers::discovery::project_files::{ + PyObject, failure_object, object, path_object, python_semantics, read_object, + stage_failure_object, tuple_object, +}; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::models::{ + DeclarationDiscoveryRequest, DeclarationTextRequest, NativeDiscoverySession, NativeProjectTree, +}; +use crate::bindings::types::CompilerDetach; + +type Located = (String, usize, usize, usize, usize); + +fn collection_kind(kind: &str) -> PyResult { + CollectionKind::from_name(kind) + .ok_or_else(|| PyValueError::new_err(format!("unknown declaration collection {kind:?}"))) +} + +fn ok_object(py: Python<'_>, mut items: Vec) -> PyResult { + items.insert(0, object(py, "ok")?); + tuple_object(py, items) +} + +fn values_object(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult { + map_to_python(py, values.to_vec()) +} + +/// `{key: value}`, or `{}` when the value is absent, for Python to project and read back. +fn single_value_object( + py: Python<'_>, + key: &str, + value: Option<&AuthoredValue>, +) -> PyResult { + map_to_python( + py, + value + .map(|value| vec![(key.to_owned(), value.clone())]) + .unwrap_or_default(), + ) +} + +fn failure_or_none(py: Python<'_>, failure: Option<&DiscoveryFailure>) -> PyResult { + failure + .map(|failure| failure_object(py, failure.clone())) + .transpose()? + .into_py_any(py) +} + +fn enum_object(py: Python<'_>, file: &EnumFile) -> PyResult { + let declarations: Vec = file + .declarations + .iter() + .map(|declaration| { + tuple_object( + py, + vec![ + object(py, &declaration.name)?, + values_object(py, &declaration.members)?, + object(py, declaration.scalar_type)?, + ], + ) + }) + .collect::>()?; + ok_object( + py, + vec![object(py, &file.contents)?, object(py, declarations)?], + ) +} + +fn constant_object(py: Python<'_>, file: &ConstantFile) -> PyResult { + let declarations: Vec = file + .declarations + .iter() + .map(|declaration| { + tuple_object( + py, + vec![ + object(py, &declaration.name)?, + single_value_object(py, "value", Some(&declaration.value))?, + object(py, &declaration.explicit_type)?, + object(py, &declaration.render_as)?, + ], + ) + }) + .collect::>()?; + ok_object( + py, + vec![ + object(py, &file.contents)?, + object(py, declarations)?, + failure_or_none(py, file.failure.as_ref())?, + ], + ) +} + +fn schema_object(py: Python<'_>, file: &SchemaFile) -> PyResult { + let declarations: Vec = file + .declarations + .iter() + .map(|declaration| { + let locations: Vec = + declaration.column_locations.iter().map(located).collect(); + tuple_object( + py, + vec![ + object(py, &declaration.name)?, + object(py, &declaration.description)?, + object(py, &declaration.extends)?, + single_value_object(py, "columns", declaration.columns.as_ref())?, + object(py, locations)?, + ], + ) + }) + .collect::>()?; + ok_object( + py, + vec![ + object(py, &file.contents)?, + object(py, declarations)?, + failure_or_none(py, file.failure.as_ref())?, + ], + ) +} + +fn located((name, span): &(String, LineColumnSpan)) -> Located { + ( + name.clone(), + span.line, + span.column, + span.end_line, + span.end_column, + ) +} + +fn audit_object(py: Python<'_>, file: &AuditFile) -> PyResult { + let blocks: Vec = file + .blocks + .iter() + .map(|block| { + tuple_object( + py, + vec![ + object(py, block.audit_index)?, + values_object(py, &block.header_values)?, + object(py, &block.sql_body)?, + object(py, &block.name)?, + object(py, block.evaluation_mode)?, + object(py, &block.measure_sql)?, + object(py, &block.evidence_sql)?, + ], + ) + }) + .collect::>()?; + ok_object(py, vec![object(py, &file.contents)?, object(py, blocks)?]) +} + +fn hook_object(py: Python<'_>, file: &HookFile) -> PyResult { + ok_object( + py, + vec![ + object(py, &file.contents)?, + values_object(py, &file.header_values)?, + object(py, &file.sql_body)?, + object(py, &file.name)?, + object(py, &file.description)?, + ], + ) +} + +fn function_object(py: Python<'_>, file: &FunctionFile) -> PyResult { + ok_object( + py, + vec![ + object(py, &file.contents)?, + values_object(py, &file.header_values)?, + object(py, &file.body_sql)?, + ], + ) +} + +fn macro_object(py: Python<'_>, file: &MacroFile) -> PyResult { + ok_object(py, vec![object(py, &file.contents)?]) +} + +fn seed_object(py: Python<'_>, _file: &SeedFile) -> PyResult { + ok_object(py, Vec::new()) +} + +fn outcome_object( + py: Python<'_>, + outcome: &FileOutcome, + parsed: fn(Python<'_>, &T) -> PyResult, +) -> PyResult { + match outcome { + FileOutcome::Parsed(value) => parsed(py, value), + FileOutcome::Failed(failure) => failure_object(py, failure.clone()), + FileOutcome::Unreadable(failure) => read_object(py, failure.clone()), + FileOutcome::Deferred => tuple_object(py, vec![object(py, "defer")?]), + } +} + +/// `(declaration kind, scope kind, ownership root, owning path, declaration root)`. +fn scope_object(py: Python<'_>, tree: &ProjectTree, scope: &FileScope) -> PyResult { + let path = |path: &Option| -> PyResult { + path.as_ref() + .map(|path| path_object(py, tree, path.clone())) + .transpose()? + .into_py_any(py) + }; + tuple_object( + py, + vec![ + object(py, scope.declaration_kind)?, + object(py, scope.scope_kind)?, + path(&scope.ownership_root)?, + path(&scope.owning_path)?, + path(&scope.declaration_root)?, + ], + ) +} + +/// Each file as `(relative_path, scope, payload)`, or the collection's failure payload. +fn collection_object( + py: Python<'_>, + tree: &ProjectTree, + collection: &Collection, + parsed: fn(Python<'_>, &T) -> PyResult, +) -> PyResult { + let files = match collection { + Ok(files) => files, + Err(CollectionFailure::Stage(failure)) => { + return stage_failure_object(py, tree, failure.clone()); + } + Err(CollectionFailure::Layout(failure)) => return failure_object(py, failure.clone()), + }; + files + .iter() + .map(|scoped| scoped_file_object(py, tree, scoped, parsed)) + .collect::>>()? + .into_py_any(py) +} + +/// `(relative_path, scope, payload)` of one file. +fn scoped_file_object( + py: Python<'_>, + tree: &ProjectTree, + scoped: &ScopedFile, + parsed: fn(Python<'_>, &T) -> PyResult, +) -> PyResult { + let scope: PyObject = scoped + .scope + .as_ref() + .map(|scope| scope_object(py, tree, scope)) + .transpose()? + .into_py_any(py)?; + tuple_object( + py, + vec![ + path_object(py, tree, scoped.file.relative_path.clone())?, + scope, + outcome_object(py, &scoped.file.outcome, parsed)?, + ], + ) +} + +fn declaration_collection_object( + py: Python<'_>, + tree: &ProjectTree, + collection: &DeclarationCollection, +) -> PyResult { + match collection { + DeclarationCollection::Enums(files) => collection_object(py, tree, files, enum_object), + DeclarationCollection::Constants(files) => { + collection_object(py, tree, files, constant_object) + } + DeclarationCollection::ModelSchemas(files) => { + collection_object(py, tree, files, schema_object) + } + DeclarationCollection::SqlFunctions(files) => { + collection_object(py, tree, files, function_object) + } + DeclarationCollection::SqlHooks(files) => collection_object(py, tree, files, hook_object), + DeclarationCollection::Audits(files) => collection_object(py, tree, files, audit_object), + DeclarationCollection::Seeds(files) => collection_object(py, tree, files, seed_object), + DeclarationCollection::Macros(files) => collection_object(py, tree, files, macro_object), + } +} + +fn options( + (function_keys, audit_keys, hook_keys): (Vec, Vec, Vec), + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult { + Ok(DeclarationFileOptions { + function_keys, + audit_keys, + hook_keys, + python: python_semantics(python_version, unicode_version)?, + }) +} + +#[pymethods] +impl NativeDiscoverySession { + #[new] + fn new(request: DeclarationDiscoveryRequest) -> PyResult { + Ok(Self { + inner: DiscoverySession::new( + ProjectRoot { + directory: PathBuf::from(request.project_dir), + display_prefix: request.display_prefix, + }, + options( + (request.function_keys, request.audit_keys, request.hook_keys), + request.python_version, + &request.unicode_version, + )?, + ), + }) + } + + /// Discover the collection `kind` once per session: one payload per file, or its failure. + #[pyo3(signature = (kind, tree, isolate_kind=false))] + fn collection( + &self, + py: Python<'_>, + kind: &str, + tree: &NativeProjectTree, + isolate_kind: bool, + ) -> PyResult { + let request = CollectionRequest { + kind: collection_kind(kind)?, + isolate_kind, + }; + let collection: Arc = py + .compiler_detach(|| { + Ok(retained_collection(&self.inner, request, || { + discover_declaration_collection(&self.inner, &tree.inner, request) + })) + }) + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; + declaration_collection_object(py, &tree.inner, &collection) + } +} + +/// Parse in-memory contents as discovery parses a `kind` declaration file named `file_path`. +#[pyfunction] +fn parse_declaration_contents( + py: Python<'_>, + request: DeclarationTextRequest, + names: (String, String), + contents: String, +) -> PyResult { + let (file_path, hook_name): (String, String) = names; + let kind: CollectionKind = collection_kind(&request.kind)?; + let options: DeclarationFileOptions = options( + (request.function_keys, request.audit_keys, request.hook_keys), + request.python_version, + &request.unicode_version, + )?; + let parsed: Option> = py + .compiler_detach(|| { + Ok(parse_declaration_text( + kind, + (&file_path, &hook_name), + contents, + &options, + )) + }) + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; + let Some(outcome) = parsed else { + return Err(PyValueError::new_err(format!( + "{} files have no authored text to parse", + kind.as_str() + ))); + }; + outcome_object(py, &outcome, |py, parsed| match parsed { + ParsedDeclarationText::Enum(file) => enum_object(py, file), + ParsedDeclarationText::Constant(file) => constant_object(py, file), + ParsedDeclarationText::ModelSchema(file) => schema_object(py, file), + ParsedDeclarationText::SqlFunction(file) => function_object(py, file), + ParsedDeclarationText::SqlHook(file) => hook_object(py, file), + ParsedDeclarationText::Audit(file) => audit_object(py, file), + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_declaration_contents, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 6ff8145be2..d1d34b11ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1 +1,2 @@ +pub(crate) mod declaration_files; pub(crate) mod project_files; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index ea826ad8b6..7600d563b7 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -27,6 +27,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { sqltext::registration::register(module)?; boundary::oracles::register(module)?; discovery::project_files::register(module)?; + discovery::declaration_files::register(module)?; scopes::scope_index::register(module)?; scopes::relationship_names::register(module)?; model_config::header_metadata::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 5e08cd1970..1d0ff9eb54 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -164,3 +164,35 @@ pub(crate) struct ModelTextRequest { pub(crate) python_version: (u8, u8), pub(crate) unicode_version: String, } + +/// The declaration collections one Python discovery pass read natively, kept for later stages. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeDiscoverySession { + pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, +} + +/// One native declaration discovery session request from the Python discovery facade. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct DeclarationDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) function_keys: Vec, + pub(crate) audit_keys: Vec, + pub(crate) hook_keys: Vec, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, +} + +/// One in-memory declaration file parse request; `kind` names the collection it belongs to. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct DeclarationTextRequest { + pub(crate) kind: String, + pub(crate) function_keys: Vec, + pub(crate) audit_keys: Vec, + pub(crate) hook_keys: Vec, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, +} diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index bb4d871983..31ef17aada 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -37,6 +37,11 @@ ) from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES + +_OMITTED_DISCOVERY_FIELDS: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( + f"{DiscoveredProjectInputs.__module__}:{DiscoveredProjectInputs.__qualname__}", frozenset() +) _CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) _NON_ASCII_COMMENT: re.Pattern[str] = re.compile(r"--[^\n]*[^\x00-\x7f]") @@ -123,7 +128,11 @@ def discovery_capture_problems(capture_text: str) -> tuple[str, ...]: return capture_problems( capture_text=capture_text, - fields=tuple(field.name for field in dataclasses.fields(DiscoveredProjectInputs)), + fields=tuple( + field.name + for field in dataclasses.fields(DiscoveredProjectInputs) + if field.name not in _OMITTED_DISCOVERY_FIELDS + ), callable_fields=CALLABLE_CAPTURE_FIELDS, ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 24d8bc4ed7..756d0a779f 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -274,7 +274,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } DISCOVERY_STAGE_CAPTURE_SUFFIX: str = "-discovered_project_inputs.json" DISCOVERY_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( - {"project_config", "local_config", "project_dir"} + {"project_config", "local_config", "project_dir", "native_session"} ) DISCOVERY_DECLARATION_FILE_COLLECTIONS: tuple[str, ...] = ( "enum_files", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 74ff9791f6..61e786b7aa 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -62,6 +62,23 @@ def load_yaml_files( request: dict[str, object], relative_paths: list[str], tree: NativeProjectTree ) -> list[tuple[object, ...]] | tuple[object, ...]: ... def load_yaml_document(file_path: str, text: str, kind: str) -> tuple[object, ...]: ... + +class NativeDiscoverySession: + def __init__(self, request: dict[str, object]) -> None: ... + def collection( + self, kind: str, tree: NativeProjectTree, isolate_kind: bool = False + ) -> ( + list[ + tuple[ + str, tuple[str, str, str | None, str | None, str | None] | None, tuple[object, ...] + ] + ] + | tuple[object, ...] + ): ... + +def parse_declaration_contents( + request: dict[str, object], names: tuple[str, str], contents: str +) -> tuple[object, ...]: ... def discover_declaration_layout( tree: NativeProjectTree, kind: str | None = None ) -> tuple[tuple[object, ...], tuple[object, ...]] | tuple[object, ...]: ... diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py index 8cf0c62c8d..f0aca97da8 100644 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py @@ -132,7 +132,9 @@ def declaration_files( ): declarations: DiscoveredDeclarationFiles = discover() recorded: StoredRender | None = _stored_render( - reads=reads, value=replace(declarations, model_files=()), reported=reported + reads=reads, + value=replace(declarations, model_files=(), native_session=None), + reported=reported, ) if recorded is not None: self._pending_groups.append((name, recorded)) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 0c61e28222..c9b09a70f5 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -52,6 +52,7 @@ "materialization_files": "materialization strategies, not graph resources", "event_exporters": "runtime event outputs, not graph resources", "command_output_sinks": "runtime command outputs, not graph resources", + "native_session": "native parse state kept for later stages, not a project input", } SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 0ee7f44a1e..c8cff23b66 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -29,6 +29,9 @@ from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( build_integration_loader_functions, ) +from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( + retained_discovery_session, +) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) @@ -206,6 +209,7 @@ def build_discovered_project_inputs( command_output_sinks=command_output_sinks, providers=providers, adapter_file=declarations.adapter_file, + native_session=declarations.native_session, ) @@ -235,6 +239,7 @@ def _discover_declaration_files( audit_files=discover_audit_files(project_dir=project_dir), macro_files=discover_macro_files(project_dir=project_dir), adapter_file=discover_adapter_file(project_dir=project_dir), + native_session=retained_discovery_session(project_dir=project_dir), ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 1d0ab8a1aa..56d4d62865 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -22,6 +22,16 @@ is_in_scoped_declaration_tree, project_relative_path, ) +from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( + native_audit_files, + native_constant_files, + native_enum_files, + native_macro_files, + native_model_schema_files, + native_seed_files, + native_sql_function_files, + native_sql_hook_files, +) from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_file_facts from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file @@ -70,7 +80,9 @@ DiscoveryFileFault, NamedDeclarationRoot, ) -from sqlbuild.compiler.discovery.types import ScopedDeclarationFile +from sqlbuild.compiler.discovery.types import NativeFileScope, ScopedDeclarationFile +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) @@ -189,6 +201,7 @@ def discover_enum_files( parse_declarations=parse_enum_declaration_file, discovered_file_type=DiscoveredEnumFile, on_fault=on_fault, + native_files=native_enum_files, ) @@ -207,6 +220,7 @@ def discover_constant_files( parse_declarations=parse_constant_declaration_file, discovered_file_type=DiscoveredConstantFile, on_fault=on_fault, + native_files=native_constant_files, ) @@ -218,6 +232,7 @@ def _discover_declaration_kind_files[DeclarationT, DiscoveredT]( parse_declarations: Callable[..., tuple[DeclarationT, ...]], discovered_file_type: Callable[..., DiscoveredT], on_fault: Callable[[DiscoveryFileFault], None] | None, + native_files: Callable[..., tuple[DiscoveredT, ...]], ) -> tuple[DiscoveredT, ...]: def parse(facts: _DeclarationFileFacts) -> DiscoveredT: contents: str = facts.file_path.read_text(encoding="utf-8") @@ -237,6 +252,13 @@ def parse(facts: _DeclarationFileFacts) -> DiscoveredT: declaration_root=facts.declaration_root, ) + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_files( + project_dir=project_dir, + isolate_declaration_kind=isolate_declaration_kind, + on_fault=on_fault, + parse_with_python=_scoped_parser(project_dir=project_dir, parse=parse), + ) return _parse_discovered_files( project_dir=project_dir, items=( @@ -253,6 +275,53 @@ def parse(facts: _DeclarationFileFacts) -> DiscoveredT: ) +def _scoped_parser[DiscoveredT]( + *, project_dir: Path, parse: Callable[[_DeclarationFileFacts], DiscoveredT] +) -> Callable[..., DiscoveredT]: + """Parse a macro, enum or constant file native discovery listed as the Python path does.""" + + def parse_listed(*, relative_path: Path, scope: NativeFileScope | None) -> DiscoveredT: + kind, scope_kind, ownership_root, owning_path, declaration_root = cast( + NativeFileScope, scope + ) + return parse( + _DeclarationFileFacts( + file_path=project_dir / relative_path, + relative_path=relative_path, + declaration_kind=DeclarationKind(kind), + scope_kind=ScopeKind(scope_kind), + ownership_root=Path(cast(str, ownership_root)), + owning_path=None if owning_path is None else Path(owning_path), + declaration_root=Path(cast(str, declaration_root)), + ) + ) + + return parse_listed + + +def _named_parser[DiscoveredT: ScopedDeclarationFile]( + *, project_dir: Path, parse: Callable[[tuple[NamedDeclarationRoot, Path]], DiscoveredT] +) -> Callable[..., DiscoveredT]: + """Parse a named declaration file native discovery listed as its Python role places it.""" + + def parse_listed(*, relative_path: Path, scope: NativeFileScope | None) -> DiscoveredT: + kind, scope_kind, ownership_root, owning_path, declaration_root = cast( + NativeFileScope, scope + ) + relative_directory: Path = Path(cast(str, declaration_root)) + root: NamedDeclarationRoot = NamedDeclarationRoot( + directory=project_dir / relative_directory, + relative_directory=relative_directory, + kind=DeclarationKind(kind), + scope_kind=ScopeKind(scope_kind), + ownership_root=None if ownership_root is None else Path(ownership_root), + owning_path=None if owning_path is None else Path(owning_path), + ) + return root.place(parse((root, project_dir / relative_path))) + + return parse_listed + + def _parse_discovered_files[ItemT, DiscoveredT]( *, project_dir: Path, @@ -316,6 +385,12 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredModelSchemaFile: ), ) + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_model_schema_files( + project_dir=project_dir, + on_fault=on_fault, + parse_with_python=_named_parser(project_dir=project_dir, parse=parse), + ) return _parse_named_declaration_files( project_dir=project_dir, kinds=frozenset({DeclarationKind.SCHEMA}), @@ -356,10 +431,6 @@ def discover_sql_function_files( ) -> tuple[DiscoveredSqlFunctionFile, ...]: """Discover SQL function files under functions/sql/.""" - function_root: Path = project_dir / "functions" / "sql" - if not function_root.is_dir(): - return () - def parse(file_path: Path) -> DiscoveredSqlFunctionFile: contents: str = file_path.read_text(encoding="utf-8") header_values: dict[str, object] @@ -373,6 +444,19 @@ def parse(file_path: Path) -> DiscoveredSqlFunctionFile: body_sql=body_sql, ) + def parse_listed( + *, relative_path: Path, scope: NativeFileScope | None + ) -> DiscoveredSqlFunctionFile: + _ = scope + return parse(project_dir / relative_path) + + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_sql_function_files( + project_dir=project_dir, on_fault=on_fault, parse_with_python=parse_listed + ) + function_root: Path = project_dir / "functions" / "sql" + if not function_root.is_dir(): + return () return _parse_discovered_paths( project_dir=project_dir, file_paths=_unscoped_files(root=function_root, pattern="*.sql", project_dir=project_dir), @@ -416,6 +500,8 @@ def discover_python_function_files( def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: """Discover seed CSV files under seeds/.""" + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_seed_files(project_dir=project_dir) seeds_root: Path = project_dir / "seeds" if not seeds_root.is_dir(): return () @@ -446,6 +532,12 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredAuditFile: declaration_kind=root.kind, ) + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_audit_files( + project_dir=project_dir, + on_fault=on_fault, + parse_with_python=_named_parser(project_dir=project_dir, parse=parse), + ) return _parse_named_declaration_files( project_dir=project_dir, kinds=frozenset({DeclarationKind.AUDIT, DeclarationKind.SINGULAR_AUDIT}), @@ -459,8 +551,8 @@ def discover_macro_files( ) -> tuple[DiscoveredMacroFile, ...]: """Discover global and scoped project macro Python files.""" - return tuple( - DiscoveredMacroFile( + def read(facts: _DeclarationFileFacts) -> DiscoveredMacroFile: + return DiscoveredMacroFile( file_path=facts.file_path, relative_path=facts.relative_path, contents=facts.file_path.read_text(encoding="utf-8"), @@ -469,6 +561,15 @@ def discover_macro_files( owning_path=facts.owning_path, declaration_root=facts.declaration_root, ) + + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_macro_files( + project_dir=project_dir, + isolate_declaration_kind=isolate_declaration_kind, + read_with_python=_scoped_parser(project_dir=project_dir, parse=read), + ) + return tuple( + read(facts) for facts in _discover_declaration_file_facts( project_dir=project_dir, declaration_kind=DeclarationKind.MACRO if isolate_declaration_kind else None, @@ -609,6 +710,12 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredSqlHookFile: relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) + if native_stage_enabled(NativeStage.DECLARATION_FILES): + return native_sql_hook_files( + project_dir=project_dir, + on_fault=on_fault, + parse_with_python=_named_parser(project_dir=project_dir, parse=parse), + ) return _parse_named_declaration_files( project_dir=project_dir, kinds=frozenset({DeclarationKind.SQL_HOOK}), diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py new file mode 100644 index 0000000000..cc5f2d2651 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py @@ -0,0 +1,573 @@ +"""Declaration files discovered by the native engine, materialised as Python discovery records.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.auditing.models import MeasurementContract +from sqlbuild.compiler.auditing.types import AuditEvaluationMode +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + materialise_native_files, + native_collection, + native_failure, + native_locations, + native_project_tree, + native_read_error, + native_request, + seed_snapshot_listings, +) +from sqlbuild.compiler.discovery._helpers.sql.declarations import typed_constant_declaration +from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values +from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.constants import ( + NATIVE_DEFERRED_TAG, + NATIVE_FAILED_TAG, + NATIVE_PARSED_TAG, + SQL_AUDIT_HEADER_KEYS, + SQL_FUNCTION_HEADER_KEYS, + SQL_HOOK_HEADER_KEYS, +) +from sqlbuild.compiler.discovery.exceptions import DeclarationParseError +from sqlbuild.compiler.discovery.models import ( + ConstantDeclaration, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredModelSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveryFileFault, + EnumDeclaration, + EnumMember, + ModelSchemaDeclaration, +) +from sqlbuild.compiler.discovery.types import NativeFileScope, NativeLocation, NativeScopeFields +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError +from sqlbuild.compiler.frontier.types import CompilerEngine, NativeStage +from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind + +type _NativeDeclarationFile = tuple[str, NativeFileScope | None, tuple[object, ...]] +type _PythonParse[RecordT] = Callable[..., RecordT] +type _Build[RecordT] = Callable[..., RecordT] +type _EnumPayload = tuple[str, dict[str, object], str] +type _ConstantPayload = tuple[str, dict[str, object], str | None, str | None] +type _SchemaPayload = tuple[str, str | None, str | None, dict[str, object], list[NativeLocation]] + +_SESSION_MEMO_KEY: str = "native_discovery_session" +_FILE_ERRORS: tuple[type[Exception], ...] = (OSError, UnicodeError, ValueError, SyntaxError) + + +def native_enum_files( + *, + project_dir: Path, + isolate_declaration_kind: bool, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredEnumFile], +) -> tuple[DiscoveredEnumFile, ...]: + """Discover the enum files natively; Python parses the files native parsing defers.""" + + return _native_records( + project_dir=project_dir, + kind=DeclarationKind.ENUM, + isolate_kind=isolate_declaration_kind, + build=_enum_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_constant_files( + *, + project_dir: Path, + isolate_declaration_kind: bool, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredConstantFile], +) -> tuple[DiscoveredConstantFile, ...]: + """Discover the constant files natively; Python normalises each constant's value.""" + + return _native_records( + project_dir=project_dir, + kind=DeclarationKind.CONSTANT, + isolate_kind=isolate_declaration_kind, + build=_constant_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_model_schema_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredModelSchemaFile], +) -> tuple[DiscoveredModelSchemaFile, ...]: + """Discover the reusable model schema files natively; Python parses each schema's columns.""" + + return _native_records( + project_dir=project_dir, + kind="model_schema", + build=_schema_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_sql_function_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredSqlFunctionFile], +) -> tuple[DiscoveredSqlFunctionFile, ...]: + """Discover the SQL function files natively.""" + + return _native_records( + project_dir=project_dir, + kind="sql_function", + build=_function_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_sql_hook_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredSqlHookFile], +) -> tuple[DiscoveredSqlHookFile, ...]: + """Discover the named SQL hook files natively.""" + + return _native_records( + project_dir=project_dir, + kind="sql_hook", + build=_hook_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_audit_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None, + parse_with_python: _PythonParse[DiscoveredAuditFile], +) -> tuple[DiscoveredAuditFile, ...]: + """Discover the generic and singular audit files natively.""" + + return _native_records( + project_dir=project_dir, + kind="audit", + build=_audit_file, + parse_with_python=parse_with_python, + on_fault=on_fault, + ) + + +def native_macro_files( + *, + project_dir: Path, + isolate_declaration_kind: bool, + read_with_python: _PythonParse[DiscoveredMacroFile], +) -> tuple[DiscoveredMacroFile, ...]: + """Discover and read the macro files natively; Python reads the files native reading defers.""" + + return _native_records( + project_dir=project_dir, + kind=DeclarationKind.MACRO, + isolate_kind=isolate_declaration_kind, + build=_macro_file, + parse_with_python=read_with_python, + on_fault=None, + ) + + +def native_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: + """Discover the seed files natively.""" + + return tuple( + DiscoveredSeedFile(file_path=project_dir / relative_path, relative_path=Path(relative_path)) + for relative_path, _scope, _payload in _native_files(project_dir=project_dir, kind="seed") + ) + + +def retained_discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession | None: + """Return the native session this discovery pass read its declaration files with, if any.""" + + session: object = DirectorySnapshot.current(project_dir=project_dir).memo.get(_SESSION_MEMO_KEY) + return session if isinstance(session, _native.NativeDiscoverySession) else None + + +def _native_records[RecordT]( + *, + project_dir: Path, + kind: str, + build: _Build[RecordT], + parse_with_python: _PythonParse[RecordT], + on_fault: Callable[[DiscoveryFileFault], None] | None, + isolate_kind: bool = False, +) -> tuple[RecordT, ...]: + files: list[_NativeDeclarationFile] = _native_files( + project_dir=project_dir, kind=kind, isolate_kind=isolate_kind + ) + return materialise_native_files( + project_dir=project_dir, + files=((Path(relative_path), (scope, payload)) for relative_path, scope, payload in files), + build=lambda relative_path, item: _record( + project_dir=project_dir, + relative_path=relative_path, + item=item, + build=build, + parse_with_python=parse_with_python, + ), + on_fault=on_fault, + ) + + +def _native_files( + *, project_dir: Path, kind: str, isolate_kind: bool = False +) -> list[_NativeDeclarationFile]: + with DirectorySnapshot.scope(project_dir=project_dir): + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: list[_NativeDeclarationFile] = native_collection( + result=_discovery_session(project_dir=project_dir).collection(kind, tree, isolate_kind), + project_dir=project_dir, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return files + + +def _discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession: + snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + session: _native.NativeDiscoverySession | None = retained_discovery_session( + project_dir=project_dir + ) + if session is not None: + return session + created: _native.NativeDiscoverySession = _native.NativeDiscoverySession( + native_request( + project_dir=project_dir, + fields={ + "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), + "audit_keys": sorted(SQL_AUDIT_HEADER_KEYS), + "hook_keys": sorted(SQL_HOOK_HEADER_KEYS), + }, + ) + ) + snapshot.memo[_SESSION_MEMO_KEY] = created + return created + + +def _record[RecordT]( + *, + project_dir: Path, + relative_path: Path, + item: tuple[NativeFileScope | None, tuple[object, ...]], + build: _Build[RecordT], + parse_with_python: _PythonParse[RecordT], +) -> RecordT: + """Build a parsed file; any other outcome is Python's to raise, and must match native.""" + + scope, payload = item + tag: object = payload[0] + if tag == NATIVE_DEFERRED_TAG: + return parse_with_python(relative_path=relative_path, scope=scope) + expected: Exception + if tag == NATIVE_PARSED_TAG: + try: + return build( + project_dir=project_dir, relative_path=relative_path, scope=scope, payload=payload + ) + except _FILE_ERRORS as error: + expected = error + elif tag == NATIVE_FAILED_TAG: + expected = native_failure(payload) + else: + expected = native_read_error(payload=payload, file_path=project_dir / relative_path) + return _python_failure( + relative_path=relative_path, + scope=scope, + expected=expected, + parse_with_python=parse_with_python, + ) + + +def _python_failure[RecordT]( + *, + relative_path: Path, + scope: NativeFileScope | None, + expected: Exception, + parse_with_python: _PythonParse[RecordT], +) -> RecordT: + """Raise Python's error for a file native discovery failed, or a mismatch if they differ.""" + + try: + _ = parse_with_python(relative_path=relative_path, scope=scope) + except _FILE_ERRORS as error: + if _same_error(error=error, expected=expected): + raise + raise _mismatch( + relative_path=relative_path, expected=expected, python_outcome=f"raised {error!r}" + ) from error + raise _mismatch(relative_path=relative_path, expected=expected, python_outcome="parsed it") + + +def _same_error(*, error: Exception, expected: Exception) -> bool: + return ( + type(error) is type(expected) + and str(error) == str(expected) + and getattr(error, "help", None) == getattr(expected, "help", None) + ) + + +def _mismatch( + *, relative_path: Path, expected: Exception, python_outcome: str +) -> NativeStageMismatchError: + return NativeStageMismatchError( + f"native {NativeStage.DECLARATION_FILES} raised {expected!r} for {relative_path} but the " + f"Python compiler {python_outcome}; rerun with {COMPILER_ENGINE_ENV_VAR}=" + f"{CompilerEngine.PYTHON} and report this mismatch" + ) + + +def _raise_partial_failure(failure: object) -> None: + if failure is not None: + raise native_failure(cast(tuple[object, ...], failure)) + + +def _scope_fields(scope: NativeFileScope | None) -> NativeScopeFields: + if scope is None: + return NativeScopeFields() + _kind, scope_kind, ownership_root, owning_path, declaration_root = scope + return NativeScopeFields( + scope_kind=ScopeKind(scope_kind), + ownership_root=_optional_path(ownership_root), + owning_path=_optional_path(owning_path), + declaration_root=_optional_path(declaration_root), + ) + + +def _optional_path(text: str | None) -> Path | None: + return None if text is None else Path(text) + + +def _audit_block(block: tuple[object, ...]) -> DiscoveredAuditBlock: + audit_index, values, sql_body, name, mode, measure_sql, evidence_sql = block + header_values: dict[str, object] = project_native_header_values(cast(dict[str, object], values)) + evaluation_mode: AuditEvaluationMode = AuditEvaluationMode(str(mode)) + return DiscoveredAuditBlock( + audit_index=cast(int, audit_index), + header_values=header_values, + sql_body=str(sql_body), + name=cast(str | None, name), + evaluation_mode=evaluation_mode, + measurement_contract=( + MeasurementContract( + value_column=cast(str, header_values["value"]), + sample_count_column=cast(str | None, header_values.get("sample_count")), + sample_unit=cast(str | None, header_values.get("sample_unit")), + ) + if evaluation_mode is AuditEvaluationMode.MEASUREMENT + else None + ), + measure_sql=cast(str | None, measure_sql), + evidence_sql=cast(str | None, evidence_sql), + ) + + +def _enum_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredEnumFile: + _tag, contents, declarations = payload + return DiscoveredEnumFile( + file_path=project_dir / relative_path, + relative_path=relative_path, + contents=str(contents), + declarations=tuple( + _enum_declaration(relative_path=relative_path, declaration=declaration) + for declaration in cast(list[_EnumPayload], declarations) + ), + **_scope_fields(scope), + ) + + +def _enum_declaration(*, relative_path: Path, declaration: _EnumPayload) -> EnumDeclaration: + name, members, scalar_type = declaration + projected: dict[str, object] = project_native_header_values(members) + return EnumDeclaration( + name=name, + members=tuple( + EnumMember(name=member, value=cast(str | int, value)) + for member, value in projected.items() + ), + scalar_type=scalar_type, + relative_path=relative_path, + ) + + +def _constant_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredConstantFile: + _tag, contents, declarations, failure = payload + file_path: Path = project_dir / relative_path + constants: tuple[ConstantDeclaration, ...] = tuple( + _constant_declaration( + file_path=file_path, relative_path=relative_path, declaration=declaration + ) + for declaration in cast(list[_ConstantPayload], declarations) + ) + _raise_partial_failure(failure) + return DiscoveredConstantFile( + file_path=file_path, + relative_path=relative_path, + contents=str(contents), + declarations=constants, + **_scope_fields(scope), + ) + + +def _constant_declaration( + *, file_path: Path, relative_path: Path, declaration: _ConstantPayload +) -> ConstantDeclaration: + name, value, explicit_type, render_as = declaration + return typed_constant_declaration( + name=name, + value=project_native_header_values(value)["value"], + explicit_type=explicit_type, + render_as_text=render_as, + file_path=file_path, + relative_path=relative_path, + model_name=None, + ) + + +def _schema_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredModelSchemaFile: + _tag, contents, declarations, failure = payload + file_path: Path = project_dir / relative_path + schemas: tuple[ModelSchemaDeclaration, ...] = tuple( + _schema_declaration( + file_path=file_path, relative_path=relative_path, declaration=declaration + ) + for declaration in cast(list[_SchemaPayload], declarations) + ) + _raise_partial_failure(failure) + return DiscoveredModelSchemaFile( + file_path=file_path, + relative_path=relative_path, + contents=str(contents), + declarations=schemas, + **_scope_fields(scope), + ) + + +def _schema_declaration( + *, file_path: Path, relative_path: Path, declaration: _SchemaPayload +) -> ModelSchemaDeclaration: + name, description, extends, columns, locations = declaration + return ModelSchemaDeclaration( + name=name, + description=description, + extends=extends, + columns=parse_schema_columns( + raw_columns=project_native_header_values(columns).get("columns"), + file_path=file_path, + label=f"schema '{name}'", + error_class=DeclarationParseError, + column_locations=native_locations(locations=locations, relative_path=relative_path), + require_columns=True, + ), + relative_path=relative_path, + ) + + +def _function_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredSqlFunctionFile: + _ = scope + _tag, contents, header_values, body_sql = payload + return DiscoveredSqlFunctionFile( + file_path=project_dir / relative_path, + relative_path=relative_path, + contents=str(contents), + header_values=project_native_header_values(cast(dict[str, object], header_values)), + body_sql=str(body_sql), + ) + + +def _hook_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredSqlHookFile: + _tag, contents, header_values, sql_body, name, description = payload + return DiscoveredSqlHookFile( + file_path=project_dir / relative_path, + relative_path=relative_path, + contents=str(contents), + header_values=project_native_header_values(cast(dict[str, object], header_values)), + sql_body=str(sql_body), + name=str(name), + description=cast(str | None, description), + **_scope_fields(scope), + ) + + +def _audit_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredAuditFile: + _tag, contents, blocks = payload + return DiscoveredAuditFile( + file_path=project_dir / relative_path, + relative_path=relative_path, + contents=str(contents), + blocks=tuple(_audit_block(block) for block in cast(list[tuple[object, ...]], blocks)), + declaration_kind=DeclarationKind(cast(NativeFileScope, scope)[0]), + **_scope_fields(scope), + ) + + +def _macro_file( + *, + project_dir: Path, + relative_path: Path, + scope: NativeFileScope | None, + payload: tuple[object, ...], +) -> DiscoveredMacroFile: + _tag, contents = payload + return DiscoveredMacroFile( + file_path=project_dir / relative_path, + relative_path=relative_path, + contents=str(contents), + **_scope_fields(scope), + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 27bf2066e9..4c2c0a83c3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -24,6 +24,8 @@ ProjectPathError, SchemaParseError, SourceParseError, + SqlAuditParseError, + SqlHookParseError, SqlScenarioParseError, SqlTestParseError, UnsupportedPythonError, @@ -34,6 +36,7 @@ NativeListing, NativeLocation, ) +from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.spec.contracts.models import SourceLocation _DISPLAY_PROBE: str = "_" @@ -56,6 +59,9 @@ "schema": SchemaParseError, "source": SourceParseError, _PROJECT_PATH_KIND: ProjectPathError, + "sql_hook": SqlHookParseError, + "sql_audit": SqlAuditParseError, + "resource_identity": ResourceIdentityError, } diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py b/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py index f384da39b2..138f3616b8 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py @@ -11,7 +11,7 @@ from sqlbuild.compiler.auditing.types import AuditEvaluationMode from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY, SQL_AUDIT_HEADER_KEYS from sqlbuild.compiler.discovery.exceptions import SqlAuditParseError from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock @@ -31,21 +31,6 @@ _IDENTIFIER_SEPARATOR: str = "_" _ESCAPE_CHARACTER: str = "\\" _SQL_QUOTES: frozenset[str] = frozenset({"'", '"', "`"}) -_SUPPORTED_AUDIT_HEADER_KEYS: frozenset[str] = frozenset( - { - _AUDIT_NAME_HEADER_KEY, - _AUDIT_SEVERITY_HEADER_KEY, - _AUDIT_RUN_SCOPE_HEADER_KEY, - _AUDIT_ALWAYS_RUN_HEADER_KEY, - _AUDIT_EVALUATION_HEADER_KEY, - _AUDIT_VALUE_HEADER_KEY, - _AUDIT_SAMPLE_COUNT_HEADER_KEY, - _AUDIT_SAMPLE_UNIT_HEADER_KEY, - _AUDIT_THRESHOLDS_HEADER_KEY, - _AUDIT_MINIMUM_SAMPLES_HEADER_KEY, - SQL_ANALYSIS_CONFIG_KEY, - } -) def parse_sql_audit_file(*, contents: str, file_path: Path) -> tuple[DiscoveredAuditBlock, ...]: @@ -145,7 +130,7 @@ def _parse_audit_header(*, header: str, header_line: int, file_path: Path) -> di ) reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_SUPPORTED_AUDIT_HEADER_KEYS, + supported_keys=SQL_AUDIT_HEADER_KEYS, statement="AUDIT()", header=header, header_line=header_line, diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py index 3a433fa277..45e31c7c9d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py @@ -423,6 +423,30 @@ def _parse_constant_declaration( file_path=file_path, label=f"constant '{parsed_name}' render_as", ) + return typed_constant_declaration( + name=parsed_name, + value=value, + explicit_type=parsed_explicit_type, + render_as_text=render_as_text, + file_path=file_path, + relative_path=relative_path, + model_name=model_name, + ) + + +def typed_constant_declaration( + *, + name: str, + value: object, + explicit_type: str | None, + render_as_text: str | None, + file_path: Path, + relative_path: Path, + model_name: str | None, +) -> ConstantDeclaration: + """Normalize a constant whose name and options are valid into its typed declaration.""" + + parsed_name: str = name try: render_as: CollectionRendering | None = ( CollectionRendering(render_as_text) if render_as_text is not None else None @@ -434,7 +458,7 @@ def _parse_constant_declaration( try: typed_value: SqlValue = normalize_sql_value( raw_value=value, - explicit_type=parsed_explicit_type, + explicit_type=explicit_type, context=f"{file_path} constant '{parsed_name}'", ) except SqlValueValidationError as error: diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py b/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py index c0f2e614ba..7802abec2c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py @@ -8,11 +8,11 @@ from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values +from sqlbuild.compiler.discovery.constants import SQL_HOOK_HEADER_KEYS from sqlbuild.compiler.discovery.exceptions import SqlHookParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile _HOOK_DESCRIPTION_HEADER_KEY: str = "description" -_SUPPORTED_HOOK_HEADER_KEYS: frozenset[str] = frozenset({_HOOK_DESCRIPTION_HEADER_KEY}) _SQL_ESCAPE_CHARACTER: str = "\\" _SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"', "`"}) _SQL_STATEMENT_TERMINATOR: str = ";" @@ -111,7 +111,7 @@ def _parse_hook_header(*, header: str, header_line: int, file_path: Path) -> dic ) reject_unsupported_header_keys( header_values=parsed_header, - supported_keys=_SUPPORTED_HOOK_HEADER_KEYS, + supported_keys=SQL_HOOK_HEADER_KEYS, statement="HOOK()", header=header, header_line=header_line, diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 4435d6ebf3..5cb584b78e 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -20,6 +20,22 @@ SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( {"arguments", "returns", "database", "schema", "tags", "description"} ) +SQL_AUDIT_HEADER_KEYS: frozenset[str] = frozenset( + { + "name", + "severity", + "run_scope", + "always_run", + "evaluation", + "value", + "sample_count", + "sample_unit", + "thresholds", + "minimum_samples", + "sql_analysis", + } +) +SQL_HOOK_HEADER_KEYS: frozenset[str] = frozenset({"description"}) SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( { "alias", @@ -183,6 +199,8 @@ NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" NATIVE_UNLISTABLE_TAG: str = "unlistable" +NATIVE_DEFERRED_TAG: str = "defer" +NATIVE_PARSED_TAG: str = "ok" NATIVE_SOURCE_YAML_KIND: str = "source" NATIVE_SCHEMA_YAML_KIND: str = "schema" NATIVE_SUPPORTED_PYTHON_VERSIONS: str = "3.12, 3.13 and 3.14" diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 5434a8446e..93edb4afc2 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path +from sqlbuild._native import NativeDiscoverySession from sqlbuild.compiler.auditing.models import MeasurementContract from sqlbuild.compiler.auditing.types import AuditEvaluationMode from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE @@ -637,6 +638,7 @@ class DiscoveredDeclarationFiles: audit_files: tuple[DiscoveredAuditFile, ...] macro_files: tuple[DiscoveredMacroFile, ...] adapter_file: DiscoveredAdapterFile | None + native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) @dataclass(frozen=True) @@ -671,6 +673,8 @@ class DiscoveredProjectInputs: command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) adapter_file: DiscoveredAdapterFile | None = None + native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) + """The native discovery session that read the declaration files, for later native stages.""" @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 7e82f1e95f..1dcdded57a 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -6,7 +6,7 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, TypedDict from sqlbuild.compiler.scopes.types import ScopeKind @@ -22,6 +22,7 @@ type NativeListing = tuple[str, list[tuple[str, bool, bool]]] type NativeDeclarationFact = tuple[str, str, str, str, str | None, str] type NativeFiles = list[tuple[str, tuple[object, ...]]] +type NativeFileScope = tuple[str, str, str | None, str | None, str | None] class LoaderConnectionMode(StrEnum): @@ -41,6 +42,15 @@ class ScopedDeclarationFile(Protocol): declaration_root: Path | None +class NativeScopeFields(TypedDict, total=False): + """The scope fields of a declaration file record that native discovery listed in a role.""" + + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + + class DirectorySnapshotEntry(NamedTuple): """One directory entry in a discovery snapshot.""" diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py index 4807ca8f74..d7b40356cc 100644 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py @@ -103,11 +103,15 @@ def _encode_container(self, value: object) -> object: if type_name in STAGE_CAPTURE_DECODED_SEQUENCES and isinstance(value, Sequence): return [self.encode(item) for item in value] if dataclasses.is_dataclass(value) and not isinstance(value, type): + omitted_fields: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( + type_name, frozenset() + ) return { - "__type__": qualified_name(type(value)), + "__type__": type_name, **{ field.name: self.encode(getattr(value, field.name, None)) for field in dataclasses.fields(value) + if field.name not in omitted_fields }, } if isinstance(value, Mapping): diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index b974d7146c..dd19d64fca 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -20,6 +20,7 @@ NativeStage.DECLARATION_SCOPES: NativeStageTier.PREVIEW, NativeStage.MODEL_CONFIG: NativeStageTier.PREVIEW, NativeStage.REFERENCE_EXTRACTION: NativeStageTier.PREVIEW, + NativeStage.DECLARATION_FILES: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), @@ -50,7 +51,8 @@ STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] = { "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( {"shared_analyses"} - ) + ), + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( {"sqlbuild.compiler.compile.models:CompactLineageFacts"} diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 56c5ac01cb..3881a1871d 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -18,6 +18,7 @@ class NativeStage(StrEnum): DECLARATION_SCOPES = "declaration_scopes" MODEL_CONFIG = "model_config" REFERENCE_EXTRACTION = "reference_extraction" + DECLARATION_FILES = "declaration_files" class NativeStageTier(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 62400192f1..9c9cfdb0af 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -2,6 +2,7 @@ import sys import unicodedata +from collections.abc import Callable from dataclasses import dataclass, field @@ -162,3 +163,69 @@ class CollidingNamesTestCase: files: tuple[tuple[str, bytes], ...] expected_enums: tuple[tuple[str, tuple[str, ...]], ...] expected_models: tuple[str, ...] = ("models/orders.sql",) + + +@dataclass(frozen=True) +class GeneratedDeclarationFileTestCase: + """Seeded declaration files of one kind discovered by the Python and native-preview engines.""" + + description: str + kind: str + relative_path: str + case_count: int + expected_minimum_parsed: int + expected_minimum_failed: int + expected_maximum_deferred: int + expected_mismatches: tuple[int, ...] = () + + +@dataclass(frozen=True) +class DeclarationFilesParityTestCase: + """A project whose declaration files discovery reads under the Python and preview engines.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_failure_type: str = "NoneType" + expected_message_fragment: str = "" + + +@dataclass(frozen=True) +class DeclarationMismatchTestCase: + """A declaration file the Python parser is patched to accept, which native discovery rejects.""" + + description: str + relative_path: str + contents: bytes + patched_parser: str + patched: Callable[..., object] + expected_error_fragment: str + + +@dataclass(frozen=True) +class NativeSessionTestCase: + """Whether a discovery pass under one engine keeps its native declaration session.""" + + description: str + engine: str + expected_session: bool + + +@dataclass(frozen=True) +class TolerantDeclarationFilesTestCase: + """A project whose broken declaration files tolerant scope discovery reports as faults.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_resource_faults: int + expected_declaration_faults: int + + +@dataclass(frozen=True) +class DeclarationReuseTestCase: + """A preview compile followed by an edit that only touches one model file.""" + + description: str + files: tuple[tuple[str, bytes], ...] + edited_path: str + edited_contents: bytes + expected_reused_session: object = None diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 860eca1d94..f3d8f9c92a 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -5,12 +5,15 @@ import datetime import json import math +import os import random import struct import tomllib +from collections import Counter from collections.abc import Callable from contextlib import suppress from itertools import compress +from operator import not_ from pathlib import Path from typing import Any, cast @@ -21,11 +24,27 @@ from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( build_tolerant_scope_discovery, ) -from sqlbuild.compiler.discovery._helpers.native.payloads import native_payload_error +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_audit_files, + discover_constant_files, + discover_enum_files, + discover_model_schema_files, + discover_sql_function_files, + discover_sql_hook_files, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_payload_error, + native_text_runtime, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, ) +from sqlbuild.compiler.discovery.constants import ( + SQL_AUDIT_HEADER_KEYS, + SQL_FUNCTION_HEADER_KEYS, + SQL_HOOK_HEADER_KEYS, +) from sqlbuild.compiler.discovery.main._model_description_inputs import ( discover_model_description_inputs, ) @@ -33,11 +52,15 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredEnumFile, DiscoveredProjectInputs, + DiscoveryFileFault, TolerantScopeDiscovery, ) from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + GeneratedDeclarationFileTestCase, +) from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] @@ -664,7 +687,9 @@ def write_project(*, project_dir: Path, files: tuple[tuple[str, bytes], ...]) -> _ = path.write_bytes(data) -def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch) -> object: +def stage_outcome( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[object, ...]: """Return the rendered discovery stage under `engine` with its failure type and message.""" monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) @@ -793,3 +818,469 @@ def _declared_enum_names(enum_file: DiscoveredEnumFile) -> tuple[str, tuple[str, enum_file.relative_path.as_posix(), tuple(declaration.name for declaration in enum_file.declarations), ) + + +DECLARATION_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { + "enum": discover_enum_files, + "constant": discover_constant_files, + "model_schema": discover_model_schema_files, + "audit": discover_audit_files, + "sql_hook": discover_sql_hook_files, + "sql_function": discover_sql_function_files, +} + + +def declaration_files_outcome( + *, project_dir: Path, kind: str, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[object, ...]: + """Return the rendered `kind` collection under `engine`, or its failure type, text and help.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + capture: FailureCapture = FailureCapture() + with capture: + return ( + "parsed", + render_stage_capture(DECLARATION_DISCOVERERS[kind](project_dir=project_dir)), + ) + failure: BaseException | None = capture.failure + return (type(failure).__name__, str(failure), getattr(failure, "help", None)) + + +def native_declaration_tag(*, file_path: Path, kind: str) -> str: + """Return whether native parsing parses, fails or defers one file's contents.""" + + payload: tuple[object, ...] = _native.parse_declaration_contents( + { + "kind": kind, + "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), + "audit_keys": sorted(SQL_AUDIT_HEADER_KEYS), + "hook_keys": sorted(SQL_HOOK_HEADER_KEYS), + **native_text_runtime(), + }, + (str(file_path), file_path.stem), + file_path.read_text(encoding="utf-8"), + ) + return str(payload[0]) + + +def tolerant_declaration_outcome( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[object, tuple[DiscoveryFileFault, ...], tuple[DiscoveryFileFault, ...]]: + """Return tolerant scope discovery's declaration collections and faults under `engine`.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + discovery: TolerantScopeDiscovery = build_tolerant_scope_discovery(project_dir=project_dir) + return ( + render_stage_capture(discovery.discovered_inputs), + discovery.resource_faults, + discovery.declaration_faults, + ) + + +# Seeded generators of declaration file contents: each builds a valid file, then often applies one +# authored mistake or awkward form, so the corpus covers every failure path and its position. +VALID_IDENTIFIERS: tuple[str, ...] = ("order_status", "orders", "cap", "a", "a1", "tier_2") +ODD_IDENTIFIERS: tuple[str, ...] = ( + "OrderStatus", "HTTPCode", "x2Y", "__dunder", "_private", "order_", "9lives", "order-status", + "caf\u00e9", "PLACED", "_", "order__status", "'quoted_name'", "1", "true", "null", "1.5", +) # fmt: skip +ENUM_MEMBERS: tuple[str, ...] = ("PLACED", "SHIPPED", "RETURNED", "A", "B_2") +ODD_ENUM_MEMBERS: tuple[str, ...] = ("placed", "Placed", "1", "'quoted'", "true", "x\u00e9", "_") +HEADER_SCALARS: tuple[str, ...] = ( + "1", "-2", "+3", "0", "007", "1.5", ".5", "5.", "true", "false", "null", "'text'", + '"quoted"', "''", "' '", "word", "PLACED", "\u0661\u0662", "x\u00e9", "1e5", "'it''s'", + "'a\\'b'", "99999999999999999999999", +) # fmt: skip +HEADER_VALUES: tuple[str, ...] = ( + "1", "'text'", "[1, 2]", "{'a', 'b'}", "1.5", "true", "'2024-01-01'", "[]", "{}", "null", + "['a', 'b']", "(x 1)", "[1, 'a']", "[[1], [2]]", "-7", +) # fmt: skip +DECLARATION_SPACES: tuple[str, ...] = ( + " ", + "", + " ", + "\n", + "\n ", + " ", + "\n", + "\t", + "\u00a0", + "\r\n", +) * 6 + (" -- note\n", " /* c */ ", "\u2028", "\x1c", "\u3000") +SQL_BODIES: tuple[str, ...] = ( + "SELECT 1", + '\n SELECT *\n FROM __ref("@model")\n WHERE NOT (@expression)\n', + "SELECT ';' AS x", + "WITH t AS (SELECT 1) SELECT * FROM t", + "SELECT 'it''s' AS x, \"q\" FROM `t`", + "\tSELECT\n\t\t1\n", + "seLECT 1 -- trailing\n", + "SELECT 1 /* ( */", +) +ODD_SQL_BODIES: tuple[str, ...] = ( + "", " ", "-- only a comment", "SELECT 1; SELECT 2", "SELECT (1", "MEASURE (SELECT 1);", + "\u017felect 1", "EVIDENCE (SELECT 1);", "SELECT 1;", "SELECT ')' AS x", +) # fmt: skip +AUDIT_OPTIONAL_KEYS: tuple[str, ...] = ( + "severity", "run_scope", "always_run", "sample_count", "sample_unit", "thresholds", + "minimum_samples", "sql_analysis", "unknown_key", "nme", +) # fmt: skip +FUNCTION_HEADER_KEYS: tuple[str, ...] = ( + "arguments", "returns", "database", "schema", "tags", "description", "argumnts", "body", +) # fmt: skip +VALID_FUNCTION_HEADER: str = 'arguments (amount DECIMAL), returns DECIMAL, description "Doubles"' +ODD_WEIGHTS: tuple[int, int] = (85, 15) + + +def _space(rng: random.Random) -> str: + return rng.choice(DECLARATION_SPACES) + + +def _pick(rng: random.Random, valid: tuple[str, ...], odd: tuple[str, ...]) -> str: + """Mostly a valid form, sometimes an odd one.""" + + return rng.choice(rng.choices((valid, odd), weights=ODD_WEIGHTS)[0]) + + +def _maybe(rng: random.Random, chance: float, entry: str) -> list[str]: + """`[entry]` with probability `chance`, else no entry.""" + + return [entry] * (rng.random() < chance) + + +def _scalar_value(rng: random.Random, depth: int) -> str: + _ = depth + return rng.choice((*HEADER_VALUES, *HEADER_SCALARS)) + + +def _list_value(rng: random.Random, depth: int) -> str: + return "[" + ", ".join(_value(rng, depth + 1) for _ in range(rng.randint(0, 3))) + "]" + + +def _set_value(rng: random.Random, depth: int) -> str: + return "{" + ", ".join(_value(rng, depth + 1) for _ in range(rng.randint(0, 3))) + "}" + + +def _map_value(rng: random.Random, depth: int) -> str: + entries: str = ", ".join( + f"key_{index} {_value(rng, depth + 1)}" for index in range(rng.randint(0, 3)) + ) + return f"({entries})" + + +def _wrapped_value(rng: random.Random, depth: int) -> str: + keys: list[str] = rng.sample(("value", "type", "render_as", "extra"), k=rng.randint(0, 3)) + return "constant(" + ", ".join(f"{key} {_value(rng, depth + 1)}" for key in keys) + ")" + + +VALUE_FORMS: tuple[Callable[[random.Random, int], str], ...] = ( + _scalar_value, _list_value, _set_value, _map_value, _wrapped_value, +) # fmt: skip +VALUE_FORM_WEIGHTS: dict[int, tuple[int, ...]] = { + 0: (50, 15, 10, 10, 15), + 1: (50, 15, 10, 10, 15), + 2: (100, 0, 0, 0, 0), +} + + +def _value(rng: random.Random, depth: int = 0) -> str: + form: Callable[[random.Random, int], str] = rng.choices( + VALUE_FORMS, weights=VALUE_FORM_WEIGHTS[min(depth, 2)] + )[0] + return form(rng, depth) + + +def _shuffled(rng: random.Random, entries: list[str]) -> str: + """Entries in authored order, or shuffled: the parsers must not depend on key order.""" + + return ", ".join(rng.choice((entries, rng.sample(entries, k=len(entries))))) + + +def _declaration_count(rng: random.Random) -> int: + return rng.choice((1, 1, 1, 1, 2, 2, 3)) + + +def _statement(rng: random.Random, keyword: str, header: str) -> list[str]: + return [_space(rng), keyword, _space(rng), f"({header})", _space(rng), ";"] + + +STATEMENT_MISTAKES: tuple[Callable[[list[str]], None], ...] = ( + lambda statement: statement.__setitem__(5, ""), + lambda statement: statement.__setitem__(5, ";;"), + lambda statement: statement.__setitem__(5, ","), + lambda statement: statement.__setitem__(1, statement[1].lower()), + lambda statement: statement.__setitem__(1, "SCHEMA"), + lambda statement: statement.__setitem__(3, statement[3][:-1]), + lambda statement: None, +) +FILE_MISTAKES: tuple[Callable[[str], str], ...] = ( + lambda text: "-- header\n" + text, + lambda text: text + "SELECT 1", + lambda text: text[:0] + "\n", + lambda text: text, +) + + +def _statements(rng: random.Random, keyword: str, headers: list[str]) -> str: + """Join `KEYWORD (header);` statements with at most one statement and one file mistake.""" + + statements: list[list[str]] = [_statement(rng, keyword, header) for header in headers] + rng.choices(STATEMENT_MISTAKES, weights=(4, 3, 3, 3, 3, 3, 81))[0](rng.choice(statements)) + text: str = "".join("".join(statement) for statement in statements) + _space(rng) + return rng.choices(FILE_MISTAKES, weights=(4, 4, 4, 88))[0](text) + + +def _identity(rng: random.Random, index: int) -> str: + return _pick(rng, (f"{rng.choice(VALID_IDENTIFIERS)}_{index}",), ODD_IDENTIFIERS) + + +def _shorthand_members(rng: random.Random, names: list[str]) -> str: + return "[" + ", ".join(_pick(rng, (name,), ODD_ENUM_MEMBERS) for name in names) + "]" + + +def _explicit_members(rng: random.Random, names: list[str]) -> str: + values: tuple[str, ...] = rng.choice((("'x'", "'y'", "'z'"), ("1", "2", "-3"))) + entries: str = ", ".join( + f"{_pick(rng, (name,), ODD_ENUM_MEMBERS)} {_pick(rng, values, HEADER_SCALARS)}" + for name in names + ) + return f"({entries})" + + +def _odd_members(rng: random.Random, names: list[str]) -> str: + _ = names + return rng.choice(("[]", "()", "1", "'x'", "{A, B}", "null", _value(rng))) + + +ENUM_MEMBER_FORMS: tuple[Callable[[random.Random, list[str]], str], ...] = ( + _shorthand_members, _explicit_members, _odd_members, +) # fmt: skip + + +def _enum_header(rng: random.Random, index: int) -> str: + names: list[str] = rng.sample(ENUM_MEMBERS, k=rng.randint(1, 3)) + names.extend(names[:1] * (rng.random() < 0.1)) + members: str = rng.choices(ENUM_MEMBER_FORMS, weights=(50, 42, 8))[0](rng, names) + entries: list[str] = [ + *_maybe(rng, 0.97, f"name {_identity(rng, index)}"), + *_maybe(rng, 0.97, f"members {members}"), + *_maybe(rng, 0.05, "extra 1"), + ] + return _shuffled(rng, entries) + + +def enum_file(*, rng: random.Random) -> str: + """Return enum file contents, mostly valid, some with one authored mistake.""" + + headers: list[str] = [_enum_header(rng, index) for index in range(_declaration_count(rng))] + return _statements(rng, "ENUM", headers) + + +def _constant_header(rng: random.Random, index: int) -> str: + value: str = rng.choice((rng.choice(HEADER_VALUES), _value(rng))) + types: tuple[str, ...] = ("INTEGER", "VARCHAR", "BOOLEAN", "decimal", "1", "true", "''") + renderings: tuple[str, ...] = ("value_list", "array", "list", "1", "''") + entries: list[str] = [ + f"name {_identity(rng, index)}", + *_maybe(rng, 0.95, f"value {value}"), + *_maybe(rng, 0.25, f"type {rng.choice(types)}"), + *_maybe(rng, 0.2, f"render_as {rng.choice(renderings)}"), + *_maybe(rng, 0.05, "unknown 1"), + ] + return _shuffled(rng, entries) + + +def constant_file(*, rng: random.Random) -> str: + """Return constant file contents with wrapped, typed and malformed constants.""" + + headers: list[str] = [_constant_header(rng, index) for index in range(_declaration_count(rng))] + return _statements(rng, "CONSTANT", headers) + + +def _column(rng: random.Random, index: int) -> str: + audits: str = _pick(rng, ("not_null", "unique", "accepted_values(values [1])"), ("1",)) + metadata: list[str] = [ + *_maybe(rng, 0.8, f"type {_pick(rng, ('INTEGER', 'VARCHAR', 'DECIMAL(10, 2)'), ('1',))}"), + *_maybe(rng, 0.3, f"nullable {_pick(rng, ('true', 'false'), ('1', 'maybe'))}"), + *_maybe(rng, 0.3, f"description {_pick(rng, ('"Order id"',), ('""', '1'))}"), + *_maybe(rng, 0.3, f"audits [{audits}]"), + *_maybe(rng, 0.03, "migrate_from old"), + ] + name: str = _pick(rng, (f"column_{index}",), ("ORDER_ID", '"spaced name"', "column_0")) + return f"{name} ({', '.join(metadata)})" + + +def _schema_header(rng: random.Random, index: int) -> str: + columns: str = ", ".join(_column(rng, column) for column in range(rng.randint(0, 3))) + description: str = _pick(rng, ('"Orders"', "word"), ('""', '" "', "1", "null")) + extends: str = _pick(rng, ("base_shape", '"base"'), ('"not ident"', "1", "null")) + entries: list[str] = [ + f"name {_identity(rng, index)}", + *_maybe(rng, 0.5, f"description {description}"), + *_maybe(rng, 0.3, f"extends {extends}"), + *_maybe( + rng, 0.95, rng.choices((f"columns ({columns})", f"columns {_value(rng)}"), (95, 5))[0] + ), + *_maybe(rng, 0.05, "extra 1"), + ] + return _shuffled(rng, entries) + + +def schema_file(*, rng: random.Random) -> str: + """Return reusable model schema file contents.""" + + headers: list[str] = [_schema_header(rng, index) for index in range(_declaration_count(rng))] + return _statements(rng, "SCHEMA", headers) + + +def _violation_audit(rng: random.Random, index: int, named: bool) -> tuple[str, str]: + entries: list[str] = [ + *_maybe(rng, named, f"name {_pick(rng, (f'check_{index}',), ('check_0', '""', '1'))}"), + *_maybe(rng, 0.05, f"evaluation {rng.choice(('violations', 'count', '1'))}"), + *(f"{key} {rng.choice(('x', 'true', '1', '""'))}" for key in _optional_keys(rng)), + ] + return _shuffled(rng, entries), _pick(rng, SQL_BODIES, ODD_SQL_BODIES) + + +def _measurement_audit(rng: random.Random, index: int, named: bool) -> tuple[str, str]: + entries: list[str] = [ + *_maybe(rng, named, f"name {_pick(rng, (f'check_{index}',), ('check_0', '""', '1'))}"), + "evaluation measurement", + *_maybe(rng, 0.95, f"value {_pick(rng, ('total', '"amount"'), ('1', '""'))}"), + *(f"{key} {rng.choice(('x', 'true', '1', '""'))}" for key in _optional_keys(rng)), + ] + measure: str = f"MEASURE{_space(rng)}({_pick(rng, SQL_BODIES, ODD_SQL_BODIES)})" + evidence: str = ( + f"{_pick(rng, ('EVIDENCE',), ('evidence', 'MEASURE', 'EVIDENCES'))} " + f"({_pick(rng, SQL_BODIES, ODD_SQL_BODIES)});" + ) + parts: list[str] = [ + *_maybe(rng, 0.95, measure + rng.choices((";", ""), weights=(95, 5))[0]), + *_maybe(rng, 0.4, evidence), + *_maybe(rng, 0.05, "SELECT 1"), + ] + return _shuffled(rng, entries), "\n".join(parts) + + +def _optional_keys(rng: random.Random) -> list[str]: + return rng.sample(AUDIT_OPTIONAL_KEYS, k=rng.choices((0, 1, 2), weights=(70, 20, 10))[0]) + + +AUDIT_FORMS: tuple[Callable[[random.Random, int, bool], tuple[str, str]], ...] = ( + _violation_audit, _measurement_audit, +) # fmt: skip + + +def audit_file(*, rng: random.Random) -> str: + """Return SQL audit file contents with violation and measurement blocks.""" + + count: int = _declaration_count(rng) + named: bool = count > 1 or rng.random() < 0.5 + blocks: list[str] = [] + for index in range(count): + header, body = rng.choices(AUDIT_FORMS, weights=(65, 35))[0](rng, index, named) + keyword: str = _pick(rng, ("AUDIT", "audit", "Audit"), ("AUD\u0131T", "AUDITS")) + close: str = _pick(rng, (");", ") ;", ");\n"), (")", "); -- c")) + blocks.append(f"{keyword}{_space(rng)}({header}{close}\n{body}\n") + leading: str = _pick(rng, ("", "\n\n", " "), ("-- lead\n", "SELECT 1;\n")) + return leading + "".join(blocks) + + +def hook_file(*, rng: random.Random) -> str: + """Return SQL hook file contents.""" + + header: str = _pick( + rng, + ("", 'description "Refresh"', "description 'it''s (fine)'", "description `q`"), + ( + 'description ""', + "description 1", + 'descripton "typo"', + 'description "a (b" , other 1', + "description 'unterminated", + "description 'a\\'b'", + ), + ) + keyword: str = _pick(rng, ("HOOK",), ("hook", "HOOKS", "-- c\nHOOK")) + close: str = _pick(rng, (");", ") ;", ");\n"), (")", "));")) + body: str = _pick(rng, SQL_BODIES, ODD_SQL_BODIES) + return f"{_space(rng)}{keyword}{_space(rng)}({header}{close}{body}" + + +def function_file(*, rng: random.Random) -> str: + """Return SQL function file contents.""" + + keys: list[str] = rng.sample(FUNCTION_HEADER_KEYS, k=rng.randint(0, 3)) + generated: str = ", ".join(f"{key} {_value(rng)}" for key in keys) + entries: str = rng.choices((VALID_FUNCTION_HEADER, generated), weights=(70, 30))[0] + keyword: str = _pick(rng, ("FUNCTION",), ("function", "FUNC")) + close: str = _pick(rng, (");", ") ;", ");\n"), (")",)) + body: str = _pick(rng, SQL_BODIES, ODD_SQL_BODIES) + return f"{_space(rng)}{keyword}{_space(rng)}({entries}{close}{body}" + + +DECLARATION_GENERATORS: dict[str, Callable[..., str]] = { + "enum": enum_file, + "constant": constant_file, + "model_schema": schema_file, + "audit": audit_file, + "sql_hook": hook_file, + "sql_function": function_file, +} + + +def generated_declaration_outcomes( + *, + project_root: Path, + test_case: GeneratedDeclarationFileTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[tuple[int, ...], Counter[str]]: + """Discover each generated file under both engines: the mismatching seeds and native tags.""" + + agreements: list[bool] = [] + tags: Counter[str] = Counter() + for seed in range(test_case.case_count): + project_dir: Path = project_root / str(seed) + contents: str = DECLARATION_GENERATORS[test_case.kind]( + rng=random.Random(f"{test_case.kind}-{seed}") + ) + write_project( + project_dir=project_dir, files=((test_case.relative_path, contents.encode("utf-8")),) + ) + python: tuple[object, ...] = declaration_files_outcome( + project_dir=project_dir, kind=test_case.kind, engine="python", monkeypatch=monkeypatch + ) + native: tuple[object, ...] = declaration_files_outcome( + project_dir=project_dir, + kind=test_case.kind, + engine="native-preview", + monkeypatch=monkeypatch, + ) + tags[ + native_declaration_tag( + file_path=project_dir / test_case.relative_path, kind=test_case.kind + ) + ] += 1 + agreements.append(native == python) + return tuple(compress(range(test_case.case_count), map(not_, agreements))), tags + + +def accept_any_declarations(**_: object) -> tuple[object, ...]: + """A patched Python parser that accepts every file.""" + + return () + + +def reject_any_contents(**_: object) -> tuple[object, ...]: + """A patched Python parser that rejects every file with an unrelated error.""" + + raise ValueError("different") + + +def write_undecodable_hook(*, project_dir: Path, contents: bytes) -> None: + """Write a project whose only SQL hook has a file name that is not valid UTF-8.""" + + write_project(project_dir=project_dir, files=()) + hooks: Path = project_dir / "hooks" / "sql" + hooks.mkdir(parents=True) + _ = Path(os.fsdecode(bytes(hooks) + b"/refresh_\xff.sql")).write_bytes(contents) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py new file mode 100644 index 0000000000..a6e57f3960 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -0,0 +1,472 @@ +"""Engine parity for the enum, constant, schema, audit, hook, function, seed and macro files.""" + +from __future__ import annotations + +import sys +from collections import Counter +from pathlib import Path + +import pytest + +from sqlbuild import _native +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import RENDER_REUSE_DECLARATIONS_PREFIX +from sqlbuild.compiler.compile.models import RenderReuseState +from sqlbuild.compiler.discovery._helpers.filesystem import core +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + DeclarationFilesParityTestCase, + DeclarationMismatchTestCase, + DeclarationReuseTestCase, + GeneratedDeclarationFileTestCase, + NativeSessionTestCase, + TolerantDeclarationFilesTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + accept_any_declarations, + generated_declaration_outcomes, + reject_any_contents, + stage_outcome, + tolerant_declaration_outcome, + write_project, + write_undecodable_hook, +) + +_PREVIEW: str = "native-preview" +_MODEL: bytes = b'MODEL (description "Orders");\nSELECT 1 AS order_id' +_MACRO: bytes = b"def cents(value):\n return f'{value} * 100'\n" +_ENUMS: bytes = ( + b"ENUM (name order_status, members [PLACED, SHIPPED]);\n" + b"ENUM (\n name priority,\n members (LOW 1, HIGH 2),\n);\n" +) +_CONSTANTS: bytes = ( + b"CONSTANT (name cap, value 3);\n" + b"CONSTANT (name regions, value ['eu', 'us'], render_as value_list);\n" + b"CONSTANT (name rate, value constant(value '1.50', type decimal));\n" +) +_SCHEMA: bytes = ( + b'SCHEMA (\r\n name order_shape,\r\n description "Order shape",\r\n' + b" columns (\r\n id (type INTEGER, audits [not_null]),\r\n" + b' total (type DECIMAL(10, 2), description "Total"),\r\n ),\r\n);\r\n' + b"SCHEMA (name wide_order, extends order_shape, columns (note (type VARCHAR)));\n" +) +_AUDIT: bytes = b'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' +_MEASUREMENTS: bytes = ( + b"AUDIT (name row_count, evaluation measurement, value total, sample_unit 'rows');\n" + b'MEASURE (SELECT count(*) AS total FROM __ref("@model"));\n' + b'EVIDENCE (SELECT * FROM __ref("@model") LIMIT 5);\n' + b'audit (name no_negative, severity warn);\n-- body\nSELECT * FROM __ref("@model") WHERE x < 0\n' +) +_HOOK: bytes = b'HOOK (\n description "Record a refresh"\n);\n\n SELECT 1 AS refreshed\n' +_FUNCTION: bytes = ( + b'FUNCTION (arguments (amount DECIMAL), returns DECIMAL, description "Doubles");\n' + b"SELECT amount * 2\n" +) +_EVERY_KIND: tuple[tuple[str, bytes], ...] = ( + ("models/marts/orders.sql", _MODEL), + ("macros/money.py", _MACRO), + ("macros/__init__.py", b""), + ("models/marts/_macros/local.py", _MACRO), + ("enums/status.sql", _ENUMS), + ("models/marts/_enums/tier.sql", b"ENUM (name tier, members [GOLD]);\n"), + ("models/marts/_sqlbuild/constants/limits.sql", _CONSTANTS), + ("constants/global.sql", b"CONSTANT (name ratio, value 1.5);\n"), + ("models/marts/_sqlbuild/_audits/generic/is_true.sql", _AUDIT), + ("models/marts/_sqlbuild/audits/singular/orders_ok.sql", _AUDIT), + ("models/marts/_sqlbuild/_schemas/order_shape.sql", _SCHEMA), + ("models/marts/_sqlbuild/hooks/sql/record_marts_refresh.sql", _HOOK), + ("audits/generic/is_true.sql", _AUDIT), + ("audits/generic/nested/measured.sql", _MEASUREMENTS), + ("audits/singular/orders_ok.sql", _AUDIT), + ("schemas/order_shape.sql", _SCHEMA), + ("hooks/sql/record_refresh.sql", _HOOK), + ("hooks/sql/_private.sql", b"not a hook"), + ("functions/sql/double_amount.sql", _FUNCTION), + ("functions/sql/finance/net_amount.sql", _FUNCTION), + ("functions/sql/_macros/helpers.py", _MACRO), + ("seeds/channels.csv", b"id\n1\n"), + ( + "seeds/seeds.yml", + b"seeds:\n - name: channels\n description: Sales channels.\n" + b" columns:\n - name: id\n type: INTEGER\n" + b" - name: stores\n description: Stores.\n" + b" columns:\n - name: id\n type: INTEGER\n", + ), + ("seeds/regional/stores.csv", b"id\n2\n"), + ("seeds/readme.md", b"seeds"), + ("seeds/archive.csv/notes.txt", b"a directory named like a seed"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedDeclarationFileTestCase( + description="generated enum files", + kind="enum", + relative_path="enums/generated.sql", + case_count=150, + expected_minimum_parsed=20, + expected_minimum_failed=60, + expected_maximum_deferred=8, + ), + GeneratedDeclarationFileTestCase( + description="generated constant files", + kind="constant", + relative_path="models/marts/_constants/generated.sql", + case_count=150, + expected_minimum_parsed=60, + expected_minimum_failed=30, + expected_maximum_deferred=8, + ), + GeneratedDeclarationFileTestCase( + description="generated reusable schema files", + kind="model_schema", + relative_path="schemas/generated.sql", + case_count=150, + expected_minimum_parsed=50, + expected_minimum_failed=30, + expected_maximum_deferred=8, + ), + GeneratedDeclarationFileTestCase( + description="generated audit files", + kind="audit", + relative_path="models/marts/_sqlbuild/audits/generic/generated.sql", + case_count=150, + expected_minimum_parsed=30, + expected_minimum_failed=60, + expected_maximum_deferred=4, + ), + GeneratedDeclarationFileTestCase( + description="generated SQL hook files", + kind="sql_hook", + relative_path="hooks/sql/generated.sql", + case_count=150, + expected_minimum_parsed=40, + expected_minimum_failed=40, + expected_maximum_deferred=4, + ), + GeneratedDeclarationFileTestCase( + description="generated SQL function files", + kind="sql_function", + relative_path="functions/sql/generated.sql", + case_count=150, + expected_minimum_parsed=50, + expected_minimum_failed=30, + expected_maximum_deferred=4, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_declaration_files_when_discovering_then_engines_agree( + test_case: GeneratedDeclarationFileTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + mismatches: tuple[int, ...] + tags: Counter[str] + + mismatches, tags = generated_declaration_outcomes( + project_root=tmp_path, test_case=test_case, monkeypatch=monkeypatch + ) + + assert mismatches == test_case.expected_mismatches + assert tags["ok"] >= test_case.expected_minimum_parsed, tags + assert tags["error"] >= test_case.expected_minimum_failed, tags + assert tags["defer"] <= test_case.expected_maximum_deferred, tags + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationFilesParityTestCase( + description="global, inherited, local and grouped declarations of every kind", + files=_EVERY_KIND, + ), + DeclarationFilesParityTestCase( + description="the first failing enum file in path order is reported", + files=( + ("enums/a_status.sql", b"ENUM (name a_status, members [placed]);\n"), + ("enums/b_status.sql", b"ENUM (name b_status);\n"), + ), + expected_failure_type="DeclarationParseError", + expected_message_fragment="member identifiers must be uppercase: 'placed'", + ), + DeclarationFilesParityTestCase( + description="an earlier constant's value fails before a later constant's name", + files=( + ( + "constants/limits.sql", + b"CONSTANT (name launch, value 'soon', type DATE);\n" + b"CONSTANT (name Bad, value 1);\n", + ), + ), + expected_failure_type="DeclarationParseError", + expected_message_fragment="constant 'launch'", + ), + DeclarationFilesParityTestCase( + description="an earlier schema's columns fail before a later schema's keys", + files=( + ( + "schemas/shapes.sql", + b"SCHEMA (name first_shape, columns (id (nullable maybe)));\n" + b"SCHEMA (name second_shape, extra 1, columns (id (type INT)));\n", + ), + ), + expected_failure_type="DeclarationParseError", + expected_message_fragment="first_shape", + ), + DeclarationFilesParityTestCase( + description="a non-canonical declaration name fails with the identity help", + files=(("enums/status.sql", b"ENUM (name OrderStatus, members [PLACED]);\n"),), + expected_failure_type="ResourceIdentityError", + expected_message_fragment="use snake_case 'order_status'", + ), + DeclarationFilesParityTestCase( + description="a nested declaration root fails before any file is parsed", + files=( + ("models/marts/_enums/macros/money.py", _MACRO), + ("enums/status.sql", b"broken"), + ), + expected_failure_type="DeclarationParseError", + expected_message_fragment="is nested inside another declaration tree", + ), + DeclarationFilesParityTestCase( + description="an unreadable audit file fails with Python's decode error", + files=(("audits/generic/is_true.sql", b"AUDIT ();\nSELECT '\xff'\n"),), + expected_failure_type="UnicodeDecodeError", + expected_message_fragment="invalid start byte", + ), + DeclarationFilesParityTestCase( + description="an unreadable macro file fails with Python's decode error", + files=(("macros/money.py", b"# \xc3"),), + expected_failure_type="UnicodeDecodeError", + expected_message_fragment="unexpected end of data", + ), + DeclarationFilesParityTestCase( + description="a hook with an unsupported key fails with the key suggestion", + files=(("hooks/sql/refresh.sql", b'HOOK (descripton "Refresh");\nSELECT 1\n'),), + expected_failure_type="SqlHookParseError", + expected_message_fragment="has unsupported keys: descripton", + ), + DeclarationFilesParityTestCase( + description="a measurement audit without MEASURE fails", + files=( + ( + "audits/generic/measured.sql", + b"AUDIT (evaluation measurement, value total);\nSELECT 1\n", + ), + ), + expected_failure_type="SqlAuditParseError", + expected_message_fragment="must define exactly one MEASURE(...) block", + ), + DeclarationFilesParityTestCase( + description="a SQL function without a body fails", + files=(("functions/sql/empty.sql", b"FUNCTION (returns INT);\n \n"),), + expected_failure_type="ModelSqlParseError", + expected_message_fragment="must contain SQL after FUNCTION(...)", + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_files_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: DeclarationFilesParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: tuple[object, ...] = stage_outcome( + project_dir=tmp_path, engine="python", monkeypatch=monkeypatch + ) + + native: tuple[object, ...] = stage_outcome( + project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch + ) + + assert native == python + assert native[1] == test_case.expected_failure_type + assert test_case.expected_message_fragment in str(native[2]) + + +@pytest.mark.skipif(sys.platform != "linux", reason="needs file names that are not UTF-8") +@pytest.mark.parametrize( + "test_case", + [ + DeclarationFilesParityTestCase( + description="a hook named with invalid UTF-8 is read by Python under both engines", + files=(("hooks/sql/refresh.sql", _HOOK),), + expected_failure_type="ResourceIdentityError", + expected_message_fragment="refresh_", + ) + ], + ids=lambda case: case.description, +) +def test_given_hook_named_with_invalid_utf8_when_discovering_then_engines_agree( + test_case: DeclarationFilesParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_undecodable_hook(project_dir=tmp_path, contents=test_case.files[0][1]) + python: tuple[object, ...] = stage_outcome( + project_dir=tmp_path, engine="python", monkeypatch=monkeypatch + ) + + native: tuple[object, ...] = stage_outcome( + project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch + ) + + assert native == python + assert native[1] == test_case.expected_failure_type + assert test_case.expected_message_fragment in str(native[2]) + + +@pytest.mark.parametrize( + "test_case", + [ + TolerantDeclarationFilesTestCase( + description="broken files of every kind become the same faults", + files=( + *_EVERY_KIND, + ("enums/broken.sql", b"ENUM (name broken);\n"), + ("constants/broken.sql", b"CONSTANT (name Broken, value 1);\n"), + ("schemas/broken.sql", b"SCHEMA (name broken_shape);\n"), + ("audits/generic/broken.sql", b"SELECT 1\n"), + ("hooks/sql/broken.sql", b"HOOK ();\n"), + ("functions/sql/broken.sql", b"SELECT 1\n"), + ), + expected_resource_faults=3, + expected_declaration_faults=3, + ) + ], + ids=lambda case: case.description, +) +def test_given_broken_declaration_files_when_discovering_tolerantly_then_faults_match( + test_case: TolerantDeclarationFilesTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: tuple[object, tuple[DiscoveryFileFault, ...], tuple[DiscoveryFileFault, ...]] = ( + tolerant_declaration_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + ) + + native: tuple[object, tuple[DiscoveryFileFault, ...], tuple[DiscoveryFileFault, ...]] = ( + tolerant_declaration_outcome(project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch) + ) + + assert native == python + assert len(native[1]) == test_case.expected_resource_faults + assert len(native[2]) == test_case.expected_declaration_faults + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationMismatchTestCase( + description="Python accepting a file native rejects is a mismatch", + relative_path="enums/status.sql", + contents=b"ENUM (name order_status);\n", + patched_parser="parse_enum_declaration_file", + patched=accept_any_declarations, + expected_error_fragment="native declaration_files raised DeclarationParseError", + ), + DeclarationMismatchTestCase( + description="Python failing differently from native is a mismatch", + relative_path="audits/generic/is_true.sql", + contents=b"SELECT 1\n", + patched_parser="parse_sql_audit_file", + patched=reject_any_contents, + expected_error_fragment="but the Python compiler raised ValueError('different')", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_disagrees_with_a_native_failure_when_discovering_then_mismatch_raises( + test_case: DeclarationMismatchTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=((test_case.relative_path, test_case.contents),)) + monkeypatch.setattr(core, test_case.patched_parser, test_case.patched) + outcome: tuple[object, ...] = stage_outcome( + project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch + ) + + assert outcome[1] == "NativeStageMismatchError" + assert test_case.expected_error_fragment in str(outcome[2]) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSessionTestCase( + description="the preview engine keeps its declaration session", + engine=_PREVIEW, + expected_session=True, + ), + NativeSessionTestCase( + description="the shipped engine reads declaration files in Python", + engine="native", + expected_session=False, + ), + NativeSessionTestCase( + description="the Python engine reads declaration files in Python", + engine="python", + expected_session=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_discovering_then_native_session_is_kept_only_for_preview( + test_case: NativeSessionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=_EVERY_KIND) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine) + + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + + assert ( + isinstance(discovered.native_session, _native.NativeDiscoverySession) + is test_case.expected_session + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationReuseTestCase( + description="a model edit reuses every stored declaration file", + files=_EVERY_KIND, + edited_path="models/marts/orders.sql", + edited_contents=_MODEL + b" -- edited\n", + ) + ], + ids=lambda case: case.description, +) +def test_given_preview_compile_when_only_a_model_changes_then_declaration_files_are_reused( + test_case: DeclarationReuseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, _PREVIEW) + first_session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=None, changed_paths=None + ) + first: DiscoveredProjectInputs = discover_project_inputs( + project_dir=tmp_path, declaration_reuse=first_session + ) + first_session.plan_models(model_files=first.model_files) + state: RenderReuseState | None = first_session.stored_state() + assert state is not None + _ = (tmp_path / test_case.edited_path).write_bytes(test_case.edited_contents) + second_session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=state, changed_paths=frozenset({test_case.edited_path}) + ) + + second: DiscoveredProjectInputs = discover_project_inputs( + project_dir=tmp_path, declaration_reuse=second_session + ) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + fresh: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + + assert any(name.startswith(RENDER_REUSE_DECLARATIONS_PREFIX) for name in state.group_payloads) + assert second.native_session is test_case.expected_reused_session + assert render_stage_capture(second) == render_stage_capture(fresh) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 7ba7b7441c..0c0d785c4d 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -3,10 +3,12 @@ from __future__ import annotations import dataclasses +from itertools import filterfalse from typing import cast from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" @@ -20,11 +22,15 @@ def constant_declaration(*, kind: str, value: object) -> dict[str, object]: def empty_inputs_capture(**overrides: object) -> dict[str, object]: - """Return a capture with every `DiscoveredProjectInputs` field, in order, then overrides.""" + """Return a capture with every captured `DiscoveredProjectInputs` field, then overrides.""" + type_name: str = "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs" + names: list[str] = [field.name for field in dataclasses.fields(DiscoveredProjectInputs)] return { - "__type__": "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", - **{field.name: [] for field in dataclasses.fields(DiscoveredProjectInputs)}, + "__type__": type_name, + **dict.fromkeys( + filterfalse(STAGE_CAPTURE_OMITTED_ATTRIBUTES[type_name].__contains__, names), [] + ), **overrides, } diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 7c1e471057..4cc4412381 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -67,6 +67,24 @@ stage=NativeStage.REFERENCE_EXTRACTION, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_declaration_files", + engine=CompilerEngine.PYTHON, + stage=NativeStage.DECLARATION_FILES, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_declaration_files", + engine=CompilerEngine.NATIVE, + stage=NativeStage.DECLARATION_FILES, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_declaration_files", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.DECLARATION_FILES, + expected_enabled=True, + ), ], ids=lambda case: case.description, ) From 98102ef2dfcb8ecedc191dc68b4b36ce172a4e48 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:52:30 +0100 Subject: [PATCH 3/4] fix(native): show authored text unchanged in discovery failure messages Discovery decoded raw-name markers across whole failure messages, so a NUL authored in a header key (for example a model's descri\0ption) was rewritten under every engine. Layout failures now show each listed path as it is displayed when the message is built, and the binding passes message text through untouched. --- .../_helpers/declaration_groups.rs | 8 +- .../declarations/_helpers/declaration_tree.rs | 5 +- .../src/declarations/_helpers/named_layout.rs | 16 ++- .../src/declarations/_helpers/scan.rs | 16 +++ .../_helpers/discovery/project_files.rs | 3 +- .../compiler/discovery/_test_types.py | 19 +++ .../discovery/test_discovery_failure_text.py | 126 ++++++++++++++++++ 7 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs index 0263ca2441..eadeaabf5b 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs @@ -5,7 +5,7 @@ use crate::constants::{ SCOPED_DECLARATION_DIRECTORIES, }; use crate::declarations::_helpers::paths::parent; -use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::_helpers::scan::{declaration_failure, shown, shown_list}; use crate::declarations::errors::ScanError; use crate::declarations::models::DeclarationGroup; use crate::models::StageFailure; @@ -56,7 +56,7 @@ pub(crate) fn validate_declaration_groups(tree: &ProjectTree) -> Result<(), Scan "Grouped declaration root {}/ must be below a concrete owner directory; use the \ project-wide macros/, enums/, constants/, audits/, schemas/, or hooks/ root \ instead", - group.directory + shown(&group.directory) ))); } let unsupported: Vec = children(tree, &group.directory)? @@ -67,8 +67,8 @@ pub(crate) fn validate_declaration_groups(tree: &ProjectTree) -> Result<(), Scan if !unsupported.is_empty() { return Err(declaration_failure(format!( "Declaration group {}/ contains unsupported entries: {}", - group.directory, - unsupported.join(", ") + shown(&group.directory), + shown_list(&unsupported) ))); } } diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs index ee4440329a..48431079ca 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -7,7 +7,7 @@ use crate::constants::{ }; use crate::declarations::_helpers::declaration_groups::validate_declaration_groups; use crate::declarations::_helpers::paths::{directory_facts, name, parent, stem}; -use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::_helpers::scan::{declaration_failure, shown}; use crate::declarations::errors::ScanError; use crate::declarations::models::{DeclarationFileFact, DeclarationKind, ScopeKind}; use crate::tree::main::directories::directories; @@ -115,7 +115,8 @@ fn is_kind(kind: Option, directory_name: &str) -> bool { fn nested_root_failure(directory: &str) -> ScanError { declaration_failure(format!( - "Declaration root {directory}/ is nested inside another declaration tree" + "Declaration root {}/ is nested inside another declaration tree", + shown(directory) )) } diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs index 0270b2fd10..323ef60e00 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs @@ -7,7 +7,7 @@ use crate::constants::{ }; use crate::declarations::_helpers::declaration_groups::owned_groups; use crate::declarations::_helpers::paths::{join, name}; -use crate::declarations::_helpers::scan::declaration_failure; +use crate::declarations::_helpers::scan::{declaration_failure, shown, shown_list}; use crate::declarations::errors::ScanError; use crate::declarations::models::DeclarationGroup; use crate::tree::main::children::children; @@ -50,9 +50,10 @@ pub(crate) fn named_declaration_groups( .find(|directory| is_nested_role_directory(name(directory))) { return Err(declaration_failure(format!( - "Declaration directory {nested}/ is not allowed inside the project-wide \ + "Declaration directory {}/ is not allowed inside the project-wide \ {top_level}/ role; scoped declarations belong under \ - /{DECLARATION_GROUP_DIRECTORY}/ below a resource tree" + /{DECLARATION_GROUP_DIRECTORY}/ below a resource tree", + shown(&nested) ))); } } @@ -78,8 +79,9 @@ fn validate_group_roles(tree: &ProjectTree, group: &str) -> Result<(), ScanError let singular: String = join(&directory, SINGULAR_AUDIT_DIRECTORY); if role == LOCAL_AUDIT_ROLE_DIRECTORY && tree.exists(&singular) { return Err(declaration_failure(format!( - "{singular}/ is invalid: singular audits are never used by name, so folder-only \ - visibility has no meaning; use {DECLARATION_GROUP_DIRECTORY}/audits/singular/" + "{}/ is invalid: singular audits are never used by name, so folder-only \ + visibility has no meaning; use {DECLARATION_GROUP_DIRECTORY}/audits/singular/", + shown(&singular) ))); } let help_text: String = format!( @@ -116,8 +118,8 @@ fn require_role_children(tree: &ProjectTree, role: &RoleChildren<'_>) -> Result< } Err(declaration_failure(format!( "Unsupported entries in {}/: {}; {}", - role.directory, - unsupported.join(", "), + shown(role.directory), + shown_list(&unsupported), role.help_text ))) } diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs index 688b4e0601..385597a36a 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs @@ -2,6 +2,22 @@ use crate::declarations::errors::ScanError; use crate::models::{DiscoveryFailure, FailureKind, StageFailure}; +use crate::tree::main::display_text::display_text; +use std::borrow::Cow; + +/// A listed relative path as a failure message shows it; message text is never decoded. +pub(crate) fn shown(relative_path: &str) -> Cow<'_, str> { + display_text(relative_path) +} + +/// The listed relative paths `paths`, shown and joined as Python lists them. +pub(crate) fn shown_list(paths: &[String]) -> String { + paths + .iter() + .map(|path| shown(path)) + .collect::>>() + .join(", ") +} /// A `DeclarationParseError` with Python's message. pub(crate) fn declaration_failure(message: String) -> ScanError { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 5a21341886..b901fc4cd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -24,7 +24,6 @@ use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_tex use sqlbuild_discovery::sql_tests::models::{ DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, }; -use sqlbuild_discovery::tree::main::display_text::display_text; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; @@ -80,7 +79,7 @@ pub(crate) fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyRes vec![ object(py, "error")?, object(py, failure.kind.as_str())?, - object(py, display_text(&failure.message))?, + object(py, failure.message)?, object(py, failure.help)?, ], ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 9c9cfdb0af..03b2960fe3 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -229,3 +229,22 @@ class DeclarationReuseTestCase: edited_path: str edited_contents: bytes expected_reused_session: object = None + + +@dataclass(frozen=True) +class FailureTextTestCase: + """A project whose discovery failure message must keep its authored text unchanged.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_error_type: str + expected_message_suffix: str + + +@dataclass(frozen=True) +class TolerantFailureTextTestCase: + """A project whose broken declaration files tolerant discovery faults with authored text.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_fault_keys: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py b/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py new file mode 100644 index 0000000000..7cff580a22 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py @@ -0,0 +1,126 @@ +"""Failure messages show authored text as written and escape only raw path names.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.models import DiscoveryFileFault +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + FailureTextTestCase, + TolerantFailureTextTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + stage_outcome, + tolerant_declaration_outcome, + write_project, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") +_MODEL: tuple[str, bytes] = ("models/orders.sql", b"MODEL ();\nSELECT 1 AS order_id") +_HOOK: tuple[str, bytes] = ("hooks/sql/refresh.sql", b"HOOK (descri\x00ption 'x');\nSELECT 1") +_CONSTANT: tuple[str, bytes] = ("constants/limits.sql", b"CONSTANT (name cap, valu\x00e 1);") + + +@pytest.mark.parametrize( + "test_case", + [ + FailureTextTestCase( + description="a NUL in a model header key", + files=(("models/orders.sql", b"MODEL (descri\x00ption 'x');\nSELECT 1"),), + expected_error_type="ModelSqlParseError", + expected_message_suffix="has unsupported keys: descri\x00ption", + ), + FailureTextTestCase( + description="a NUL in a SQL hook header key", + files=(_MODEL, _HOOK), + expected_error_type="SqlHookParseError", + expected_message_suffix="has unsupported keys: descri\x00ption", + ), + FailureTextTestCase( + description="a NUL in a constant key", + files=(_MODEL, _CONSTANT), + expected_error_type="DeclarationParseError", + expected_message_suffix="constant has unknown keys: valu\x00e", + ), + FailureTextTestCase( + description="a NUL in an audit header key", + files=( + _MODEL, + ("audits/generic/positive.sql", b"AUDIT (sever\x00ity warn);\nSELECT 1"), + ), + expected_error_type="SqlAuditParseError", + expected_message_suffix="has unsupported keys: sever\x00ity", + ), + FailureTextTestCase( + description="a NUL in a SQL function header key", + files=( + _MODEL, + ( + "functions/sql/cents.sql", + b'FUNCTION (arguments (), returns INTEGER, descri\x00ption "x");\nSELECT 1', + ), + ), + expected_error_type="ModelSqlParseError", + expected_message_suffix="has unsupported keys: descri\x00ption", + ), + FailureTextTestCase( + description="a raw entry name in a declaration group", + files=(_MODEL, (os.fsdecode(b"models/orders/_sqlbuild/caf\xe9.txt"), b"x")), + expected_error_type="DeclarationParseError", + expected_message_suffix="unsupported entries: models/orders/_sqlbuild/caf\\xe9.txt", + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_project_when_discovering_then_every_engine_shows_authored_text( + test_case: FailureTextTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + outcomes: list[tuple[object, ...]] = [ + stage_outcome(project_dir=tmp_path, engine=engine, monkeypatch=monkeypatch) + for engine in _ENGINES + ] + + assert {outcome[1:] for outcome in outcomes} == {outcomes[0][1:]} + assert outcomes[0][1] == test_case.expected_error_type + assert str(outcomes[0][2]).endswith(test_case.expected_message_suffix) + + +@pytest.mark.parametrize( + "test_case", + [ + TolerantFailureTextTestCase( + description="NUL header keys are faulted with their authored text", + files=(_MODEL, _HOOK, _CONSTANT), + expected_fault_keys=("descri\x00ption", "valu\x00e"), + ) + ], + ids=lambda case: case.description, +) +def test_given_nul_header_keys_when_discovering_tolerantly_then_faults_match_python( + test_case: TolerantFailureTextTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + python: tuple[object, tuple[DiscoveryFileFault, ...], tuple[DiscoveryFileFault, ...]] = ( + tolerant_declaration_outcome(project_dir=tmp_path, engine="python", monkeypatch=monkeypatch) + ) + + native: tuple[object, tuple[DiscoveryFileFault, ...], tuple[DiscoveryFileFault, ...]] = ( + tolerant_declaration_outcome( + project_dir=tmp_path, engine="native-preview", monkeypatch=monkeypatch + ) + ) + + assert native == python + assert ( + tuple(sorted(fault.message.rsplit("keys: ", 1)[1] for fault in (*native[1], *native[2]))) + == test_case.expected_fault_keys + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From c2110eb3eba3745db34904d49b8045893c155cfd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:52:30 +0100 Subject: [PATCH 4/4] fix(native): never reuse a native discovery session for edited inputs Document that a missing session means readers use the Python objects, and drop the session from the inputs each Rule fix pass lints. --- src/sqlbuild/compiler/discovery/models.py | 11 +++++- src/sqlbuild/lint/_helpers/fixes.py | 1 + .../sqlbuild/cli/commands/main/_test_types.py | 10 +++++ .../src/sqlbuild/cli/commands/main/helpers.py | 23 +++++++++++ .../cli/commands/main/test_format_fix.py | 38 +++++++++++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 93edb4afc2..7238f531a0 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -639,6 +639,11 @@ class DiscoveredDeclarationFiles: macro_files: tuple[DiscoveredMacroFile, ...] adapter_file: DiscoveredAdapterFile | None native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) + """The native session that read these declaration files, for later native stages. + + `None` means no session: readers must use the Python objects. Any copy that changes a + discovered collection must set it to `None` so it never describes other files. + """ @dataclass(frozen=True) @@ -674,7 +679,11 @@ class DiscoveredProjectInputs: providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) adapter_file: DiscoveredAdapterFile | None = None native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) - """The native discovery session that read the declaration files, for later native stages.""" + """The native session that read these declaration files, for later native stages. + + `None` means no session: readers must use the Python objects. Any copy that changes a + discovered collection must set it to `None` so it never describes other files. + """ @dataclass(frozen=True) diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 445ad9bbb5..2bc0ecb9de 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -439,4 +439,5 @@ def _updated_inputs( else model for model in inputs.model_files ), + native_session=None, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 337cf7677d..a4be012149 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1050,3 +1050,13 @@ class InferredFreshnessCliTestCase: description: str sources_yaml: str expected_sources: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class FixSessionTestCase: + """A Rule fix under an engine whose discovery retains a native session.""" + + description: str + engine: str + expected_original_sessions: tuple[bool, ...] + expected_fix_pass_sessions: frozenset[bool] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 444e0ba56d..ac4748279d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -34,6 +34,8 @@ ) from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.lint._helpers import fixes as lint_fixes +from sqlbuild.lint.main import run_format as run_format_module from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) @@ -189,6 +191,27 @@ def recorded_plan(**kwargs: Any) -> PlannedStaticSqlTests: return threads +def record_fix_sessions(monkeypatch: pytest.MonkeyPatch) -> tuple[list[bool], list[bool]]: + """Record whether the inputs a Rule fix starts from, and each fix pass lints, hold a session.""" + + original: list[bool] = [] + passes: list[bool] = [] + plan: Callable[..., Any] = run_format_module.plan_rule_fixes + lint: Callable[..., Any] = lint_fixes.run_lint + + def recorded_plan(**kwargs: Any) -> Any: + original.append(getattr(kwargs["discovered_inputs"], "native_session", None) is not None) + return plan(**kwargs) + + def recorded_lint(**kwargs: Any) -> Any: + passes.append(kwargs["discovered_inputs"].native_session is not None) + return lint(**kwargs) + + monkeypatch.setattr(run_format_module, "plan_rule_fixes", recorded_plan) + monkeypatch.setattr(lint_fixes, "run_lint", recorded_lint) + return original, passes + + def skip_artifact_preparation(self: PreparedCompileArtifacts, **kwargs: object) -> None: """Disable background preparation so the write phase plans SQL tests inline.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py index d1f246becd..0e808937a8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -10,13 +10,16 @@ import pytest from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + FixSessionTestCase, FormatCompileIntegrationTestCase, SemanticFixTestCase, UnusedOutputDifferentialCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( built_model_rows, + record_fix_sessions, rule_fix_statuses, ) @@ -31,6 +34,41 @@ ) +@pytest.mark.parametrize( + "test_case", + [ + FixSessionTestCase( + description="preview fix passes drop the session of the unfixed files", + engine="native-preview", + expected_original_sessions=(True,), + expected_fix_pass_sessions=frozenset({False}), + ), + ], + ids=lambda case: case.description, +) +def test_given_retained_native_session_when_fixing_then_fix_passes_never_reuse_it( + test_case: FixSessionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + 'MODEL (description "Orders");\nSELECT 1 AS order_id UNION SELECT 2 AS order_id\n' + ) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine) + original, passes = record_fix_sessions(monkeypatch) + + exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + + capsys.readouterr() + assert exit_code == 0 + assert tuple(original) == test_case.expected_original_sessions + assert frozenset(passes) == test_case.expected_fix_pass_sessions + + @pytest.mark.parametrize( "test_case", [FormatCompileIntegrationTestCase("bare union", "UNION DISTINCT")],