feat(native): discover declarations, SQL tests and YAML files natively

crates/sqlbuild-config/src/yaml/_helpers/composer.rs

@@ -7,8 +7,8 @@ use crate::yaml::_helpers::anchors::{
 use crate::yaml::_helpers::block_scalars::{column_of, scan_block_scalar};
 use crate::yaml::_helpers::resolver::resolve_scalar;
 use crate::yaml::constants::{
-    FLOW_KEY_INDICATOR, MAP_TAG, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG,
-    PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN,
+    FLOW_KEY_INDICATOR, MAP_TAG, MAX_NATIVE_IMPLICIT_KEY_CHARACTERS, MAX_NESTING_DEPTH,
+    NON_SPECIFIC_TAG, PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN,
 };
 use crate::yaml::models::{ComposedDocument, Node, NodeContent};
 use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag};
@@ -142,8 +142,8 @@ impl<'input> Composer<'input> {
         self.chars.get(index).copied()
     }
 
-    /// Whether an implicit key, from its first property to its `:` indicator, covers a line break.
-    fn spans_lines(&self, span: Span) -> bool {
+    /// An implicit key's extent from its first property to its `:` indicator; `None` if explicit.
+    fn implicit_key_extent(&self, span: Span) -> Option<(usize, usize)> {
         let key_end = span.end.index().min(self.chars.len());
         let indicator = (key_end..self.chars.len())
             .find(|index| !matches!(self.chars[*index], ' ' | '\t' | '\n' | '\r'))
@@ -166,10 +166,22 @@ impl<'input> Composer<'input> {
                 self.char_at(start + 1),
                 None | Some(' ' | '\t' | '\n' | '\r')
             );
-        !explicit
-            && self.chars[start.min(end)..end]
+        (!explicit).then_some((start.min(end), end))
+    }
+
+    /// Whether an implicit key, from its first property to its `:` indicator, covers a line break.
+    fn spans_lines(&self, span: Span) -> bool {
+        self.implicit_key_extent(span).is_some_and(|(start, end)| {
+            self.chars[start..end]
                 .iter()
                 .any(|character| matches!(character, '\n' | '\r'))
+        })
+    }
+
+    /// Whether an implicit key nears PyYAML's 1024-character simple-key limit.
+    fn is_long_implicit_key(&self, span: Span) -> bool {
+        self.implicit_key_extent(span)
+            .is_some_and(|(start, end)| end - start > MAX_NATIVE_IMPLICIT_KEY_CHARACTERS)
     }
 
     /// Whether the collection starting at `span` is written in flow style.
@@ -327,6 +339,11 @@ impl<'input> Composer<'input> {
                         "a plain scalar in a flow collection cannot start with ':'",
                     ));
                 }
+                if placement.key && self.is_long_implicit_key(span) {
+                    return Err(unsupported(
+                        "implicit keys near PyYAML's 1024-character limit are left to Python",
+                    ));
+                }
                 if placement.key && placement.flow && self.spans_lines(span) {
                     return Err(ConfigError::new(
                         ConfigErrorKind::Syntax,

crates/sqlbuild-config/src/yaml/constants.rs

@@ -60,3 +60,5 @@ pub(crate) const STRIP_INDICATOR: char = '-';
 pub(crate) const SEQUENCE_ENTRY_TOKEN: &str = "-";
 pub(crate) const DIRECTIVE_INDICATOR: char = '%';
 pub(crate) const PLAIN_FORBIDDEN_FIRST_CHARACTERS: [char; 5] = ['|', '>', '%', '@', '`'];
+/// Longer implicit keys are left to Python, which rejects simple keys beyond 1024 characters.
+pub(crate) const MAX_NATIVE_IMPLICIT_KEY_CHARACTERS: usize = 1000;

crates/sqlbuild-core/src/text/_helpers/cleandoc.rs

@@ -0,0 +1,34 @@
+//! Line helpers behind the `inspect.cleandoc` port.
+
+const TAB_SIZE: usize = 8;
+
+/// `str.expandtabs()`: a tab advances to the next multiple of eight columns.
+pub(crate) fn expand_tabs(text: &str) -> String {
+    let mut expanded: String = String::with_capacity(text.len());
+    let mut column: usize = 0;
+    for character in text.chars() {
+        match character {
+            '\t' => {
+                let width = TAB_SIZE - column % TAB_SIZE;
+                expanded.extend(std::iter::repeat_n(' ', width));
+                column += width;
+            }
+            '\n' | '\r' => {
+                expanded.push(character);
+                column = 0;
+            }
+            _ => {
+                expanded.push(character);
+                column += 1;
+            }
+        }
+    }
+    expanded
+}
+
+/// `line[count:]` in code points.
+pub(crate) fn skip_chars(line: &str, count: usize) -> &str {
+    line.char_indices()
+        .nth(count)
+        .map_or("", |(index, _)| &line[index..])
+}

crates/sqlbuild-core/src/text/_helpers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod alnum_ranges;
+pub(crate) mod cleandoc;
 pub(crate) mod sequence_matcher;

crates/sqlbuild-core/src/text/main/mod.rs

@@ -4,4 +4,5 @@ pub mod is_python_alnum;
 pub mod is_python_space;
 pub mod is_python_word;
 pub mod python_alnum_unicode_version;
+pub mod python_cleandoc;
 pub mod python_strip;

crates/sqlbuild-core/src/text/main/python_cleandoc.rs

@@ -0,0 +1,34 @@
+//! Python's `inspect.cleandoc` (3.12), which SQL test and scenario bodies go through.
+
+use crate::text::_helpers::cleandoc::{expand_tabs, skip_chars};
+use crate::text::main::is_python_space::is_python_space;
+
+/// `inspect.cleandoc(text)`: expand tabs, dedent later lines and drop outer empty lines.
+pub fn python_cleandoc(text: &str) -> String {
+    let expanded: String = expand_tabs(text);
+    let mut lines: Vec<&str> = expanded.split('\n').collect();
+    let margin: Option<usize> = lines[1..]
+        .iter()
+        .filter_map(|line| {
+            let content = line.trim_start_matches(is_python_space);
+            (!content.is_empty()).then(|| line.chars().count() - content.chars().count())
+        })
+        .min();
+    lines[0] = lines[0].trim_start_matches(is_python_space);
+    let mut cleaned: Vec<&str> = Vec::with_capacity(lines.len());
+    cleaned.push(lines[0]);
+    cleaned.extend(
+        lines[1..]
+            .iter()
+            .map(|line| margin.map_or(*line, |margin| skip_chars(line, margin))),
+    );
+    let last: usize = cleaned
+        .iter()
+        .rposition(|line| !line.is_empty())
+        .map_or(0, |index| index + 1);
+    let first: usize = cleaned[..last]
+        .iter()
+        .position(|line| !line.is_empty())
+        .unwrap_or(last);
+    cleaned[first..last].join("\n")
+}

crates/sqlbuild-discovery/Cargo.toml

@@ -9,6 +9,7 @@ rust-version.workspace = true
 
 [dependencies]
 rayon.workspace = true
+sqlbuild-config.workspace = true
 sqlbuild-core.workspace = true
 sqlbuild-sqltext.workspace = true
 

crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs → crates/sqlbuild-discovery/src/_helpers/header_keys.rs

@@ -7,6 +7,7 @@ use sqlbuild_core::text::main::is_python_word::is_python_word;
 
 /// The facts of one header that declares keys outside its supported set.
 pub(crate) struct UnsupportedKeys<'a> {
+    pub(crate) kind: FailureKind,
     pub(crate) statement: &'a str,
     pub(crate) file_path: &'a str,
     pub(crate) header: &'a str,
@@ -18,6 +19,7 @@ pub(crate) struct UnsupportedKeys<'a> {
 
 pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> DiscoveryFailure {
     let UnsupportedKeys {
+        kind,
         statement,
         file_path,
         header,
@@ -33,7 +35,7 @@ pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> Discovery
             .filter(|character| **character == '\n')
             .count();
     DiscoveryFailure {
-        kind: FailureKind::ModelSql,
+        kind: *kind,
         message: format!(
             "{statement} in '{file_path}:{line}' has unsupported keys: {}",
             keys.join(", ")

crates/sqlbuild-discovery/src/_helpers/mod.rs

@@ -1,4 +1,6 @@
+pub(crate) mod header_keys;
 pub(crate) mod pool;
 pub(crate) mod reading;
 pub(crate) mod scoped_paths;
+pub(crate) mod statement_headers;
 pub(crate) mod suggestions;

crates/sqlbuild-discovery/src/_helpers/statement_headers.rs

@@ -0,0 +1,66 @@
+//! Parse a statement header and reject its unsupported keys, as Python's discovery does.
+
+use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure};
+use crate::models::{DiscoveryFailure, FailureKind};
+use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one;
+use sqlbuild_sqltext::compiler::models::AuthoredValue;
+
+/// One statement kind's header contract.
+pub(crate) struct StatementHeader<'a> {
+    pub(crate) kind: FailureKind,
+    /// The name in syntax errors, such as `TEST`.
+    pub(crate) statement_name: &'a str,
+    /// The name in unsupported-key errors, such as `TEST()`.
+    pub(crate) statement: &'a str,
+    pub(crate) file_path: &'a str,
+    pub(crate) supported_keys: &'a [String],
+}
+
+/// `parse_header_values` then `reject_unsupported_header_keys`; `header_line` is one-based.
+pub(crate) fn parse_statement_header(
+    contract: &StatementHeader<'_>,
+    header: &str,
+    header_line: usize,
+) -> Result<Vec<(String, AuthoredValue)>, DiscoveryFailure> {
+    let values: Vec<(String, AuthoredValue)> = match parse_one(header) {
+        (_, _, Some(error)) => return Err(syntax_failure(contract, &error)),
+        (Some(AuthoredValue::Map(values)), _, None) => values,
+        _ => {
+            return Err(syntax_failure(
+                contract,
+                "Native MODEL header parser returned neither values nor an error",
+            ));
+        }
+    };
+    let unsupported: Vec<&str> = values
+        .iter()
+        .map(|(key, _)| key.as_str())
+        .filter(|key| !is_supported(contract.supported_keys, key))
+        .collect();
+    if unsupported.is_empty() {
+        return Ok(values);
+    }
+    Err(unsupported_keys_failure(&UnsupportedKeys {
+        kind: contract.kind,
+        statement: contract.statement,
+        file_path: contract.file_path,
+        header,
+        header_line,
+        keys: &unsupported,
+        supported_keys: contract.supported_keys,
+    }))
+}
+
+fn syntax_failure(contract: &StatementHeader<'_>, error: &str) -> DiscoveryFailure {
+    DiscoveryFailure::new(
+        contract.kind,
+        format!(
+            "{}(...) in '{}' contains invalid SQLBuild header syntax: {error}",
+            contract.statement_name, contract.file_path
+        ),
+    )
+}
+
+fn is_supported(supported_keys: &[String], key: &str) -> bool {
+    supported_keys.iter().any(|supported| supported == key)
+}

crates/sqlbuild-discovery/src/constants.rs

@@ -25,5 +25,29 @@ pub const CANONICAL_AUTHORED_ROOTS: [&[&str]; 5] = [
 ];
 /// The SQL test root, whose `macros/` SQL files are tests rather than declarations.
 pub const SQL_TESTS_ROOT: &[&str] = &["tests", "unit"];
+/// The SQL scenario root.
+pub const SQL_SCENARIOS_ROOT: &str = "tests/scenarios";
 /// The directory of macro tests below the SQL test root.
 pub const MACRO_TESTS_DIRECTORY: &str = "macros";
+/// Inherited declaration directories, which may also be project-wide global roots.
+pub const GLOBAL_DECLARATION_DIRECTORIES: [&str; 3] = ["macros", "enums", "constants"];
+/// Folder-local declaration directories.
+pub const LOCAL_DECLARATION_DIRECTORIES: [&str; 3] = ["_macros", "_enums", "_constants"];
+/// Directories a declaration group may hold for named declarations.
+pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [
+    "audits", "_audits", "schemas", "_schemas", "hooks", "_hooks",
+];
+/// Project-wide named declaration roles.
+pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"];
+/// Root of seed files, walked so the Python snapshot shares its listings.
+pub const SEEDS_DIRECTORY: &str = "seeds";
+/// Suffix of macro files.
+pub const PYTHON_FILE_SUFFIX: &str = ".py";
+/// The stem of package initialisers, which are not macro files.
+pub const PYTHON_INIT_MODULE_STEM: &str = "__init__";
+/// The project-wide audit role directory.
+pub const AUDIT_ROLE_DIRECTORY: &str = "audits";
+/// The folder-local audit role directory inside a declaration group.
+pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits";
+/// The singular audit directory below an audit role.
+pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular";

crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs

@@ -0,0 +1,82 @@
+//! Python's `_validate_declaration_groups` and `_declaration_groups`.
+
+use crate::constants::{
+    CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GROUPED_NAMED_DECLARATION_DIRECTORIES,
+    SCOPED_DECLARATION_DIRECTORIES,
+};
+use crate::declarations::_helpers::paths::parent;
+use crate::declarations::_helpers::scan::declaration_failure;
+use crate::declarations::errors::ScanError;
+use crate::declarations::models::DeclarationGroup;
+use crate::models::StageDeferral;
+use crate::tree::main::children::children;
+use crate::tree::main::rglob::rglob;
+use crate::tree::models::{ProjectTree, TreeEntry};
+
+/// Every `_sqlbuild/` directory below a canonical root, root by root, in path order.
+fn all_groups(tree: &ProjectTree) -> Result<Vec<DeclarationGroup>, StageDeferral> {
+    let mut groups: Vec<DeclarationGroup> = Vec::new();
+    for root_parts in CANONICAL_AUTHORED_ROOTS {
+        let root: String = root_parts.join("/");
+        if !tree.is_dir(&root) {
+            continue;
+        }
+        for directory in rglob(tree, &root, |entry| {
+            entry.name == DECLARATION_GROUP_DIRECTORY
+        })? {
+            if tree.is_dir(&directory) {
+                groups.push(DeclarationGroup {
+                    root: root.clone(),
+                    directory,
+                });
+            }
+        }
+    }
+    Ok(groups)
+}
+
+/// Python's `_declaration_groups`: the groups below a concrete owner directory.
+pub(crate) fn owned_groups(tree: &ProjectTree) -> Result<Vec<DeclarationGroup>, StageDeferral> {
+    Ok(all_groups(tree)?
+        .into_iter()
+        .filter(|group| parent(&group.directory) != group.root)
+        .collect())
+}
+
+pub(crate) fn validate_declaration_groups(tree: &ProjectTree) -> Result<(), ScanError> {
+    if tree.exists(DECLARATION_GROUP_DIRECTORY) {
+        return Err(declaration_failure(format!(
+            "Grouped declaration root {DECLARATION_GROUP_DIRECTORY}/ must be below a canonical \
+             authored root"
+        )));
+    }
+    for group in all_groups(tree)? {
+        if parent(&group.directory) == group.root {
+            return Err(declaration_failure(format!(
+                "Grouped declaration root {}/ must be below a concrete owner directory; use the \
+                 project-wide macros/, enums/, constants/, audits/, schemas/, or hooks/ root \
+                 instead",
+                group.directory
+            )));
+        }
+        let unsupported: Vec<String> = children(tree, &group.directory)?
+            .into_iter()
+            .filter(|(_, entry)| !is_group_role(entry))
+            .map(|(path, _)| path)
+            .collect();
+        if !unsupported.is_empty() {
+            return Err(declaration_failure(format!(
+                "Declaration group {}/ contains unsupported entries: {}",
+                group.directory,
+                unsupported.join(", ")
+            )));
+        }
+    }
+    Ok(())
+}
+
+fn is_group_role(entry: &TreeEntry) -> bool {
+    entry.is_dir
+        && (SCOPED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str())
+            || GROUPED_NAMED_DECLARATION_DIRECTORIES.contains(&entry.name.as_str()))
+}