feat(native): parse declaration files natively behind the preview engine (#1014)

* feat(native): parse declaration files natively in sqlbuild-discovery

Add native listing, reading and parsing for enum, constant, model schema, audit, SQL hook, SQL function, seed and macro files, in Python's discovery order, with a deferred outcome for inputs only Python can judge and a session that retains each collection and its layout for later stages.

* feat(native): discover declaration files through the native session behind the preview engine

Add the preview-tier DECLARATION_FILES stage. Each declaration collection is read once per pass through a retained native discovery session; Python builds the records, normalises constant values and parses schema columns, and re-runs its own parser for any file native rejects or defers, raising NativeStageMismatchError when the outcomes differ. Default engines are unchanged.

* fix(native): show authored text unchanged in discovery failure messages

Discovery decoded raw-name markers across whole failure messages, so a NUL authored in a header key (for example a model's descri\0ption) was rewritten under every engine. Layout failures now show each listed path as it is displayed when the message is built, and the binding passes message text through untouched.

* fix(native): never reuse a native discovery session for edited inputs

Document that a missing session means readers use the Python objects, and drop the session from the inputs each Rule fix pass lints.

Author: kvlonge

…ry/src/model_files/_helpers/locations.rs …uild-discovery/src/_helpers/locations.rscrates/sqlbuild-discovery/src/model_files/_helpers/locations.rs renamed to crates/sqlbuild-discovery/src/_helpers/locations.rs crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs renamed to crates/sqlbuild-discovery/src/_helpers/locations.rs

@@ -1,4 +1,5 @@
 pub(crate) mod header_keys;
+pub(crate) mod locations;
 pub(crate) mod pool;
 pub(crate) mod reading;
 pub(crate) mod scoped_paths;

crates/sqlbuild-discovery/src/_helpers/mod.rs

@@ -0,0 +1,77 @@
+//! Python's `validate_resource_identity` for declaration names that are already SQL identifiers.
+
+use crate::declaration_files::_helpers::checks::stops::ParseStop;
+use crate::models::{DiscoveryFailure, FailureKind};
+
+const IDENTITY_HELP: &str = "Rename the authored identity and update its references, selectors, \
+                             and integration keys. SQLBuild does not silently normalize resource \
+                             identities. Double underscores remain valid.";
+const FALLBACK_IDENTITY: &str = "resource_name";
+
+/// Reject a public identity unless it is lowercase snake_case; `name` matches `[A-Za-z_]\w*`.
+pub(crate) fn validate_public_identity(
+    name: &str,
+    kind: &str,
+    file_path: &str,
+) -> Result<(), ParseStop> {
+    if is_snake_case(name) {
+        return Ok(());
+    }
+    Err(ParseStop::Failed(DiscoveryFailure {
+        kind: FailureKind::ResourceIdentity,
+        message: format!(
+            "Invalid public {kind} identity '{name}' in {file_path}; use snake_case '{}'",
+            suggested_identity(name)
+        ),
+        help: Some(IDENTITY_HELP.to_owned()),
+    }))
+}
+
+/// `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`.
+fn is_snake_case(name: &str) -> bool {
+    let bytes = name.as_bytes();
+    let tail_ok = |byte: &u8| byte.is_ascii_lowercase() || byte.is_ascii_digit();
+    match bytes {
+        [] => false,
+        [first] => first.is_ascii_lowercase(),
+        [first, middle @ .., last] => {
+            first.is_ascii_lowercase()
+                && middle.iter().all(|byte| tail_ok(byte) || *byte == b'_')
+                && tail_ok(last)
+        }
+    }
+}
+
+/// `suggest_resource_identity` for an ASCII identifier: split acronyms and words, then lowercase.
+fn suggested_identity(name: &str) -> String {
+    let characters: Vec<char> = name.chars().collect();
+    let mut acronyms: Vec<char> = Vec::with_capacity(characters.len() * 2);
+    for (index, character) in characters.iter().enumerate() {
+        let splits = index > 0
+            && characters[index - 1].is_ascii_uppercase()
+            && character.is_ascii_uppercase()
+            && characters
+                .get(index + 1)
+                .is_some_and(char::is_ascii_lowercase);
+        if splits {
+            acronyms.push('_');
+        }
+        acronyms.push(*character);
+    }
+    let mut words: String = String::with_capacity(acronyms.len() * 2);
+    for (index, character) in acronyms.iter().enumerate() {
+        if index > 0
+            && (acronyms[index - 1].is_ascii_lowercase() || acronyms[index - 1].is_ascii_digit())
+            && character.is_ascii_uppercase()
+        {
+            words.push('_');
+        }
+        words.push(character.to_ascii_lowercase());
+    }
+    let corrected: &str = words.trim_matches('_');
+    if corrected.is_empty() {
+        FALLBACK_IDENTITY.to_owned()
+    } else {
+        corrected.to_owned()
+    }
+}

crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/identities.rs

@@ -0,0 +1,3 @@
+pub(crate) mod identities;
+pub(crate) mod python_values;
+pub(crate) mod stops;

crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/mod.rs

@@ -0,0 +1,122 @@
+//! The Python type of a projected header value, as the discovery parsers' `isinstance` checks see it.
+
+use crate::declaration_files::_helpers::checks::stops::ParseStop;
+use crate::models::{DiscoveryFailure, FailureKind};
+use sqlbuild_core::text::main::python_strip::python_strip;
+use sqlbuild_sqltext::compiler::models::AuthoredValue;
+
+/// The Python type a header value projects to.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum PythonType {
+    None,
+    Bool,
+    Int,
+    Float,
+    Str,
+    List,
+    Dict,
+    /// `AuthoredSqlSet`, a tuple, a `constant(...)` call or a hook entry.
+    Other,
+}
+
+/// The projected type; a bare word with non-ASCII text may be a Unicode number, so it defers.
+pub(crate) fn python_type(value: &AuthoredValue) -> Result<PythonType, ParseStop> {
+    Ok(match value {
+        AuthoredValue::Null => PythonType::None,
+        AuthoredValue::Boolean(_) => PythonType::Bool,
+        AuthoredValue::BareWord(word) => word_type(word)?,
+        AuthoredValue::String(_) => PythonType::Str,
+        AuthoredValue::List(_) => PythonType::List,
+        AuthoredValue::Map(_) => PythonType::Dict,
+        AuthoredValue::Set(_)
+        | AuthoredValue::Tuple(_)
+        | AuthoredValue::TypedConstant(_)
+        | AuthoredValue::InlineSqlHook(_)
+        | AuthoredValue::NamedSqlHook(..)
+        | AuthoredValue::PythonHook(..) => PythonType::Other,
+    })
+}
+
+/// The text of a value that projects to `str`, or `None` for any other type.
+pub(crate) fn python_str(value: &AuthoredValue) -> Result<Option<&str>, ParseStop> {
+    Ok(match value {
+        AuthoredValue::String(text) => Some(text),
+        AuthoredValue::BareWord(word) if word_type(word)? == PythonType::Str => Some(word),
+        _ => None,
+    })
+}
+
+/// `isinstance(value, str) and value.strip()`.
+pub(crate) fn non_empty_str(value: &AuthoredValue) -> Result<Option<&str>, ParseStop> {
+    Ok(python_str(value)?.filter(|text| !python_strip(text).is_empty()))
+}
+
+/// Python's `dict.get(key)`.
+pub(crate) fn get<'a>(
+    values: &'a [(String, AuthoredValue)],
+    key: &str,
+) -> Option<&'a AuthoredValue> {
+    values
+        .iter()
+        .find(|(candidate, _)| candidate == key)
+        .map(|(_, value)| value)
+}
+
+/// The keys outside `allowed`, sorted as Python's `sorted(set(values) - allowed)`.
+pub(crate) fn unknown_keys(values: &[(String, AuthoredValue)], allowed: &[&str]) -> Vec<String> {
+    let mut unknown: Vec<String> = values
+        .iter()
+        .map(|(key, _)| key.clone())
+        .filter(|key| !allowed.contains(&key.as_str()))
+        .collect();
+    unknown.sort_unstable();
+    unknown.dedup();
+    unknown
+}
+
+/// `re.fullmatch(r"^[A-Za-z_][A-Za-z0-9_]*$", text)`.
+pub(crate) fn is_identifier(text: &str) -> bool {
+    let mut characters = text.chars();
+    characters
+        .next()
+        .is_some_and(|first| first.is_ascii_alphabetic() || first == '_')
+        && characters.all(|character| character.is_ascii_alphanumeric() || character == '_')
+}
+
+/// A failure of `kind` with `message` and no help.
+pub(crate) fn failure(kind: FailureKind, message: String) -> ParseStop {
+    ParseStop::Failed(DiscoveryFailure::new(kind, message))
+}
+
+/// `_parse_word_value`: `true`, `false` and `null` are parsed as values; numbers become numbers.
+fn word_type(word: &str) -> Result<PythonType, ParseStop> {
+    if !word.is_ascii() {
+        return Err(ParseStop::Deferred);
+    }
+    Ok(match word {
+        "true" | "false" => PythonType::Bool,
+        "null" => PythonType::None,
+        _ if is_integer(word) => PythonType::Int,
+        _ if is_float(word) => PythonType::Float,
+        _ => PythonType::Str,
+    })
+}
+
+fn unsigned(word: &str) -> &str {
+    word.strip_prefix(['+', '-']).unwrap_or(word)
+}
+
+/// `^[+-]?\d+$` over ASCII text.
+fn is_integer(word: &str) -> bool {
+    let digits = unsigned(word);
+    !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
+}
+
+/// `^[+-]?(?:\d+\.\d*|\d*\.\d+)$` over ASCII text.
+fn is_float(word: &str) -> bool {
+    let Some((whole, fraction)) = unsigned(word).split_once('.') else {
+        return false;
+    };
+    let digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit());
+    digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty())
+}

crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs

@@ -0,0 +1,16 @@
+//! Why native parsing of one declaration file stopped.
+
+use crate::models::DiscoveryFailure;
+
+/// The first failure Python raises for the file, or a file only the Python parser reproduces.
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) enum ParseStop {
+    Failed(DiscoveryFailure),
+    Deferred,
+}
+
+impl From<DiscoveryFailure> for ParseStop {
+    fn from(failure: DiscoveryFailure) -> Self {
+        Self::Failed(failure)
+    }
+}

crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs

@@ -0,0 +1,127 @@
+//! Read and parse the files of one collection into its retained native form.
+
+use crate::declaration_files::_helpers::listing::collections::{
+    NamedKind, declaration_files, file_name, function_files, named_declaration_files, read_files,
+    seed_files,
+};
+use crate::declaration_files::_helpers::parsing::audit_files::parse_audit_file;
+use crate::declaration_files::_helpers::parsing::constant_files::parse_constant_file;
+use crate::declaration_files::_helpers::parsing::enum_files::parse_enum_file;
+use crate::declaration_files::_helpers::parsing::function_files::parse_function_file;
+use crate::declaration_files::_helpers::parsing::hook_files::parse_hook_file;
+use crate::declaration_files::_helpers::parsing::schema_files::parse_schema_file;
+use crate::declaration_files::models::{
+    CollectionFailure, CollectionKind, CollectionRequest, DeclarationCollection,
+    DeclarationFileOptions, DiscoverySession, MacroFile, ScopedFile, SeedFile,
+};
+use crate::declarations::models::DeclarationKind;
+use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageFailure};
+use crate::tree::models::ProjectTree;
+
+/// The project a collection is read from and the options its parsers use.
+pub(crate) struct CollectionSource<'a> {
+    pub(crate) session: &'a DiscoverySession,
+    pub(crate) tree: &'a ProjectTree,
+}
+
+/// Read and parse the collection `request` names.
+pub(crate) fn read_collection(
+    source: &CollectionSource<'_>,
+    request: CollectionRequest,
+) -> DeclarationCollection {
+    let tree: &ProjectTree = source.tree;
+    let root: &ProjectRoot = &source.session.root;
+    let options: &DeclarationFileOptions = &source.session.options;
+    let isolate_kind: bool = request.isolate_kind;
+    match request.kind {
+        CollectionKind::Enums => DeclarationCollection::Enums(
+            declaration_files(source, DeclarationKind::Enum, isolate_kind).map(|files| {
+                read_files(root, tree, files, |file_path, _, contents| {
+                    parse_enum_file(file_path, contents)
+                })
+            }),
+        ),
+        CollectionKind::Constants => DeclarationCollection::Constants(
+            declaration_files(source, DeclarationKind::Constant, isolate_kind).map(|files| {
+                read_files(root, tree, files, |file_path, _, contents| {
+                    parse_constant_file(file_path, contents)
+                })
+            }),
+        ),
+        CollectionKind::Macros => DeclarationCollection::Macros(
+            declaration_files(source, DeclarationKind::Macro, isolate_kind).map(|files| {
+                read_files(root, tree, files, |_, _, contents| {
+                    Ok(MacroFile { contents })
+                })
+            }),
+        ),
+        CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas(
+            named_declaration_files(source, &[NamedKind::Schema], false).map(|files| {
+                read_files(root, tree, files, |file_path, _, contents| {
+                    parse_schema_file(file_path, contents)
+                })
+            }),
+        ),
+        CollectionKind::Audits => DeclarationCollection::Audits(
+            named_declaration_files(source, &[NamedKind::Audit, NamedKind::SingularAudit], false)
+                .map(|files| {
+                    read_files(root, tree, files, |file_path, _, contents| {
+                        parse_audit_file(file_path, contents, options)
+                    })
+                }),
+        ),
+        CollectionKind::SqlHooks => DeclarationCollection::SqlHooks(
+            named_declaration_files(source, &[NamedKind::SqlHook], true).map(|files| {
+                read_files(root, tree, files, |file_path, relative_path, contents| {
+                    parse_hook_file(file_path, stem(file_name(relative_path)), contents, options)
+                })
+            }),
+        ),
+        CollectionKind::SqlFunctions => {
+            DeclarationCollection::SqlFunctions(function_files(tree).map(|files| {
+                read_files(root, tree, files, |file_path, _, contents| {
+                    parse_function_file(file_path, contents, options)
+                })
+            }))
+        }
+        CollectionKind::Seeds => DeclarationCollection::Seeds(
+            seed_files(tree).map(|paths| paths.into_iter().map(seed_file).collect()),
+        ),
+    }
+}
+
+fn seed_file(relative_path: String) -> ScopedFile<SeedFile> {
+    ScopedFile {
+        file: DiscoveredFile {
+            relative_path,
+            outcome: FileOutcome::Parsed(SeedFile),
+        },
+        scope: None,
+    }
+}
+
+/// The collection `kind` failing as a whole.
+pub(crate) fn failed_collection(
+    kind: CollectionKind,
+    failure: StageFailure,
+) -> DeclarationCollection {
+    let failure = CollectionFailure::Stage(failure);
+    match kind {
+        CollectionKind::Enums => DeclarationCollection::Enums(Err(failure)),
+        CollectionKind::Constants => DeclarationCollection::Constants(Err(failure)),
+        CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas(Err(failure)),
+        CollectionKind::SqlFunctions => DeclarationCollection::SqlFunctions(Err(failure)),
+        CollectionKind::SqlHooks => DeclarationCollection::SqlHooks(Err(failure)),
+        CollectionKind::Audits => DeclarationCollection::Audits(Err(failure)),
+        CollectionKind::Seeds => DeclarationCollection::Seeds(Err(failure)),
+        CollectionKind::Macros => DeclarationCollection::Macros(Err(failure)),
+    }
+}
+
+/// Python's `Path.stem` of a file name.
+fn stem(file_name: &str) -> &str {
+    match file_name.rfind('.') {
+        Some(index) if index > 0 && index + 1 < file_name.len() => &file_name[..index],
+        _ => file_name,
+    }
+}