From ad2b78280f967ea6a638c31a80b497b3d6326bcb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:38:38 +0100 Subject: [PATCH 1/7] feat(native): add the native declaration scope crate The new sqlbuild-scopes crate projects the scope builder's walked resources and declarations into the static scope index with the Python builder's record orders, duplicate and naming diagnostics and macro dependency usages, the lookup's canonical orders and repr-ordered groups, declaration visibility and the expected-model and tested-macro relationship grants. Any path Python would reject defers the stage. --- Cargo.lock | 5 + Cargo.toml | 3 + crates/README.md | 9 +- crates/sqlbuild-scopes/Cargo.toml | 11 + crates/sqlbuild-scopes/src/lib.rs | 5 + .../src/scope_index/_helpers/diagnostics.rs | 162 ++++++++ .../src/scope_index/_helpers/grants.rs | 192 +++++++++ .../src/scope_index/_helpers/identities.rs | 90 ++++ .../src/scope_index/_helpers/mod.rs | 8 + .../src/scope_index/_helpers/ordering.rs | 99 +++++ .../src/scope_index/_helpers/paths.rs | 74 ++++ .../src/scope_index/_helpers/records.rs | 55 +++ .../src/scope_index/_helpers/roots.rs | 74 ++++ .../src/scope_index/_helpers/visibility.rs | 139 +++++++ .../src/scope_index/constants.rs | 30 ++ .../src/scope_index/main/build_scope_index.rs | 64 +++ .../src/scope_index/main/mod.rs | 3 + .../scope_index/main/relationship_grants.rs | 18 + .../src/scope_index/main/scope_lookup.rs | 79 ++++ crates/sqlbuild-scopes/src/scope_index/mod.rs | 7 + .../sqlbuild-scopes/src/scope_index/models.rs | 388 ++++++++++++++++++ .../sqlbuild-scopes/src/scope_index/tests.rs | 10 + .../src/scope_index/tests/helpers.rs | 228 ++++++++++ .../tests/test_build_scope_index.rs | 250 +++++++++++ .../tests/test_relationship_grants.rs | 87 ++++ .../scope_index/tests/test_scope_lookup.rs | 143 +++++++ .../src/scope_index/tests/test_types.rs | 85 ++++ fensu.toml | 1 + 28 files changed, 2315 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-scopes/Cargo.toml create mode 100644 crates/sqlbuild-scopes/src/lib.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/diagnostics.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/records.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/roots.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/constants.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/build_scope_index.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/mod.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/relationship_grants.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/mod.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/models.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/test_build_scope_index.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/test_relationship_grants.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs create mode 100644 crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs diff --git a/Cargo.lock b/Cargo.lock index 9272e1e774..c8be4c3ca6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,6 +684,7 @@ dependencies = [ "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-rules", + "sqlbuild-scopes", "sqlbuild-sqltext", ] @@ -708,6 +709,10 @@ dependencies = [ "toml", ] +[[package]] +name = "sqlbuild-scopes" +version = "0.139.0" + [[package]] name = "sqlbuild-sqltext" version = "0.139.0" diff --git a/Cargo.toml b/Cargo.toml index 87981cc1b6..062b1353d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/sqlbuild-sqltext", "crates/sqlbuild-config", "crates/sqlbuild-discovery", + "crates/sqlbuild-scopes", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -19,6 +20,7 @@ order = [ "sqlbuild-sqltext", "sqlbuild-config", "sqlbuild-discovery", + "sqlbuild-scopes", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -68,6 +70,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } +sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } stacker = "0.1" sqlparser = { version = "0.58", features = ["visitor"] } diff --git a/crates/README.md b/crates/README.md index 1775579703..c6775425b4 100644 --- a/crates/README.md +++ b/crates/README.md @@ -10,6 +10,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants and the scope lookup groups. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -20,13 +21,13 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-discovery -> sqlbuild-config + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core ``` -`sqlbuild-config` does not depend on the crates below it today; its place in the order only -fixes which crates may use it. The JSON emitter lives in `sqlbuild-core` so that any layer can -produce text that must equal Python's `json.dumps` output. +`sqlbuild-config` and `sqlbuild-scopes` do not depend on the crates below them today; their +place in the order only fixes which crates may use them. The JSON emitter lives in +`sqlbuild-core` so that any layer can produce text that must equal Python's `json.dumps` output. Each crate may depend only on crates below it, and may also skip layers. The order is declared in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and diff --git a/crates/sqlbuild-scopes/Cargo.toml b/crates/sqlbuild-scopes/Cargo.toml new file mode 100644 index 0000000000..acb059c867 --- /dev/null +++ b/crates/sqlbuild-scopes/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "sqlbuild-scopes" +version.workspace = true +description = "Native declaration scopes for SQLBuild: the scope index, visibility and relationship grants." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-scopes/src/lib.rs b/crates/sqlbuild-scopes/src/lib.rs new file mode 100644 index 0000000000..42fe53ff2b --- /dev/null +++ b/crates/sqlbuild-scopes/src/lib.rs @@ -0,0 +1,5 @@ +//! Native declaration scopes: the scope index, its lookup groups, visibility and relationship grants. + +#![forbid(unsafe_code)] + +pub mod scope_index; diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/diagnostics.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/diagnostics.rs new file mode 100644 index 0000000000..94e240c544 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/diagnostics.rs @@ -0,0 +1,162 @@ +//! The builder's duplicate and naming diagnostics, with Python's exact messages and order. + +use crate::scope_index::_helpers::identities::{declaration_text, resource_text}; +use crate::scope_index::_helpers::ordering::{declaration_key, group_in_order, resource_key}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, ResourceRecord, ScopeDiagnostic, +}; +use crate::scope_index::models::{ScopeDiagnosticCode, ScopeKind}; +use std::cmp::Ordering; + +/// Python's builder diagnostics: resource then declaration diagnostics, sorted together. +pub(crate) fn scope_diagnostics( + resources: &[ResourceRecord], + declarations: &[DeclarationRecord], +) -> Vec { + let mut diagnostics: Vec = resource_diagnostics(resources); + diagnostics.extend(declaration_diagnostics(declarations)); + diagnostics.sort_by(compare_diagnostics); + diagnostics +} + +fn resource_diagnostics(resources: &[ResourceRecord]) -> Vec { + let mut diagnostics: Vec = Vec::new(); + let positions: Vec = (0..resources.len()).collect(); + for group in group_in_order(&positions, |index| resources[index].identity.clone()) { + if group.len() <= 1 { + continue; + } + let mut sorted: Vec = group.clone(); + sorted.sort_by_cached_key(|index| resource_key(&resources[*index])); + let locations: String = sorted + .iter() + .map(|index| resources[*index].path.as_str()) + .collect::>() + .join(", "); + for index in group { + let record: &ResourceRecord = &resources[index]; + diagnostics.push(ScopeDiagnostic { + code: ScopeDiagnosticCode::DuplicateResource, + message: format!( + "Duplicate resource '{}' at {locations}", + resource_text(&record.identity) + ), + path: Some(record.path.clone()), + line: Some(1), + column: Some(1), + declaration: None, + resource: Some(index), + }); + } + } + diagnostics.sort_by(compare_diagnostics); + diagnostics +} + +fn declaration_diagnostics(declarations: &[DeclarationRecord]) -> Vec { + let mut diagnostics: Vec = Vec::new(); + let mut public: Vec = Vec::new(); + let mut private: Vec = Vec::new(); + for (index, record) in declarations.iter().enumerate() { + if let Some((code, message)) = naming_problem(record) { + diagnostics.push(declaration_diagnostic(declarations, index, code, message)); + } + if record.identity.owner.is_none() { + public.push(index); + } else { + private.push(index); + } + } + let public_groups: Vec> = group_in_order(&public, |index| { + let identity: &DeclarationIdentity = &declarations[index].identity; + (identity.kind, identity.name.clone()) + }); + let private_groups: Vec> = + group_in_order(&private, |index| declarations[index].identity.clone()); + for group in public_groups.into_iter().chain(private_groups) { + if group.len() <= 1 { + continue; + } + let mut sorted: Vec = group.clone(); + sorted.sort_by_cached_key(|index| declaration_key(&declarations[*index])); + let locations: String = sorted + .iter() + .map(|index| { + let item: &DeclarationRecord = &declarations[*index]; + format!("{}:{}:{}", item.path, item.line, item.column) + }) + .collect::>() + .join(", "); + for index in group { + let message: String = format!( + "Duplicate declaration '{}' at {locations}", + declaration_text(&declarations[index].identity) + ); + diagnostics.push(declaration_diagnostic( + declarations, + index, + ScopeDiagnosticCode::DuplicateDeclaration, + message, + )); + } + } + diagnostics.sort_by(compare_diagnostics); + diagnostics +} + +fn naming_problem(record: &DeclarationRecord) -> Option<(ScopeDiagnosticCode, String)> { + let name: &str = &record.identity.name; + if name.starts_with("__") { + return Some(( + ScopeDiagnosticCode::ReservedDeclarationName, + format!("Declaration name '{name}' uses reserved '__' prefix"), + )); + } + let private: bool = record.scope == ScopeKind::Private; + if private && !name.starts_with('_') { + return Some(( + ScopeDiagnosticCode::InvalidDeclarationName, + format!("Private declaration name '{name}' must have exactly one leading underscore"), + )); + } + if !private && name.starts_with('_') { + return Some(( + ScopeDiagnosticCode::InvalidDeclarationName, + format!("Public declaration name '{name}' must not start with underscore"), + )); + } + None +} + +fn declaration_diagnostic( + declarations: &[DeclarationRecord], + index: usize, + code: ScopeDiagnosticCode, + message: String, +) -> ScopeDiagnostic { + let record: &DeclarationRecord = &declarations[index]; + ScopeDiagnostic { + code, + message, + path: Some(record.path.clone()), + line: Some(record.line), + column: Some(record.column), + declaration: Some(index), + resource: None, + } +} + +/// Python's `_diagnostic_key` ordering. +fn compare_diagnostics(left: &ScopeDiagnostic, right: &ScopeDiagnostic) -> Ordering { + let key = |item: &ScopeDiagnostic| { + ( + item.path.clone().unwrap_or_default(), + item.line.unwrap_or(0), + item.column.unwrap_or(0), + item.code.as_str(), + ) + }; + key(left) + .cmp(&key(right)) + .then_with(|| left.message.cmp(&right.message)) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs new file mode 100644 index 0000000000..cde7244b9a --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs @@ -0,0 +1,192 @@ +//! The grant resolver behind Python's expected-model and tested-macro relationship grants. + +use crate::scope_index::_helpers::paths::{normalize_path, parent}; +use crate::scope_index::_helpers::visibility::{VisibilityIndex, path_visibility}; +use crate::scope_index::constants::{ + CURRENT_PATH_COMPONENT, MACRO_TEST_LEXICAL_FILE, PATH_RESOURCE_PREFIX, PATH_RESOURCE_SUFFIX, +}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, GrantKind, GrantRecord, GrantThrough, + RelationshipFact, ResourceIdentity, ResourceKind, ScopeDeferral, ScopeIndex, ScopeKind, +}; +use std::collections::{HashMap, HashSet}; + +/// Grants in order, each first occurrence kept as Python's `dict.fromkeys` does. +pub(crate) fn unique_grants(index: &ScopeIndex, grants: Vec) -> Vec { + let mut seen: HashSet> = HashSet::new(); + let unique: Vec = grants + .iter() + .map(|grant| seen.insert(grant_key(index, grant))) + .collect(); + drop(seen); + grants + .into_iter() + .zip(unique) + .filter_map(|(grant, unique)| unique.then_some(grant)) + .collect() +} + +type GrantKey<'a> = ( + &'a ResourceIdentity, + &'a DeclarationIdentity, + GrantThroughKey<'a>, + GrantKind, +); + +#[derive(PartialEq, Eq, Hash)] +enum GrantThroughKey<'a> { + Model(&'a str), + Declaration(&'a DeclarationIdentity), +} + +fn grant_key<'a>(index: &'a ScopeIndex, grant: &'a GrantRecord) -> GrantKey<'a> { + ( + &grant.resource, + &index.declarations[grant.declaration].identity, + match &grant.through { + GrantThrough::Model(name) => GrantThroughKey::Model(name), + GrantThrough::Declaration(declaration) => { + GrantThroughKey::Declaration(&index.declarations[*declaration].identity) + } + }, + grant.kind, + ) +} + +/// Visibility and identity lookups over one index, as the Python grant functions use them. +pub(crate) struct GrantResolver<'a> { + index: &'a ScopeIndex, + visibility: VisibilityIndex, + /// Canonical resource records per identity. + resources: HashMap<&'a ResourceIdentity, Vec>, + /// The first canonical declaration record per identity. + first_declarations: HashMap<&'a DeclarationIdentity, usize>, +} + +impl<'a> GrantResolver<'a> { + pub(crate) fn new(index: &'a ScopeIndex) -> Result { + let mut resources: HashMap<&ResourceIdentity, Vec> = HashMap::new(); + for resource in &index.canonical_resources { + resources + .entry(&index.resources[*resource].identity) + .or_default() + .push(*resource); + } + let mut first_declarations: HashMap<&DeclarationIdentity, usize> = HashMap::new(); + for declaration in &index.canonical_declarations { + first_declarations + .entry(&index.declarations[*declaration].identity) + .or_insert(*declaration); + } + Ok(Self { + index, + visibility: VisibilityIndex::build(index)?, + resources, + first_declarations, + }) + } + + /// Python's `resolve_declaration_visibility(...).visible` as first declaration records. + fn visible_declarations( + &self, + resource: &ResourceIdentity, + ) -> Result, ScopeDeferral> { + let mut visible: Vec = Vec::new(); + for record in self.resources.get(resource).map_or(&[][..], Vec::as_slice) { + for position in self + .visibility + .visible_positions(&self.index.resources[*record])? + { + let identity: &DeclarationIdentity = + &self.index.declarations[self.index.canonical_declarations[position]].identity; + visible.push(self.first_declarations[identity]); + } + } + Ok(visible) + } + + /// Python's `_expected_model_grants` for one fact. + pub(crate) fn expected_model_grants( + &self, + fact: &RelationshipFact, + ) -> Result, ScopeDeferral> { + let called: HashSet<&str> = fact.called_macros.iter().map(String::as_str).collect(); + let mut grants: Vec = Vec::new(); + for model_name in &fact.expected_models { + let through: ResourceIdentity = ResourceIdentity { + kind: ResourceKind::Model, + name: model_name.clone(), + }; + for declaration in self.visible_declarations(&through)? { + let record: &DeclarationRecord = &self.index.declarations[declaration]; + if record.scope == ScopeKind::Private + || (record.identity.kind == DeclarationKind::Macro + && !called.contains(record.identity.name.as_str())) + { + continue; + } + grants.push(GrantRecord { + resource: fact.resource.clone(), + declaration, + through: GrantThrough::Model(model_name.clone()), + kind: GrantKind::ExpectedModel, + }); + } + } + Ok(grants) + } + + /// Python's `_tested_macro_grants` for one fact. + pub(crate) fn tested_macro_grants( + &self, + fact: &RelationshipFact, + ) -> Result, ScopeDeferral> { + if fact.tested_macros.is_empty() { + return Ok(Vec::new()); + } + let directly_visible: HashSet<&DeclarationIdentity> = self + .visible_declarations(&fact.resource)? + .into_iter() + .map(|declaration| &self.index.declarations[declaration].identity) + .collect(); + let mut grants: Vec = Vec::new(); + for macro_name in &fact.tested_macros { + let identity: DeclarationIdentity = DeclarationIdentity { + kind: DeclarationKind::Macro, + name: macro_name.clone(), + owner: None, + }; + let Some(tested_macro) = self.first_declarations.get(&identity).copied() else { + continue; + }; + let owning_path: &str = self.index.declarations[tested_macro] + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_PATH_COMPONENT); + let lexical_path: String = + normalize_path(&format!("{owning_path}/{MACRO_TEST_LEXICAL_FILE}"))?; + let path_resource: ResourceIdentity = ResourceIdentity { + kind: ResourceKind::Model, + name: format!("{PATH_RESOURCE_PREFIX}{lexical_path}{PATH_RESOURCE_SUFFIX}"), + }; + let folder: String = parent(&lexical_path)?; + for declaration in &self.index.canonical_declarations { + let record: &DeclarationRecord = &self.index.declarations[*declaration]; + if !path_visibility(record, &path_resource, &folder)? + || record.scope == ScopeKind::Private + || directly_visible.contains(&record.identity) + { + continue; + } + grants.push(GrantRecord { + resource: fact.resource.clone(), + declaration: *declaration, + through: GrantThrough::Declaration(tested_macro), + kind: GrantKind::TestedMacro, + }); + } + } + Ok(grants) + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs new file mode 100644 index 0000000000..12d4640057 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs @@ -0,0 +1,90 @@ +//! Qualified identity text (`format_identity`) and the `repr` text Python orders lookup keys by. + +use crate::scope_index::models::{ + DeclarationIdentity, GrantRecord, GrantThrough, ResourceIdentity, ScopeIndex, +}; + +/// Python's `format_identity` for a resource. +pub(crate) fn resource_text(identity: &ResourceIdentity) -> String { + format!("{}:{}", identity.kind.as_str(), identity.name) +} + +/// Python's `format_identity` for a declaration. +pub(crate) fn declaration_text(identity: &DeclarationIdentity) -> String { + match &identity.owner { + None => format!("{}:{}", identity.kind.as_str(), identity.name), + Some(owner) => format!( + "{}:{}:{}.{}", + identity.kind.as_str(), + owner.kind.as_str(), + owner.name, + identity.name + ), + } +} + +/// Python's `format_identity` of what a grant came through. +pub(crate) fn grant_through_text(index: &ScopeIndex, grant: &GrantRecord) -> String { + match &grant.through { + GrantThrough::Model(name) => format!("model:{name}"), + GrantThrough::Declaration(declaration) => { + declaration_text(&index.declarations[*declaration].identity) + } + } +} + +/// Python's `repr` of a string, or `None` when it holds text whose escaping needs Unicode tables. +pub(crate) fn python_str_repr(value: &str) -> Option { + if !value.is_ascii() { + return None; + } + let quote: char = if value.contains('\'') && !value.contains('"') { + '"' + } else { + '\'' + }; + let mut text: String = String::with_capacity(value.len() + 2); + text.push(quote); + for character in value.chars() { + match character { + '\\' => text.push_str("\\\\"), + '\t' => text.push_str("\\t"), + '\n' => text.push_str("\\n"), + '\r' => text.push_str("\\r"), + _ if character == quote => { + text.push('\\'); + text.push(character); + } + _ if character < ' ' || character == '\x7f' => { + text.push_str(&format!("\\x{:02x}", u32::from(character))); + } + _ => text.push(character), + } + } + text.push(quote); + Some(text) +} + +/// Python's dataclass `repr` of a `ResourceIdentity`. +pub(crate) fn resource_repr(identity: &ResourceIdentity) -> Option { + Some(format!( + "ResourceIdentity(kind=, name={})", + identity.kind.member_name(), + identity.kind.as_str(), + python_str_repr(&identity.name)? + )) +} + +/// Python's dataclass `repr` of a `DeclarationIdentity`. +pub(crate) fn declaration_repr(identity: &DeclarationIdentity) -> Option { + let owner: String = match &identity.owner { + None => "None".to_owned(), + Some(owner) => resource_repr(owner)?, + }; + Some(format!( + "DeclarationIdentity(kind=, name={}, owner={owner})", + identity.kind.member_name(), + identity.kind.as_str(), + python_str_repr(&identity.name)? + )) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs new file mode 100644 index 0000000000..3e418ab8e3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod diagnostics; +pub(crate) mod grants; +pub(crate) mod identities; +pub(crate) mod ordering; +pub(crate) mod paths; +pub(crate) mod records; +pub(crate) mod roots; +pub(crate) mod visibility; diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs new file mode 100644 index 0000000000..f9c566c15a --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs @@ -0,0 +1,99 @@ +//! Python's builder and lookup sort keys, and grouping in Python's mapping order. + +use crate::scope_index::_helpers::identities::{declaration_text, resource_text}; +use crate::scope_index::_helpers::paths::normalize_path; +use crate::scope_index::models::{DeclarationRecord, KeyedGroups, ResourceRecord, ScopeDeferral}; +use std::collections::HashMap; +use std::hash::Hash; + +/// The builder's `_resource_key`. +pub(crate) fn resource_key(record: &ResourceRecord) -> (String, String) { + (record.path.clone(), resource_text(&record.identity)) +} + +/// The builder's `_declaration_key`, which compares line and column as integers. +pub(crate) fn declaration_key(record: &DeclarationRecord) -> (String, String, i64, i64) { + ( + declaration_text(&record.identity), + record.path.clone(), + record.line, + record.column, + ) +} + +/// The lookup's `resource_sort_key`. +pub(crate) fn canonical_resource_key( + record: &ResourceRecord, +) -> Result<(String, String), ScopeDeferral> { + Ok(( + resource_text(&record.identity), + normalize_path(&record.path)?, + )) +} + +/// The lookup's `declaration_sort_key`, which compares line and column as text. +pub(crate) fn canonical_declaration_key( + record: &DeclarationRecord, +) -> Result<(String, String, String, String), ScopeDeferral> { + Ok(( + declaration_text(&record.identity), + normalize_path(&record.path)?, + record.line.to_string(), + record.column.to_string(), + )) +} + +/// Stable sort of `members` by a fallible key, as Python's `sorted(..., key=...)`. +pub(crate) fn sorted_by_key( + members: &[usize], + key: impl Fn(usize) -> Result, +) -> Result, ScopeDeferral> { + let mut keyed: Vec<(K, usize)> = members + .iter() + .map(|member| Ok((key(*member)?, *member))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_key, member)| member).collect()) +} + +/// Members grouped by key, groups in first-occurrence order and members in input order. +pub(crate) fn group_in_order( + members: &[usize], + key: impl Fn(usize) -> K, +) -> Vec> { + let mut positions: HashMap = HashMap::new(); + let mut groups: Vec> = Vec::new(); + for member in members { + let group_key: K = key(*member); + match positions.get(&group_key) { + Some(position) => groups[*position].push(*member), + None => { + positions.insert(group_key, groups.len()); + groups.push(vec![*member]); + } + } + } + groups +} + +/// Python's `group_records`: groups ordered by `repr(key)` when every key's repr is known. +pub(crate) fn keyed_groups( + members: &[usize], + key: impl Fn(usize) -> K, + repr: impl Fn(&K) -> Option, +) -> KeyedGroups { + let groups: Vec> = group_in_order(members, &key); + let reprs: Option> = groups.iter().map(|group| repr(&key(group[0]))).collect(); + let Some(reprs) = reprs else { + return KeyedGroups { + groups, + repr_ordered: false, + }; + }; + let mut ordered: Vec<(String, Vec)> = reprs.into_iter().zip(groups).collect(); + ordered.sort_by(|left, right| left.0.cmp(&right.0)); + KeyedGroups { + groups: ordered.into_iter().map(|(_repr, group)| group).collect(), + repr_ordered: true, + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs new file mode 100644 index 0000000000..38b631e5f0 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs @@ -0,0 +1,74 @@ +//! Python's scope path normalization and the lexical path relations visibility uses. + +use crate::scope_index::constants::{ + CURRENT_PATH_COMPONENT, PARENT_PATH_COMPONENT, PATH_SEPARATOR, +}; +use crate::scope_index::models::ScopeDeferral; + +/// Python's `normalize_path`; a path Python rejects defers the stage so Python raises its error. +pub(crate) fn normalize_path(display_path: &str) -> Result { + let raw: String = display_path.replace('\\', "/"); + let has_drive: bool = raw.chars().nth(1) == Some(':'); + if raw.is_empty() || raw.starts_with(PATH_SEPARATOR) || has_drive { + return Err(deferral(display_path)); + } + let mut components: Vec<&str> = Vec::new(); + for component in raw.split(PATH_SEPARATOR) { + if component.is_empty() || component == CURRENT_PATH_COMPONENT { + continue; + } + if component == PARENT_PATH_COMPONENT { + if components.pop().is_none() { + return Err(deferral(display_path)); + } + continue; + } + components.push(component); + } + if components.is_empty() { + return Ok(CURRENT_PATH_COMPONENT.to_owned()); + } + Ok(components.join("/")) +} + +/// Python's visibility `_parent`: the normalized folder of `path`, or `.` at the project root. +pub(crate) fn parent(path: &str) -> Result { + let normalized: String = normalize_path(path)?; + Ok(match normalized.rfind(PATH_SEPARATOR) { + Some(index) => normalized[..index].to_owned(), + None => CURRENT_PATH_COMPONENT.to_owned(), + }) +} + +/// Python's `_canonical_ancestors`: every owner whose inherited declarations reach `path`. +pub(crate) fn canonical_ancestors(path: &str) -> Vec<&str> { + let mut prefixes: Vec<&str> = vec![CURRENT_PATH_COMPONENT]; + for (index, character) in path.char_indices() { + if character == PATH_SEPARATOR { + prefixes.push(&path[..index]); + } + } + prefixes.push(path); + let mut unique: Vec<&str> = Vec::with_capacity(prefixes.len()); + for prefix in prefixes { + if !unique.contains(&prefix) { + unique.push(prefix); + } + } + unique +} + +/// Python's `_is_canonical_descendant`. +pub(crate) fn is_canonical_descendant(path: &str, ancestor: &str) -> bool { + ancestor == CURRENT_PATH_COMPONENT + || path == ancestor + || path + .strip_prefix(ancestor) + .is_some_and(|rest| rest.starts_with(PATH_SEPARATOR)) +} + +fn deferral(display_path: &str) -> ScopeDeferral { + ScopeDeferral { + reason: format!("scope path Python rejects: {display_path}"), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/records.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/records.rs new file mode 100644 index 0000000000..dae9b1fc41 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/records.rs @@ -0,0 +1,55 @@ +//! Builder records from walked inputs, and the macro dependency usages between them. + +use crate::scope_index::_helpers::paths::normalize_path; +use crate::scope_index::_helpers::roots::{declaration_root, resource_root}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationInput, DeclarationRecord, ResourceInput, ResourceRecord, + ScopeDeferral, UsageRecord, +}; +use std::collections::HashSet; + +/// Python's `_resource_record` and the seed and Python function records. +pub(crate) fn resource_record(input: ResourceInput) -> Result { + Ok(ResourceRecord { + ownership_root: resource_root(input.identity.kind, input.root)?, + path: normalize_path(&input.path)?, + identity: input.identity, + }) +} + +/// Python's declaration records, before value metadata the facade attaches. +pub(crate) fn declaration_record( + input: DeclarationInput, +) -> Result { + Ok(DeclarationRecord { + ownership_root: declaration_root(input.ownership_root.as_deref(), &input.root_fallback)?, + path: normalize_path(&input.path)?, + owning_path: input + .owning_path + .as_deref() + .map(normalize_path) + .transpose()?, + identity: input.identity, + line: input.line, + column: 1, + scope: input.scope, + dependencies: input.dependencies, + }) +} + +/// Python's `_macro_dependency_usages`: every dependency edge once, in declaration input order. +pub(crate) fn dependency_usages(declarations: &[DeclarationRecord]) -> Vec { + let mut seen: HashSet<(&DeclarationIdentity, &DeclarationIdentity)> = HashSet::new(); + let mut usages: Vec = Vec::new(); + for (consumer, record) in declarations.iter().enumerate() { + for (dependency, identity) in record.dependencies.iter().enumerate() { + if seen.insert((&record.identity, identity)) { + usages.push(UsageRecord { + consumer, + dependency, + }); + } + } + } + usages +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/roots.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/roots.rs new file mode 100644 index 0000000000..d8086c74b5 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/roots.rs @@ -0,0 +1,74 @@ +//! Ownership roots exactly as the Python builder's `_root` and resource roots derive them. + +use crate::scope_index::_helpers::paths::normalize_path; +use crate::scope_index::constants::{ + GLOBAL_DECLARATION_DIRECTORIES, PYTHON_FUNCTION_ROOT, RESOURCE_ROOTS, SEED_ROOT, +}; +use crate::scope_index::models::{OwnershipRoot, ScopeDeferral}; +use crate::scope_index::models::{OwnershipRootKind, ResourceKind, ResourceRoot}; + +/// The root a resource of `kind` belongs to. +pub(crate) fn resource_root( + kind: ResourceKind, + root: ResourceRoot, +) -> Result { + Ok(match root { + ResourceRoot::Kind => OwnershipRoot { + path: kind_root(kind) + .ok_or_else(|| ScopeDeferral { + reason: format!("no resource root for {}", kind.as_str()), + })? + .to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(kind), + }, + ResourceRoot::Seed => OwnershipRoot { + path: SEED_ROOT.to_owned(), + kind: OwnershipRootKind::Global, + resource_kind: Some(ResourceKind::Seed), + }, + ResourceRoot::PythonFunction => OwnershipRoot { + path: PYTHON_FUNCTION_ROOT.to_owned(), + kind: OwnershipRootKind::Global, + resource_kind: Some(ResourceKind::Function), + }, + }) +} + +/// Python's `_root(path=..., fallback=...)`; the fallback is normalized as the named roles do. +pub(crate) fn declaration_root( + path: Option<&str>, + fallback: &str, +) -> Result { + let fallback: String = normalize_path(fallback)?; + let Some(path) = path else { + return Ok(OwnershipRoot { + path: fallback, + kind: OwnershipRootKind::Global, + resource_kind: None, + }); + }; + let normalized: String = normalize_path(path)?; + let is_global: bool = GLOBAL_DECLARATION_DIRECTORIES.contains(&normalized.as_str()); + let resource_kind: Option = RESOURCE_ROOTS + .iter() + .find(|(_kind, root)| *root == normalized) + .map(|(kind, _root)| *kind); + Ok(OwnershipRoot { + path: normalized, + kind: if is_global { + OwnershipRootKind::Global + } else { + OwnershipRootKind::Resource + }, + resource_kind, + }) +} + +/// The root path of a resource kind, or `None` for seeds, which only have a global root. +pub(crate) fn kind_root(kind: ResourceKind) -> Option<&'static str> { + RESOURCE_ROOTS + .iter() + .find(|(root_kind, _path)| *root_kind == kind) + .map(|(_kind, path)| *path) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs new file mode 100644 index 0000000000..d9e33f22ee --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs @@ -0,0 +1,139 @@ +//! Python's `build_visibility_index`, `_visible_positions` and `_visibility_reason`. + +use crate::scope_index::_helpers::paths::{ + canonical_ancestors, is_canonical_descendant, normalize_path, parent, +}; +use crate::scope_index::constants::CURRENT_PATH_COMPONENT; +use crate::scope_index::models::ScopeKind; +use crate::scope_index::models::{ + DeclarationRecord, ResourceIdentity, ResourceRecord, ScopeDeferral, ScopeIndex, + VisibilityGroups, +}; +use std::collections::HashMap; +use std::hash::Hash; + +/// Canonical declaration positions per key, keys in first-occurrence order. +struct OrderedPositions { + entries: Vec<(K, Vec)>, + lookup: HashMap, +} + +impl OrderedPositions { + fn new() -> Self { + Self { + entries: Vec::new(), + lookup: HashMap::new(), + } + } + + fn push(&mut self, key: K, position: usize) { + match self.lookup.get(&key) { + Some(entry) => self.entries[*entry].1.push(position), + None => { + self.lookup.insert(key.clone(), self.entries.len()); + self.entries.push((key, vec![position])); + } + } + } + + fn get(&self, key: &K) -> &[usize] { + self.lookup + .get(key) + .map_or(&[], |entry| self.entries[*entry].1.as_slice()) + } + + fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +/// Canonical declaration positions grouped by the scope facts that make them visible. +pub(crate) struct VisibilityIndex { + global: Vec, + private: OrderedPositions, + local: OrderedPositions, + inherited: OrderedPositions, +} + +impl VisibilityIndex { + /// Python's `build_visibility_index` over the canonical declarations. + pub(crate) fn build(index: &ScopeIndex) -> Result { + let mut visibility: Self = Self { + global: Vec::new(), + private: OrderedPositions::new(), + local: OrderedPositions::new(), + inherited: OrderedPositions::new(), + }; + for (position, declaration_index) in index.canonical_declarations.iter().enumerate() { + let declaration: &DeclarationRecord = &index.declarations[*declaration_index]; + match declaration.scope { + ScopeKind::Global => visibility.global.push(position), + ScopeKind::Private => { + if let Some(owner) = &declaration.identity.owner { + visibility.private.push(owner.clone(), position); + } + } + ScopeKind::Local => visibility.local.push(owner_path(declaration)?, position), + ScopeKind::Inherited => { + visibility + .inherited + .push(owner_path(declaration)?, position); + } + } + } + Ok(visibility) + } + + /// Python's `_visible_positions`: canonical positions a resource sees, in position order. + pub(crate) fn visible_positions( + &self, + resource: &ResourceRecord, + ) -> Result, ScopeDeferral> { + let mut positions: Vec = self.global.clone(); + positions.extend_from_slice(self.private.get(&resource.identity)); + if !self.local.is_empty() || !self.inherited.is_empty() { + let folder: String = parent(&resource.path)?; + positions.extend_from_slice(self.local.get(&folder)); + for ancestor in canonical_ancestors(&folder) { + positions.extend_from_slice(self.inherited.get(&ancestor.to_owned())); + } + } + positions.sort_unstable(); + Ok(positions) + } + + /// The positions as Python's `DeclarationVisibilityIndex` mappings hold them. + pub(crate) fn groups(&self) -> VisibilityGroups { + VisibilityGroups { + global: self.global.clone(), + private: self.private.entries.clone(), + local: self.local.entries.clone(), + inherited: self.inherited.entries.clone(), + } + } +} + +/// Python's `_visibility_reason(...) is not None` for a synthetic path resource. +pub(crate) fn path_visibility( + declaration: &DeclarationRecord, + path_resource: &ResourceIdentity, + folder: &str, +) -> Result { + Ok(match declaration.scope { + ScopeKind::Global => true, + ScopeKind::Private => declaration.identity.owner.as_ref() == Some(path_resource), + ScopeKind::Local => folder == owner_path(declaration)?, + ScopeKind::Inherited => is_canonical_descendant(folder, &owner_path(declaration)?), + }) +} + +/// Python's `_declaration_owner`. +fn owner_path(declaration: &DeclarationRecord) -> Result { + normalize_path( + declaration + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_PATH_COMPONENT), + ) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/constants.rs b/crates/sqlbuild-scopes/src/scope_index/constants.rs new file mode 100644 index 0000000000..117a8ccc14 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/constants.rs @@ -0,0 +1,30 @@ +//! Scope layout names, mirroring the Python scope builder and `sqlbuild.compiler.scopes.constants`. + +use crate::scope_index::models::ResourceKind; + +/// Python's builder `_ROOTS`, in its insertion order. +pub const RESOURCE_ROOTS: [(ResourceKind, &str); 5] = [ + (ResourceKind::Model, "models"), + (ResourceKind::Test, "tests/unit"), + (ResourceKind::Scenario, "tests/scenarios"), + (ResourceKind::Function, "functions/sql"), + (ResourceKind::Source, "sources"), +]; +/// The global root of seed resources. +pub const SEED_ROOT: &str = "seeds"; +/// The global root of Python function resources. +pub const PYTHON_FUNCTION_ROOT: &str = "functions/python"; +/// Declaration directories whose ownership root is global. +pub const GLOBAL_DECLARATION_DIRECTORIES: [&str; 3] = ["macros", "enums", "constants"]; +/// Python's `PATH_SEPARATOR`. +pub const PATH_SEPARATOR: char = '/'; +/// Python's `CURRENT_PATH_COMPONENT`. +pub const CURRENT_PATH_COMPONENT: &str = "."; +/// Python's `PARENT_PATH_COMPONENT`. +pub const PARENT_PATH_COMPONENT: &str = ".."; +/// The file name a tested macro's folder is resolved through. +pub const MACRO_TEST_LEXICAL_FILE: &str = "__macro_test__.sql"; +/// The prefix of the synthetic model identity a lexical path resolves as. +pub const PATH_RESOURCE_PREFIX: &str = " Result { + let resources: Vec = inputs + .resources + .into_iter() + .map(resource_record) + .collect::>()?; + let declarations: Vec = inputs + .declarations + .into_iter() + .map(declaration_record) + .collect::>()?; + let resource_indexes: Vec = (0..resources.len()).collect(); + let declaration_indexes: Vec = (0..declarations.len()).collect(); + let resource_order: Vec = sorted_by_key(&resource_indexes, |index| { + Ok(resource_key(&resources[index])) + })?; + let declaration_order: Vec = sorted_by_key(&declaration_indexes, |index| { + Ok(declaration_key(&declarations[index])) + })?; + let canonical_resources: Vec = sorted_by_key(&resource_order, |index| { + canonical_resource_key(&resources[index]) + })?; + let canonical_declarations: Vec = sorted_by_key(&declaration_order, |index| { + canonical_declaration_key(&declarations[index]) + })?; + let usages: Vec = dependency_usages(&declarations); + let diagnostics = scope_diagnostics(&resources, &declarations); + let has_scoped_relationship_declarations: bool = declarations.iter().any(|declaration| { + declaration.scope != ScopeKind::Global + && matches!( + declaration.identity.kind, + DeclarationKind::Enum | DeclarationKind::Constant | DeclarationKind::Macro + ) + }); + let index: ScopeIndex = ScopeIndex { + resources, + declarations, + resource_order, + declaration_order, + canonical_resources, + canonical_declarations, + usages, + diagnostics, + has_scoped_relationship_declarations, + }; + VisibilityIndex::build(&index)?; + Ok(index) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs new file mode 100644 index 0000000000..2d04f761b5 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -0,0 +1,3 @@ +pub mod build_scope_index; +pub mod relationship_grants; +pub mod scope_lookup; diff --git a/crates/sqlbuild-scopes/src/scope_index/main/relationship_grants.rs b/crates/sqlbuild-scopes/src/scope_index/main/relationship_grants.rs new file mode 100644 index 0000000000..128aa738e7 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/relationship_grants.rs @@ -0,0 +1,18 @@ +//! Expected-model and tested-macro grants, as Python's `build_scope_relationship_grants` makes them. + +use crate::scope_index::_helpers::grants::{GrantResolver, unique_grants}; +use crate::scope_index::models::{GrantRecord, RelationshipFact, ScopeDeferral, ScopeIndex}; + +/// The relationship grants of every fact, in Python's order and deduplicated like `dict.fromkeys`. +pub fn relationship_grants( + index: &ScopeIndex, + facts: &[RelationshipFact], +) -> Result, ScopeDeferral> { + let resolver: GrantResolver<'_> = GrantResolver::new(index)?; + let mut grants: Vec = Vec::new(); + for fact in facts { + grants.extend(resolver.expected_model_grants(fact)?); + grants.extend(resolver.tested_macro_grants(fact)?); + } + Ok(unique_grants(index, grants)) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs b/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs new file mode 100644 index 0000000000..9c96d417e2 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs @@ -0,0 +1,79 @@ +//! The groupings Python's `build_lookup` derives from an index and its relationship grants. + +use crate::scope_index::_helpers::identities::{ + declaration_repr, declaration_text, grant_through_text, python_str_repr, resource_repr, + resource_text, +}; +use crate::scope_index::_helpers::ordering::{keyed_groups, sorted_by_key}; +use crate::scope_index::_helpers::paths::normalize_path; +use crate::scope_index::_helpers::visibility::VisibilityIndex; +use crate::scope_index::models::{ + DeclarationIdentity, GrantRecord, ScopeDeferral, ScopeIndex, ScopeLookupGroups, +}; + +/// Every lookup grouping of `index` once `grants` are attached to it. +pub fn scope_lookup_groups( + index: &ScopeIndex, + grants: &[GrantRecord], +) -> Result { + let dependency = |usage: usize| -> &DeclarationIdentity { + let record = index.usages[usage]; + &index.declarations[record.consumer].dependencies[record.dependency] + }; + let consumer = |usage: usize| -> &DeclarationIdentity { + &index.declarations[index.usages[usage].consumer].identity + }; + let usage_indexes: Vec = (0..index.usages.len()).collect(); + let canonical_usages: Vec = sorted_by_key(&usage_indexes, |usage| { + Ok(( + declaration_text(consumer(usage)), + declaration_text(dependency(usage)), + )) + })?; + let grant_indexes: Vec = (0..grants.len()).collect(); + let canonical_grants: Vec = sorted_by_key(&grant_indexes, |grant| { + let record: &GrantRecord = &grants[grant]; + Ok(( + resource_text(&record.resource), + declaration_text(&index.declarations[record.declaration].identity), + grant_through_text(index, record), + record.kind.as_str(), + )) + })?; + let path_keys: Vec = index + .resources + .iter() + .map(|record| normalize_path(&record.path)) + .collect::>()?; + Ok(ScopeLookupGroups { + resources: keyed_groups( + &index.canonical_resources, + |resource| &index.resources[resource].identity, + |identity| resource_repr(identity), + ), + resources_by_path: keyed_groups( + &index.canonical_resources, + |resource| path_keys[resource].as_str(), + |path| python_str_repr(path), + ), + declarations: keyed_groups( + &index.canonical_declarations, + |declaration| &index.declarations[declaration].identity, + |identity| declaration_repr(identity), + ), + usages_by_consumer: keyed_groups(&canonical_usages, consumer, |identity| { + declaration_repr(identity) + }), + usages_by_declaration: keyed_groups(&canonical_usages, dependency, |identity| { + declaration_repr(identity) + }), + grants_by_resource: keyed_groups( + &canonical_grants, + |grant| &grants[grant].resource, + |identity| resource_repr(identity), + ), + visibility: VisibilityIndex::build(index)?.groups(), + canonical_usages, + canonical_grants, + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/mod.rs b/crates/sqlbuild-scopes/src/scope_index/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/scope_index/models.rs b/crates/sqlbuild-scopes/src/scope_index/models.rs new file mode 100644 index 0000000000..ac712864bf --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/models.rs @@ -0,0 +1,388 @@ +//! Scope enums, inputs, records and lookup groups exchanged with the Python scope facade. + +/// Python's `ResourceKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ResourceKind { + Model, + Test, + Scenario, + Function, + Source, + Seed, +} + +impl ResourceKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Test => "test", + Self::Scenario => "scenario", + Self::Function => "function", + Self::Source => "source", + Self::Seed => "seed", + } + } + + /// Python's enum member name, as `repr` prints it. + pub fn member_name(self) -> &'static str { + match self { + Self::Model => "MODEL", + Self::Test => "TEST", + Self::Scenario => "SCENARIO", + Self::Function => "FUNCTION", + Self::Source => "SOURCE", + Self::Seed => "SEED", + } + } + + /// The kind with Python's enum value `value`. + pub fn parse(value: &str) -> Option { + [ + Self::Model, + Self::Test, + Self::Scenario, + Self::Function, + Self::Source, + Self::Seed, + ] + .into_iter() + .find(|kind| kind.as_str() == value) + } +} + +/// Python's `DeclarationKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DeclarationKind { + Macro, + Enum, + Constant, + Audit, + SingularAudit, + Schema, + SqlHook, + PythonHook, +} + +impl DeclarationKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Macro => "macro", + Self::Enum => "enum", + Self::Constant => "constant", + Self::Audit => "audit", + Self::SingularAudit => "singular_audit", + Self::Schema => "schema", + Self::SqlHook => "sql_hook", + Self::PythonHook => "python_hook", + } + } + + /// Python's enum member name, as `repr` prints it. + pub fn member_name(self) -> &'static str { + match self { + Self::Macro => "MACRO", + Self::Enum => "ENUM", + Self::Constant => "CONSTANT", + Self::Audit => "AUDIT", + Self::SingularAudit => "SINGULAR_AUDIT", + Self::Schema => "SCHEMA", + Self::SqlHook => "SQL_HOOK", + Self::PythonHook => "PYTHON_HOOK", + } + } + + /// The kind with Python's enum value `value`. + pub fn parse(value: &str) -> Option { + [ + Self::Macro, + Self::Enum, + Self::Constant, + Self::Audit, + Self::SingularAudit, + Self::Schema, + Self::SqlHook, + Self::PythonHook, + ] + .into_iter() + .find(|kind| kind.as_str() == value) + } +} + +/// Python's `ScopeKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ScopeKind { + Global, + Inherited, + Local, + Private, +} + +impl ScopeKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Global => "global", + Self::Inherited => "inherited", + Self::Local => "local", + Self::Private => "private", + } + } + + /// The scope with Python's enum value `value`. + pub fn parse(value: &str) -> Option { + [Self::Global, Self::Inherited, Self::Local, Self::Private] + .into_iter() + .find(|scope| scope.as_str() == value) + } +} + +/// Python's `OwnershipRootKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OwnershipRootKind { + Resource, + Global, +} + +impl OwnershipRootKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::Resource => "resource", + Self::Global => "global", + } + } +} + +/// Python's `GrantKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum GrantKind { + ExpectedModel, + TestedMacro, +} + +impl GrantKind { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::ExpectedModel => "expected_model", + Self::TestedMacro => "tested_macro", + } + } +} + +/// The `ScopeDiagnosticCode`s the static index builder reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ScopeDiagnosticCode { + DuplicateDeclaration, + InvalidDeclarationName, + ReservedDeclarationName, + DuplicateResource, +} + +impl ScopeDiagnosticCode { + /// Python's enum value. + pub fn as_str(self) -> &'static str { + match self { + Self::DuplicateDeclaration => "S003", + Self::InvalidDeclarationName => "S004", + Self::ReservedDeclarationName => "S005", + Self::DuplicateResource => "S018", + } + } +} + +/// Which ownership root an authored resource belongs to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResourceRoot { + /// The resource kind's own root, such as `models` or `tests/unit`. + Kind, + /// The global `seeds` root. + Seed, + /// The global `functions/python` root. + PythonFunction, +} + +/// Python's `ResourceIdentity`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ResourceIdentity { + pub kind: ResourceKind, + pub name: String, +} + +/// Python's `DeclarationIdentity`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct DeclarationIdentity { + pub kind: DeclarationKind, + pub name: String, + pub owner: Option, +} + +/// Python's `OwnershipRoot`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct OwnershipRoot { + pub path: String, + pub kind: OwnershipRootKind, + pub resource_kind: Option, +} + +/// One authored resource as the Python builder walks it, with its path as `str(Path)` prints it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceInput { + pub identity: ResourceIdentity, + pub path: String, + pub root: ResourceRoot, +} + +/// One declaration as the Python builder walks it, with paths as `str(Path)` prints them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationInput { + pub identity: DeclarationIdentity, + pub path: String, + pub line: i64, + pub scope: ScopeKind, + /// The discovered ownership root, when discovery recorded one. + pub ownership_root: Option, + /// The global root used without a discovered ownership root. + pub root_fallback: String, + pub owning_path: Option, + /// A loaded macro's declaration dependencies; empty for every other declaration. + pub dependencies: Vec, +} + +/// The builder's inputs in Python's walk order: resources, then declarations. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopeInputs { + pub resources: Vec, + pub declarations: Vec, +} + +/// Python's `ResourceRecord`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceRecord { + pub identity: ResourceIdentity, + pub path: String, + pub ownership_root: OwnershipRoot, +} + +/// Python's `DeclarationRecord` without its value metadata, which the facade keeps. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationRecord { + pub identity: DeclarationIdentity, + pub path: String, + pub line: i64, + pub column: i64, + pub scope: ScopeKind, + pub ownership_root: OwnershipRoot, + pub owning_path: Option, + pub dependencies: Vec, +} + +/// A macro declaration dependency: `declarations[consumer].dependencies[dependency]`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UsageRecord { + pub consumer: usize, + pub dependency: usize, +} + +/// Python's `ScopeDiagnostic`; `declaration` and `resource` index the index's records. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeDiagnostic { + pub code: ScopeDiagnosticCode, + pub message: String, + pub path: Option, + pub line: Option, + pub column: Option, + pub declaration: Option, + pub resource: Option, +} + +/// The static scope index with every ordering the Python builder and lookup produce. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopeIndex { + /// Resource records in input order. + pub resources: Vec, + /// Declaration records in input order. + pub declarations: Vec, + /// The builder's `ScopeIndex.resources` order. + pub resource_order: Vec, + /// The builder's `ScopeIndex.declarations` order. + pub declaration_order: Vec, + /// The lookup's canonical resource order. + pub canonical_resources: Vec, + /// The lookup's canonical declaration order; visibility positions index this list. + pub canonical_declarations: Vec, + /// The builder's deduplicated macro dependency usages. + pub usages: Vec, + /// The builder's sorted diagnostics. + pub diagnostics: Vec, + /// Whether any macro, enum or constant is scoped, which makes relationship grants matter. + pub has_scoped_relationship_declarations: bool, +} + +/// The relationship facts the Python facade extracted for one test block or scenario. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RelationshipFact { + pub resource: ResourceIdentity, + /// `__expected__` names in authored order. + pub expected_models: Vec, + /// Macro names the block calls, including nested calls; empty for scenarios. + pub called_macros: Vec, + /// Macros called in a macro test's `__actual__` CTE, in encounter order. + pub tested_macros: Vec, +} + +/// What a grant came through: an expected model, or a tested macro declaration. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum GrantThrough { + Model(String), + Declaration(usize), +} + +/// Python's `GrantRecord`; `declaration` indexes the index's declaration records. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GrantRecord { + pub resource: ResourceIdentity, + pub declaration: usize, + pub through: GrantThrough, + pub kind: GrantKind, +} + +/// One lookup mapping's groups; members index the records the mapping groups. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct KeyedGroups { + pub groups: Vec>, + /// Whether `groups` follow Python's `repr(key)` order; otherwise first-occurrence order. + pub repr_ordered: bool, +} + +/// Python's `DeclarationVisibilityIndex` positions over the canonical declarations. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct VisibilityGroups { + pub global: Vec, + /// Private positions per owner, in first-occurrence order. + pub private: Vec<(ResourceIdentity, Vec)>, + pub local: Vec<(String, Vec)>, + pub inherited: Vec<(String, Vec)>, +} + +/// Every grouping Python's `build_lookup` derives from one index and its grants. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopeLookupGroups { + pub canonical_usages: Vec, + pub canonical_grants: Vec, + pub resources: KeyedGroups, + pub resources_by_path: KeyedGroups, + pub declarations: KeyedGroups, + pub usages_by_consumer: KeyedGroups, + pub usages_by_declaration: KeyedGroups, + pub grants_by_resource: KeyedGroups, + pub visibility: VisibilityGroups, +} + +/// The native scope stage cannot reproduce Python for this project; the Python stage runs instead. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeDeferral { + pub reason: String, +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs new file mode 100644 index 0000000000..6ee4a5e471 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/test_build_scope_index.rs"] +mod build_scope_index; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_relationship_grants.rs"] +mod relationship_grants; +#[path = "tests/test_scope_lookup.rs"] +mod scope_lookup; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs new file mode 100644 index 0000000000..ebc938bdfb --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs @@ -0,0 +1,228 @@ +use crate::scope_index::_helpers::identities::{ + declaration_text, grant_through_text, resource_text, +}; +use crate::scope_index::main::build_scope_index::build_scope_index; +use crate::scope_index::main::relationship_grants::relationship_grants; +use crate::scope_index::main::scope_lookup::scope_lookup_groups; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationInput, DeclarationKind, RelationshipFact, ResourceIdentity, + ResourceInput, ResourceKind, ResourceRoot, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, + ScopeLookupGroups, +}; +use crate::scope_index::tests::test_types::{ + DeclarationRow, FactRow, GrantTestCase, IndexFactsTestCase, LookupTestCase, ResourceRow, +}; + +const GLOBAL_ROOT_FALLBACKS: [(DeclarationKind, &str); 3] = [ + (DeclarationKind::Macro, "macros"), + (DeclarationKind::Enum, "enums"), + (DeclarationKind::Constant, "constants"), +]; +const ROOTS: [(&str, ResourceRoot); 1] = [("seed", ResourceRoot::Seed)]; + +/// Scope inputs as the Python facade walks them for these resource and declaration rows. +pub(super) fn scope_inputs( + resources: &[ResourceRow], + declarations: &[DeclarationRow], +) -> ScopeInputs { + ScopeInputs { + resources: resources.iter().map(resource_input).collect(), + declarations: declarations.iter().map(declaration_input).collect(), + } +} + +/// The native index of these rows, or its deferral. +pub(super) fn index_outcome( + resources: &[ResourceRow], + declarations: &[DeclarationRow], +) -> Result { + build_scope_index(scope_inputs(resources, declarations)) +} + +/// The index of these rows, which must not defer. +pub(super) fn built_index( + resources: &[ResourceRow], + declarations: &[DeclarationRow], +) -> ScopeIndex { + index_outcome(resources, declarations).expect("no deferral") +} + +/// Qualified resource identities in `order`. +pub(super) fn resource_names(index: &ScopeIndex, order: &[usize]) -> Vec { + order + .iter() + .map(|resource| resource_text(&index.resources[*resource].identity)) + .collect() +} + +/// `path:line` of the declarations in `order`. +pub(super) fn declaration_locations(index: &ScopeIndex, order: &[usize]) -> Vec { + order + .iter() + .map(|declaration| { + let record = &index.declarations[*declaration]; + format!("{}:{}", record.path, record.line) + }) + .collect() +} + +/// `code message` of every diagnostic, in order. +pub(super) fn diagnostic_texts(index: &ScopeIndex) -> Vec { + index + .diagnostics + .iter() + .map(|item| format!("{} {}", item.code.as_str(), item.message)) + .collect() +} + +/// The index of a facts case, whose first declaration depends on the case's dependencies. +pub(super) fn index_with_dependencies(test_case: &IndexFactsTestCase) -> ScopeIndex { + let mut inputs: ScopeInputs = scope_inputs(&[], test_case.declarations); + inputs.declarations[0].dependencies = test_case + .dependencies + .iter() + .map(|name| DeclarationIdentity { + kind: DeclarationKind::Macro, + name: (*name).to_owned(), + owner: None, + }) + .collect(); + build_scope_index(inputs).expect("no deferral") +} + +/// `consumer -> dependency` text of every usage, in order. +pub(super) fn usage_texts(index: &ScopeIndex) -> Vec { + index + .usages + .iter() + .map(|usage| { + let consumer = &index.declarations[usage.consumer]; + format!( + "{} -> {}", + declaration_text(&consumer.identity), + declaration_text(&consumer.dependencies[usage.dependency]) + ) + }) + .collect() +} + +/// `resource declaration through kind` text of every grant a case's facts produce. +pub(super) fn grant_texts(test_case: &GrantTestCase) -> Vec { + let index: ScopeIndex = built_index(test_case.resources, test_case.declarations); + let facts: Vec = test_case.facts.iter().map(relationship_fact).collect(); + relationship_grants(&index, &facts) + .expect("no deferral") + .iter() + .map(|grant| { + format!( + "{} {} {} {}", + resource_text(&grant.resource), + declaration_text(&index.declarations[grant.declaration].identity), + grant_through_text(&index, grant), + grant.kind.as_str() + ) + }) + .collect() +} + +/// The first resource identity of each resource group of a lookup case, and its repr flag. +pub(super) fn resource_groups(test_case: &LookupTestCase) -> (Vec, bool) { + let index: ScopeIndex = built_index(test_case.resources, test_case.declarations); + let groups: ScopeLookupGroups = scope_lookup_groups(&index, &[]).expect("no deferral"); + ( + groups + .resources + .groups + .iter() + .map(|group| resource_text(&index.resources[group[0]].identity)) + .collect(), + groups.resources.repr_ordered, + ) +} + +/// The visibility position groups of a lookup case as `kind owner positions` text. +pub(super) fn visibility_texts(test_case: &LookupTestCase) -> Vec { + let index: ScopeIndex = built_index(test_case.resources, test_case.declarations); + let visibility = scope_lookup_groups(&index, &[]) + .expect("no deferral") + .visibility; + let mut texts: Vec = vec![format!("global {:?}", visibility.global)]; + texts.extend( + visibility + .private + .iter() + .map(|(owner, positions)| format!("private {} {positions:?}", resource_text(owner))), + ); + texts.extend( + visibility + .local + .iter() + .map(|(owner, positions)| format!("local {owner} {positions:?}")), + ); + texts.extend( + visibility + .inherited + .iter() + .map(|(owner, positions)| format!("inherited {owner} {positions:?}")), + ); + texts +} + +fn resource_input(row: &ResourceRow) -> ResourceInput { + let (kind, name, path) = *row; + ResourceInput { + identity: resource(kind, name), + path: path.to_owned(), + root: ROOTS + .iter() + .find(|(root_kind, _root)| *root_kind == kind) + .map_or(ResourceRoot::Kind, |(_kind, root)| *root), + } +} + +fn declaration_input(row: &DeclarationRow) -> DeclarationInput { + let (kind, name, path, line, scope, owning_path) = *row; + let scope: ScopeKind = ScopeKind::parse(scope).expect("scope kind"); + let kind: DeclarationKind = DeclarationKind::parse(kind).expect("declaration kind"); + let stem: &str = path.rsplit('/').next().unwrap_or(path); + DeclarationInput { + identity: DeclarationIdentity { + kind, + name: name.to_owned(), + owner: (scope == ScopeKind::Private) + .then(|| resource("model", stem.trim_end_matches(".sql"))), + }, + path: path.to_owned(), + line, + scope, + ownership_root: (scope != ScopeKind::Global).then(|| "models".to_owned()), + root_fallback: GLOBAL_ROOT_FALLBACKS + .iter() + .find(|(fallback_kind, _root)| *fallback_kind == kind) + .map_or("macros", |(_kind, root)| *root) + .to_owned(), + owning_path: owning_path.map(str::to_owned), + dependencies: Vec::new(), + } +} + +fn relationship_fact(row: &FactRow) -> RelationshipFact { + let (name, expected, called, tested) = *row; + RelationshipFact { + resource: resource("test", name), + expected_models: owned(expected), + called_macros: owned(called), + tested_macros: owned(tested), + } +} + +fn resource(kind: &str, name: &str) -> ResourceIdentity { + ResourceIdentity { + kind: ResourceKind::parse(kind).expect("resource kind"), + name: name.to_owned(), + } +} + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_build_scope_index.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_build_scope_index.rs new file mode 100644 index 0000000000..85ae161e5f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_build_scope_index.rs @@ -0,0 +1,250 @@ +use crate::scope_index::tests::helpers::{ + built_index, declaration_locations, diagnostic_texts, index_outcome, index_with_dependencies, + resource_names, usage_texts, +}; +use crate::scope_index::tests::test_types::{DeferralTestCase, IndexFactsTestCase, IndexTestCase}; + +#[test] +fn given_authored_projects_when_building_then_orders_and_diagnostics_match_python() { + let test_cases = [ + IndexTestCase { + description: "builder sorts resources by path, the lookup by identity", + resources: &[ + ("model", "zeta", "models/a/zeta.sql"), + ("model", "alpha", "models/b/./alpha.sql"), + ("test", "alpha", "tests/unit/alpha.sql"), + ("seed", "lookup", "seeds/lookup.csv"), + ], + declarations: &[ + ("macro", "pick", "macros/b.py", 10, "global", None), + ("macro", "pick_more", "macros/a.py", 9, "global", None), + ("constant", "limit", "constants/a.sql", 1, "global", None), + ], + expected_resource_order: &["model:zeta", "model:alpha", "seed:lookup", "test:alpha"], + expected_canonical_resources: &[ + "model:alpha", + "model:zeta", + "seed:lookup", + "test:alpha", + ], + expected_declaration_order: &["constants/a.sql:1", "macros/b.py:10", "macros/a.py:9"], + expected_canonical_declarations: &[ + "constants/a.sql:1", + "macros/b.py:10", + "macros/a.py:9", + ], + expected_diagnostics: &[], + }, + IndexTestCase { + description: "duplicates and naming problems keep Python's messages and order", + resources: &[ + ("model", "orders", "models/b/orders.sql"), + ("model", "orders", "models/a/orders.sql"), + ], + declarations: &[ + ( + "constant", + "rate", + "models/a/_constants/x.sql", + 1, + "local", + Some("models/a"), + ), + ("constant", "rate", "constants/rates.sql", 1, "global", None), + ("enum", "__status", "enums/status.sql", 1, "global", None), + ("enum", "_status", "enums/other.sql", 1, "global", None), + ( + "constant", + "bad", + "models/a/orders.sql", + 1, + "private", + Some("models/a"), + ), + ], + expected_resource_order: &["model:orders", "model:orders"], + expected_canonical_resources: &["model:orders", "model:orders"], + expected_declaration_order: &[ + "models/a/orders.sql:1", + "constants/rates.sql:1", + "models/a/_constants/x.sql:1", + "enums/status.sql:1", + "enums/other.sql:1", + ], + expected_canonical_declarations: &[ + "models/a/orders.sql:1", + "constants/rates.sql:1", + "models/a/_constants/x.sql:1", + "enums/status.sql:1", + "enums/other.sql:1", + ], + expected_diagnostics: &[ + "S003 Duplicate declaration 'constant:rate' at constants/rates.sql:1:1, \ + models/a/_constants/x.sql:1:1", + "S004 Public declaration name '_status' must not start with underscore", + "S005 Declaration name '__status' uses reserved '__' prefix", + "S003 Duplicate declaration 'constant:rate' at constants/rates.sql:1:1, \ + models/a/_constants/x.sql:1:1", + "S004 Private declaration name 'bad' must have exactly one leading underscore", + "S018 Duplicate resource 'model:orders' at models/a/orders.sql, models/b/orders.sql", + "S018 Duplicate resource 'model:orders' at models/a/orders.sql, models/b/orders.sql", + ], + }, + IndexTestCase { + description: "duplicate lines sort as integers in the builder and as text in the lookup", + resources: &[], + declarations: &[ + ("macro", "pick", "macros/a.py", 10, "global", None), + ("macro", "pick", "macros/a.py", 9, "global", None), + ], + expected_resource_order: &[], + expected_canonical_resources: &[], + expected_declaration_order: &["macros/a.py:9", "macros/a.py:10"], + expected_canonical_declarations: &["macros/a.py:10", "macros/a.py:9"], + expected_diagnostics: &[ + "S003 Duplicate declaration 'macro:pick' at macros/a.py:9:1, macros/a.py:10:1", + "S003 Duplicate declaration 'macro:pick' at macros/a.py:9:1, macros/a.py:10:1", + ], + }, + ]; + + for test_case in test_cases { + let index = built_index(test_case.resources, test_case.declarations); + + assert_eq!( + resource_names(&index, &index.resource_order), + test_case.expected_resource_order, + "{}", + test_case.description + ); + assert_eq!( + resource_names(&index, &index.canonical_resources), + test_case.expected_canonical_resources, + "{}", + test_case.description + ); + assert_eq!( + declaration_locations(&index, &index.declaration_order), + test_case.expected_declaration_order, + "{}", + test_case.description + ); + assert_eq!( + declaration_locations(&index, &index.canonical_declarations), + test_case.expected_canonical_declarations, + "{}", + test_case.description + ); + assert_eq!( + diagnostic_texts(&index), + test_case.expected_diagnostics, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_paths_python_rejects_when_building_then_the_stage_defers() { + let test_cases = [ + DeferralTestCase { + description: "absolute resource path", + resources: &[("model", "a", "/models/a.sql")], + declarations: &[], + expected_deferral: true, + }, + DeferralTestCase { + description: "drive-qualified resource path", + resources: &[("model", "a", "C:/models/a.sql")], + declarations: &[], + expected_deferral: true, + }, + DeferralTestCase { + description: "resource path escaping the project", + resources: &[("model", "a", "../models/a.sql")], + declarations: &[], + expected_deferral: true, + }, + DeferralTestCase { + description: "owning path escaping the project", + resources: &[], + declarations: &[( + "macro", + "pick", + "models/_macros/a.py", + 1, + "local", + Some("models/../.."), + )], + expected_deferral: true, + }, + DeferralTestCase { + description: "dot segments normalize like Python", + resources: &[("model", "a", "models/./b/../a.sql")], + declarations: &[], + expected_deferral: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + index_outcome(test_case.resources, test_case.declarations).is_err(), + test_case.expected_deferral, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_declarations_when_building_then_relationship_flag_and_usages_match_python() { + let test_cases = [ + IndexFactsTestCase { + description: "global declarations never need relationship grants", + declarations: &[("macro", "pick", "macros/a.py", 1, "global", None)], + dependencies: &[], + expected_scoped_relationships: false, + expected_usages: &[], + }, + IndexFactsTestCase { + description: "a scoped macro needs relationship grants", + declarations: &[( + "macro", + "pick", + "models/a/_macros/a.py", + 1, + "local", + Some("models/a"), + )], + dependencies: &[], + expected_scoped_relationships: true, + expected_usages: &[], + }, + IndexFactsTestCase { + description: "repeated dependencies become one usage each, in order", + declarations: &[ + ("macro", "outer", "macros/a.py", 1, "global", None), + ("macro", "inner", "macros/a.py", 5, "global", None), + ], + dependencies: &["inner", "first", "inner"], + expected_scoped_relationships: false, + expected_usages: &["macro:outer -> macro:inner", "macro:outer -> macro:first"], + }, + ]; + + for test_case in test_cases { + let index = index_with_dependencies(&test_case); + + assert_eq!( + index.has_scoped_relationship_declarations, test_case.expected_scoped_relationships, + "{}", + test_case.description + ); + assert_eq!( + usage_texts(&index), + test_case.expected_usages, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_relationship_grants.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_relationship_grants.rs new file mode 100644 index 0000000000..9b1ba52f74 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_relationship_grants.rs @@ -0,0 +1,87 @@ +use crate::scope_index::tests::helpers::grant_texts; +use crate::scope_index::tests::test_types::{DeclarationRow, GrantTestCase, ResourceRow}; + +const RESOURCES: &[ResourceRow] = &[ + ("model", "orders", "models/sales/orders.sql"), + ("model", "other", "models/other/other.sql"), + ("test", "t1", "tests/unit/t1.sql"), + ("test", "t2", "tests/unit/t2.sql"), +]; +const DECLARATIONS: &[DeclarationRow] = &[ + ( + "constant", + "rate", + "models/sales/_sqlbuild/constants/r.sql", + 1, + "inherited", + Some("models/sales"), + ), + ( + "macro", + "fmt", + "models/sales/_macros/m.py", + 3, + "local", + Some("models/sales"), + ), + ( + "macro", + "unused", + "models/sales/_macros/m.py", + 9, + "local", + Some("models/sales"), + ), + ("macro", "g", "macros/g.py", 1, "global", None), + ( + "constant", + "_own", + "models/sales/orders.sql", + 1, + "private", + Some("models/sales"), + ), +]; + +#[test] +fn given_relationship_facts_when_granting_then_grants_match_python() { + let test_cases = [ + GrantTestCase { + description: "expected models grant visible non-private declarations; macros if called", + resources: RESOURCES, + declarations: DECLARATIONS, + facts: &[("t1", &["orders", "missing", "orders"], &["fmt"], &[])], + expected_grants: &[ + "test:t1 constant:rate model:orders expected_model", + "test:t1 macro:fmt model:orders expected_model", + ], + }, + GrantTestCase { + description: "tested macros grant their folder's declarations the test cannot see", + resources: RESOURCES, + declarations: DECLARATIONS, + facts: &[("t2", &[], &[], &["fmt", "unknown"])], + expected_grants: &[ + "test:t2 constant:rate macro:fmt tested_macro", + "test:t2 macro:fmt macro:fmt tested_macro", + "test:t2 macro:unused macro:fmt tested_macro", + ], + }, + GrantTestCase { + description: "a model without scoped neighbours grants only called global macros", + resources: RESOURCES, + declarations: DECLARATIONS, + facts: &[("t1", &["other"], &["g"], &[])], + expected_grants: &["test:t1 macro:g model:other expected_model"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + grant_texts(&test_case), + test_case.expected_grants, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs new file mode 100644 index 0000000000..3265fb915e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs @@ -0,0 +1,143 @@ +use crate::scope_index::_helpers::identities::python_str_repr; +use crate::scope_index::tests::helpers::{resource_groups, visibility_texts}; +use crate::scope_index::tests::test_types::{LookupTestCase, ReprTestCase}; + +#[test] +fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { + let test_cases = [ + LookupTestCase { + description: "ASCII keys order by repr, where quotes and punctuation sort first", + resources: &[ + ("model", "ab", "models/ab.sql"), + ("model", "ab!", "models/ab_bang.sql"), + ("model", "it's", "models/its.sql"), + ("function", "zz", "functions/sql/zz.sql"), + ("seed", "a", "seeds/a.csv"), + ], + declarations: &[], + expected_resource_groups: &[ + "function:zz", + "model:it's", + "model:ab!", + "model:ab", + "seed:a", + ], + expected_repr_ordered: true, + expected_visibility: &["global []"], + }, + LookupTestCase { + description: "non-ASCII keys are left for Python to order", + resources: &[("model", "caf\u{e9}", "models/cafe.sql")], + declarations: &[], + expected_resource_groups: &["model:caf\u{e9}"], + expected_repr_ordered: false, + expected_visibility: &["global []"], + }, + LookupTestCase { + description: "visibility positions group by scope over the canonical declarations", + resources: &[("model", "orders", "models/sales/orders.sql")], + declarations: &[ + ("macro", "g", "macros/g.py", 1, "global", None), + ( + "macro", + "l", + "models/sales/_macros/l.py", + 1, + "local", + Some("models/sales"), + ), + ( + "enum", + "i", + "models/_sqlbuild/enums/i.sql", + 1, + "inherited", + Some("models"), + ), + ( + "constant", + "_p", + "models/sales/orders.sql", + 1, + "private", + Some("models/sales"), + ), + ], + expected_resource_groups: &["model:orders"], + expected_repr_ordered: true, + expected_visibility: &[ + "global [2]", + "private model:orders [0]", + "local models/sales [3]", + "inherited models [1]", + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + resource_groups(&test_case), + ( + test_case + .expected_resource_groups + .iter() + .map(|name| (*name).to_owned()) + .collect(), + test_case.expected_repr_ordered + ), + "{}", + test_case.description + ); + assert_eq!( + visibility_texts(&test_case), + test_case.expected_visibility, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_key_text_when_taking_repr_then_it_matches_python() { + let test_cases = [ + ReprTestCase { + description: "plain text uses single quotes", + value: "plain", + expected_repr: Some("'plain'"), + }, + ReprTestCase { + description: "a single quote alone switches to double quotes", + value: "it's", + expected_repr: Some("\"it's\""), + }, + ReprTestCase { + description: "both quotes keep single quotes and escape them", + value: "both'\"", + expected_repr: Some("'both\\'\"'"), + }, + ReprTestCase { + description: "whitespace controls and backslashes use short escapes", + value: "tab\tnew\nline\r\\", + expected_repr: Some("'tab\\tnew\\nline\\r\\\\'"), + }, + ReprTestCase { + description: "other controls use hex escapes", + value: "bell\u{7}del\u{7f}", + expected_repr: Some("'bell\\x07del\\x7f'"), + }, + ReprTestCase { + description: "non-ASCII text is left to Python", + value: "caf\u{e9}", + expected_repr: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + python_str_repr(test_case.value).as_deref(), + test_case.expected_repr, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs new file mode 100644 index 0000000000..e5d4b76d04 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -0,0 +1,85 @@ +/// `(kind, name, path)` of one resource in walk order. +pub(super) type ResourceRow = (&'static str, &'static str, &'static str); + +/// `(kind, name, path, line, scope, owning path)` of one declaration in walk order. +pub(super) type DeclarationRow = ( + &'static str, + &'static str, + &'static str, + i64, + &'static str, + Option<&'static str>, +); + +/// `(test name, expected models, called macros, tested macros)`. +pub(super) type FactRow = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [&'static str], +); + +/// One scope-index case: authored resources and declarations, then the expected projection. +pub(super) struct IndexTestCase { + pub(super) description: &'static str, + pub(super) resources: &'static [ResourceRow], + pub(super) declarations: &'static [DeclarationRow], + /// Resource identities in the builder's order. + pub(super) expected_resource_order: &'static [&'static str], + /// Resource identities in the lookup's canonical order. + pub(super) expected_canonical_resources: &'static [&'static str], + /// `path:line` of declarations in the builder's order. + pub(super) expected_declaration_order: &'static [&'static str], + /// `path:line` of declarations in the lookup's canonical order. + pub(super) expected_canonical_declarations: &'static [&'static str], + /// `code message` diagnostics in the builder's order. + pub(super) expected_diagnostics: &'static [&'static str], +} + +/// One case whose only expectation is whether the native stage defers to Python. +pub(super) struct DeferralTestCase { + pub(super) description: &'static str, + pub(super) resources: &'static [ResourceRow], + pub(super) declarations: &'static [DeclarationRow], + pub(super) expected_deferral: bool, +} + +/// One case for the relationship flag and the macro dependency usages. +pub(super) struct IndexFactsTestCase { + pub(super) description: &'static str, + pub(super) declarations: &'static [DeclarationRow], + /// Dependency names of the first declaration, which must be a macro. + pub(super) dependencies: &'static [&'static str], + pub(super) expected_scoped_relationships: bool, + /// `consumer -> dependency` usages in the builder's order. + pub(super) expected_usages: &'static [&'static str], +} + +/// One relationship case: a scoped project, the extracted facts and the expected grants. +pub(super) struct GrantTestCase { + pub(super) description: &'static str, + pub(super) resources: &'static [ResourceRow], + pub(super) declarations: &'static [DeclarationRow], + pub(super) facts: &'static [FactRow], + /// `resource declaration through kind` qualified identities. + pub(super) expected_grants: &'static [&'static str], +} + +/// One lookup case: resources and declarations, the resource groups and visibility positions. +pub(super) struct LookupTestCase { + pub(super) description: &'static str, + pub(super) resources: &'static [ResourceRow], + pub(super) declarations: &'static [DeclarationRow], + /// First resource identity of each resource group, in mapping order. + pub(super) expected_resource_groups: &'static [&'static str], + pub(super) expected_repr_ordered: bool, + /// `global`, `private`, `local` and `inherited` position groups as text. + pub(super) expected_visibility: &'static [&'static str], +} + +/// One Python `repr` case for a lookup key string. +pub(super) struct ReprTestCase { + pub(super) description: &'static str, + pub(super) value: &'static str, + pub(super) expected_repr: Option<&'static str>, +} diff --git a/fensu.toml b/fensu.toml index ae8ef366a2..32db96582f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -310,6 +310,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-python", "sqlbuild-rules", + "sqlbuild-scopes", "sqlbuild-sqltext", ] From 6370616494223900d30cfe71e967851c73eac637 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:38:46 +0100 Subject: [PATCH 2/7] feat(native): resolve declaration scopes natively behind the compiler engine switch With the native engine, the scope index, its validation, relationship grants and the scope lookup are built by sqlbuild-scopes and materialised as the same Python records, orders and mappings. Python keeps the relationship-name extraction and every error message: an invalid index or a relationship fault raises exactly as before, and a project the native index cannot reproduce runs the Python stage. sqb scope uses the same path. The Python engine is unchanged. --- crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/catalog.rs | 2 +- .../src/bindings/_helpers/functions.rs | 6 +- .../src/bindings/_helpers/mod.rs | 2 +- .../_helpers/normalization_results.rs | 17 - .../src/bindings/_helpers/panics.rs | 15 + .../src/bindings/_helpers/scopes.rs | 338 ++++++++++++++++++ src/sqlbuild/_native.pyi | 59 +++ .../_helpers/attachment/declaration_scope.py | 64 +++- .../attachment/scope_relationships.py | 110 ++++++ .../compile/_helpers/render/declarations.py | 4 +- .../compiler/scopes/_helpers/builder.py | 73 ++-- .../compiler/scopes/classes/__init__.py | 1 + .../scopes/classes/native_scope_index.py | 298 +++++++++++++++ .../scopes/classes/native_scope_rows.py | 323 +++++++++++++++++ .../scopes/main/_open_native_scope_index.py | 32 ++ src/sqlbuild/compiler/scopes/models.py | 23 ++ src/sqlbuild/compiler/scopes/types.py | 15 + .../src/sqlbuild/compiler/scopes/__init__.py | 0 .../sqlbuild/compiler/scopes/_test_types.py | 35 ++ .../src/sqlbuild/compiler/scopes/helpers.py | 318 ++++++++++++++++ .../scopes/test_native_scope_parity.py | 165 +++++++++ 22 files changed, 1851 insertions(+), 50 deletions(-) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scopes.rs create mode 100644 src/sqlbuild/compiler/scopes/classes/__init__.py create mode 100644 src/sqlbuild/compiler/scopes/classes/native_scope_index.py create mode 100644 src/sqlbuild/compiler/scopes/classes/native_scope_rows.py create mode 100644 src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py create mode 100644 tests/integration/src/sqlbuild/compiler/scopes/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/scopes/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/scopes/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 0b4f3a971a..84f69c98dc 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -25,6 +25,7 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-rules.workspace = true +sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true [lints] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs index 8adfad3d40..7235d925b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs @@ -1,8 +1,8 @@ //! Python methods of the compile-owned binding catalog and its compact analysis jobs. use crate::bindings::_helpers::conversions; -use crate::bindings::_helpers::normalization_results::normalization_results; use crate::bindings::_helpers::panics::compiler_error; +use crate::bindings::_helpers::panics::normalization_results; use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; use crate::bindings::types::CompilerDetach; use pyo3::exceptions::PyValueError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs index 50af8a6f98..6bfc7c7c37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs @@ -56,10 +56,7 @@ fn normalize_analysis_sqls( ), ) }); - crate::bindings::_helpers::normalization_results::normalization_results( - py, - results.map_err(value_error)?, - ) + crate::bindings::_helpers::panics::normalization_results(py, results.map_err(value_error)?) } #[pyfunction] @@ -595,6 +592,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; crate::bindings::_helpers::oracles::register(module)?; crate::bindings::_helpers::discovery::register(module)?; + crate::bindings::_helpers::scopes::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8c2763913b..4bb1f63809 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -4,7 +4,7 @@ mod config_values; pub(crate) mod conversions; mod discovery; pub(crate) mod functions; -mod normalization_results; mod oracles; pub(crate) mod panics; mod positions; +mod scopes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs deleted file mode 100644 index 88cb1a632a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs +++ /dev/null @@ -1,17 +0,0 @@ -use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; - -/// Return normalized SQL in order, carrying each failure as the exception a single call raises. -pub(crate) fn normalization_results( - py: Python<'_>, - results: Vec>, -) -> PyResult>> { - results - .into_iter() - .map(|result| match result { - Ok(sql) => Ok(sql.into_pyobject(py)?.into_any().unbind()), - Err(error) => Ok(crate::bindings::_helpers::panics::compiler_error(error) - .into_value(py) - .into_any()), - }) - .collect() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/panics.rs b/crates/sqlbuild-python/src/bindings/_helpers/panics.rs index 623dfb3b53..50e7e2f9de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/panics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/panics.rs @@ -2,6 +2,7 @@ use pyo3::Python; use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult}; use pyo3::{PyErr, create_exception}; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; @@ -38,3 +39,17 @@ impl CompilerDetach for Python<'_> { self.detach(|| catch_compiler_panic(operation)) } } + +/// Return normalized SQL in order, carrying each failure as the exception a single call raises. +pub(crate) fn normalization_results( + py: Python<'_>, + results: Vec>, +) -> PyResult>> { + results + .into_iter() + .map(|result| match result { + Ok(sql) => Ok(sql.into_pyobject(py)?.into_any().unbind()), + Err(error) => Ok(compiler_error(error).into_value(py).into_any()), + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes.rs new file mode 100644 index 0000000000..30c9aab065 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes.rs @@ -0,0 +1,338 @@ +//! Native declaration scopes as the plain rows the Python scope facade materialises. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{IntoPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; +use sqlbuild_scopes::scope_index::main::scope_lookup::scope_lookup_groups; +use sqlbuild_scopes::scope_index::models::{ + DeclarationIdentity, DeclarationInput, DeclarationRecord, GrantRecord, GrantThrough, + KeyedGroups, RelationshipFact, ResourceIdentity, ResourceInput, ScopeDeferral, ScopeDiagnostic, + ScopeIndex, ScopeInputs, ScopeLookupGroups, +}; +use sqlbuild_scopes::scope_index::models::{ + DeclarationKind, ResourceKind, ResourceRoot, ScopeKind, +}; + +use crate::bindings::_helpers::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +type IdentityRow = (String, String); +type DeclarationIdentityRow = (String, String, Option); +type ResourceRow = (String, String, String, String); +type DeclarationRow = ( + String, + String, + Option, + String, + i64, + String, + Option, + String, + Option, + Vec, +); +type FactRow = (String, String, Vec, Vec, Vec); +type DeclarationRecordRow = ( + String, + String, + &'static str, + Option<&'static str>, + Option, +); +type DiagnosticRow = ( + &'static str, + String, + Option, + Option, + Option, + Option, + Option, +); +type IndexRows = ( + Vec, + Vec, + Vec, + Vec, + Vec<(usize, usize)>, + Vec, + bool, +); +type GrantRow = (&'static str, String, usize, ThroughRow, &'static str); +type GroupRows = (Vec>, bool); +type VisibilityRows = ( + Vec, + Vec<(IdentityRow, Vec)>, + Vec<(String, Vec)>, + Vec<(String, Vec)>, +); +type LookupRows = ( + Vec, + Vec, + Vec, + Vec, + ( + GroupRows, + GroupRows, + GroupRows, + GroupRows, + GroupRows, + GroupRows, + ), + VisibilityRows, +); + +/// What a grant came through: an expected model's name or a tested macro's declaration index. +#[derive(IntoPyObject)] +enum ThroughRow { + Model(String), + Declaration(usize), +} + +/// A built scope index held natively between the facade's index, grant and lookup steps. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct NativeScopeIndex { + index: ScopeIndex, + grants: Vec, +} + +#[pymethods] +impl NativeScopeIndex { + /// The index's records, builder orders, usages, diagnostics and relationship flag. + fn records(&self) -> IndexRows { + let index: &ScopeIndex = &self.index; + ( + index + .resources + .iter() + .map(|record| record.path.clone()) + .collect(), + index.declarations.iter().map(declaration_row).collect(), + index.resource_order.clone(), + index.declaration_order.clone(), + index + .usages + .iter() + .map(|usage| (usage.consumer, usage.dependency)) + .collect(), + index.diagnostics.iter().map(diagnostic_row).collect(), + index.has_scoped_relationship_declarations, + ) + } + + /// Resolve and keep the relationship grants of `facts`; `None` when Python must run. + fn grant(&mut self, py: Python<'_>, facts: Vec) -> PyResult>> { + let Some(facts) = facts + .into_iter() + .map(relationship_fact) + .collect::>>() + else { + return Ok(None); + }; + let index: &ScopeIndex = &self.index; + let grants: Result, ScopeDeferral> = py + .compiler_detach(|| Ok(relationship_grants(index, &facts))) + .map_err(compiler_error)?; + let Ok(grants) = grants else { + return Ok(None); + }; + let rows: Vec = grants.iter().map(grant_row).collect(); + self.grants = grants; + Ok(Some(rows)) + } + + /// The lookup orders and groups of the index with its kept grants; `None` when Python must run. + fn lookup(&self, py: Python<'_>) -> PyResult> { + let index: &ScopeIndex = &self.index; + let grants: &[GrantRecord] = &self.grants; + let groups: Result = py + .compiler_detach(|| Ok(scope_lookup_groups(index, grants))) + .map_err(compiler_error)?; + let Ok(groups) = groups else { + return Ok(None); + }; + Ok(Some(( + index.canonical_resources.clone(), + index.canonical_declarations.clone(), + groups.canonical_usages, + groups.canonical_grants, + ( + group_rows(groups.resources), + group_rows(groups.resources_by_path), + group_rows(groups.declarations), + group_rows(groups.usages_by_consumer), + group_rows(groups.usages_by_declaration), + group_rows(groups.grants_by_resource), + ), + ( + groups.visibility.global, + groups + .visibility + .private + .into_iter() + .map(|(owner, positions)| { + ((owner.kind.as_str().to_owned(), owner.name), positions) + }) + .collect(), + groups.visibility.local, + groups.visibility.inherited, + ), + ))) + } +} + +/// Build the static scope index natively, or return `None` when Python must build it. +#[pyfunction] +fn build_native_scope_index( + py: Python<'_>, + resources: Vec, + declarations: Vec, +) -> PyResult> { + let Some(inputs) = scope_inputs(resources, declarations) else { + return Ok(None); + }; + let index: Result = py + .compiler_detach(|| Ok(build_scope_index(inputs))) + .map_err(compiler_error)?; + match index { + Ok(index) => Ok(Some(NativeScopeIndex { + index, + grants: Vec::new(), + })), + Err(_deferral) => Ok(None), + } +} + +fn scope_inputs( + resources: Vec, + declarations: Vec, +) -> Option { + Some(ScopeInputs { + resources: resources + .into_iter() + .map(|(kind, name, path, root)| { + Some(ResourceInput { + identity: resource_identity((kind, name))?, + path, + root: match root.as_str() { + "kind" => ResourceRoot::Kind, + "seed" => ResourceRoot::Seed, + "python_function" => ResourceRoot::PythonFunction, + _ => return None, + }, + }) + }) + .collect::>()?, + declarations: declarations + .into_iter() + .map(declaration_input) + .collect::>()?, + }) +} + +fn declaration_input(row: DeclarationRow) -> Option { + let ( + kind, + name, + owner, + path, + line, + scope, + ownership_root, + root_fallback, + owning_path, + dependencies, + ) = row; + Some(DeclarationInput { + identity: declaration_identity((kind, name, owner))?, + path, + line, + scope: ScopeKind::parse(&scope)?, + ownership_root, + root_fallback, + owning_path, + dependencies: dependencies + .into_iter() + .map(declaration_identity) + .collect::>()?, + }) +} + +fn resource_identity((kind, name): IdentityRow) -> Option { + Some(ResourceIdentity { + kind: ResourceKind::parse(&kind)?, + name, + }) +} + +fn declaration_identity( + (kind, name, owner): DeclarationIdentityRow, +) -> Option { + Some(DeclarationIdentity { + kind: DeclarationKind::parse(&kind)?, + name, + owner: match owner { + Some(owner) => Some(resource_identity(owner)?), + None => None, + }, + }) +} + +fn relationship_fact( + (kind, name, expected_models, called_macros, tested_macros): FactRow, +) -> Option { + Some(RelationshipFact { + resource: resource_identity((kind, name))?, + expected_models, + called_macros, + tested_macros, + }) +} + +fn declaration_row(record: &DeclarationRecord) -> DeclarationRecordRow { + ( + record.path.clone(), + record.ownership_root.path.clone(), + record.ownership_root.kind.as_str(), + record + .ownership_root + .resource_kind + .map(ResourceKind::as_str), + record.owning_path.clone(), + ) +} + +fn diagnostic_row(diagnostic: &ScopeDiagnostic) -> DiagnosticRow { + ( + diagnostic.code.as_str(), + diagnostic.message.clone(), + diagnostic.path.clone(), + diagnostic.line, + diagnostic.column, + diagnostic.declaration, + diagnostic.resource, + ) +} + +fn grant_row(grant: &GrantRecord) -> GrantRow { + ( + grant.resource.kind.as_str(), + grant.resource.name.clone(), + grant.declaration, + match &grant.through { + GrantThrough::Model(name) => ThroughRow::Model(name.clone()), + GrantThrough::Declaration(declaration) => ThroughRow::Declaration(*declaration), + }, + grant.kind.as_str(), + ) +} + +fn group_rows(groups: KeyedGroups) -> GroupRows { + (groups.groups, groups.repr_ordered) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_native_scope_index, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5b66d1ea59..ee7a3037fd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -68,6 +68,65 @@ def discover_declaration_layout( ] | None ): ... + +class NativeScopeIndex: + def records( + self, + ) -> tuple[ + list[str], + list[tuple[str, str, str, str | None, str | None]], + list[int], + list[int], + list[tuple[int, int]], + list[tuple[str, str, str | None, int | None, int | None, int | None, int | None]], + bool, + ]: ... + def grant( + self, facts: list[tuple[str, str, list[str], list[str], list[str]]] + ) -> list[tuple[str, str, int, str | int, str]] | None: ... + def lookup( + self, + ) -> ( + tuple[ + list[int], + list[int], + list[int], + list[int], + tuple[ + tuple[list[list[int]], bool], + tuple[list[list[int]], bool], + tuple[list[list[int]], bool], + tuple[list[list[int]], bool], + tuple[list[list[int]], bool], + tuple[list[list[int]], bool], + ], + tuple[ + list[int], + list[tuple[tuple[str, str], list[int]]], + list[tuple[str, list[int]]], + list[tuple[str, list[int]]], + ], + ] + | None + ): ... + +def build_native_scope_index( + resources: list[tuple[str, str, str, str]], + declarations: list[ + tuple[ + str, + str, + tuple[str, str] | None, + str, + int, + str, + str | None, + str, + str | None, + list[tuple[str, str, tuple[str, str] | None]], + ] + ], +) -> NativeScopeIndex | None: ... def substitute_static_project_vars( sqls: list[str], variables: list[tuple[str, str]] ) -> list[tuple[int, str | None]]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 53ffdb65c0..331bccd8ed 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -10,6 +10,7 @@ from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( build_scope_relationship_grants, + extract_scope_relationship_facts, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( build_declaration_scope_resolver, @@ -21,10 +22,14 @@ ScopeRelationshipBuild, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index +from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.scopes.models import RelationshipFact, ScopeIndex, ScopeLookup from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -38,6 +43,15 @@ def build_declaration_scope( ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" + if resolve_compiler_engine() is CompilerEngine.NATIVE: + native_scope: DeclarationScopeBuild | None = _build_native_declaration_scope( + discovered_inputs=discovered_inputs, + loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, + ) + if native_scope is not None: + return native_scope index: ScopeIndex = build_scope_index( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) @@ -80,6 +94,54 @@ def build_declaration_scope( ) +def _build_native_declaration_scope( + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None, +) -> DeclarationScopeBuild | None: + """Build the scope natively with Python's errors and order, or None for the Python stage.""" + + native: NativeScopeIndex | None = open_native_scope_index( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + ) + if native is None: + return None + try: + validate_scope_index(index=native.index) + except ScopeValidationError as error: + raise CompileInputError(str(error)) from error + if native.has_scoped_relationship_declarations and ( + discovered_inputs.test_files or discovered_inputs.scenario_files + ): + facts: tuple[RelationshipFact, ...] + fault: str | None + facts, fault = extract_scope_relationship_facts( + discovered_inputs=discovered_inputs, + sql_lexical_syntax=sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, + ) + if fault is not None: + raise CompileInputError(fault) + if not native.grant(facts): + return None + index: ScopeIndex = native.index_with_relationships() + lookup: ScopeLookup | None = native.lookup(index=index) + if lookup is None: + return None + return DeclarationScopeBuild( + loaded_macros=loaded_macros, + index=index, + resolver=build_declaration_scope_resolver( + discovered_inputs=discovered_inputs, + scope_index=index, + loaded_macros=loaded_macros, + lookup=lookup, + ), + ) + + def rebind_declaration_scope( *, scope: DeclarationScopeBuild, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index b1f921b905..cc93264a23 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -41,6 +41,7 @@ DeclarationRecord, DeclarationVisibility, GrantRecord, + RelationshipFact, ResourceIdentity, ResourceRecord, ScopeIndex, @@ -87,6 +88,115 @@ def build_scope_relationship_grants( ) +def extract_scope_relationship_facts( + *, + discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, +) -> tuple[tuple[RelationshipFact, ...], str | None]: + """Return the relationship names grants are resolved from, and the first extraction fault.""" + + facts: list[RelationshipFact] = [] + faults: list[ScopeRelationshipFault] = [] + with FactCacheStore( + root=compile_cache_dir, + namespace=SQL_TEST_FACT_CACHE_NAMESPACE, + algorithm=SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, + ) as fact_cache: + cache_keys: dict[int, str] = ( + { + file_index: _expected_models_fact_key( + test_file=test_file, fact_cache=fact_cache, syntax=sql_lexical_syntax + ) + for file_index, test_file in enumerate(discovered_inputs.test_files) + } + if fact_cache.enabled + else {} + ) + cached_names: dict[str, object] = fact_cache.read_many( + tuple( + (_expected_models_fact_slot(discovered_inputs.test_files[file_index]), cache_key) + for file_index, cache_key in cache_keys.items() + ) + ) + for file_index, test_file in enumerate(discovered_inputs.test_files): + cache_key: str | None = cache_keys.get(file_index) + cached_file_names: tuple[tuple[str, ...], ...] | None = _cached_expected_names( + value=None if cache_key is None else cached_names.get(cache_key), + block_count=len(test_file.blocks), + ) + extracted_names: list[tuple[str, ...]] = [] + for block_index, block in enumerate(test_file.blocks): + try: + expected_names: tuple[str, ...] = ( + cached_file_names[block_index] + if cached_file_names is not None + else extract_sql_test_expected_model_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=sql_lexical_syntax, + mode=block.mode, + ) + ) + extracted_names.append(expected_names) + called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) + facts.append( + RelationshipFact( + resource=ResourceIdentity( + ResourceKind.TEST, block.name or test_file.relative_path.stem + ), + expected_models=expected_names, + called_macros=called_macros, + tested_macros=( + _tested_macro_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=sql_lexical_syntax, + ) + if block.mode is SqlTestMode.MACRO + else () + ), + ) + ) + except Exception as error: + faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) + if ( + cache_key is not None + and cached_file_names is None + and len(extracted_names) == len(test_file.blocks) + ): + fact_cache.stage( + key=cache_key, + slot=_expected_models_fact_slot(test_file), + value=tuple(extracted_names), + ) + for scenario in discovered_inputs.scenario_files: + try: + facts.append( + RelationshipFact( + resource=ResourceIdentity(ResourceKind.SCENARIO, scenario.name), + expected_models=extract_sql_scenario_expected_model_names( + sql=scenario.sql_body, + file_label=str(scenario.relative_path), + syntax=sql_lexical_syntax, + ), + ) + ) + except Exception as error: + faults.append(ScopeRelationshipFault(scenario.relative_path, str(error))) + return tuple(facts), faults[0].message if faults else None + + +def _tested_macro_names(*, sql: str, file_label: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: + test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( + sql=sql, file_label=file_label, syntax=syntax + ) + actual_cte: CompileSqlTestCte | None = next( + (cte for cte in test_ctes if cte.name == MACRO_ACTUAL_TEST_CTE_NAME), None + ) + return () if actual_cte is None else find_macro_call_names(actual_cte.sql_body) + + def _test_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index c9678e7cb9..8f8c6eca41 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -48,6 +48,7 @@ ResourceIdentity, ResourceRecord, ScopeIndex, + ScopeLookup, UsageRecord, VisibilityRecord, ) @@ -115,6 +116,7 @@ def build_declaration_scope_resolver( discovered_inputs: DiscoveredProjectInputs, scope_index: ScopeIndex, loaded_macros: Mapping[str, LoadedMacro] | None = None, + lookup: ScopeLookup | None = None, ) -> DeclarationScopeResolver: """Pair the serializable static index with original process-local declaration values.""" @@ -152,7 +154,7 @@ def build_declaration_scope_resolver( declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro return DeclarationScopeResolver( project_dir=discovered_inputs.project_dir, - lookup=build_scope_lookup(index=scope_index), + lookup=build_scope_lookup(index=scope_index) if lookup is None else lookup, projection=DeclarationRuntimeProjection(declarations=MappingProxyType(declarations)), resource_specific=frozenset( declaration.identity.owner diff --git a/src/sqlbuild/compiler/scopes/_helpers/builder.py b/src/sqlbuild/compiler/scopes/_helpers/builder.py index 95a8235ef7..454752abb4 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/builder.py +++ b/src/sqlbuild/compiler/scopes/_helpers/builder.py @@ -71,6 +71,9 @@ ResourceKind.FUNCTION: "functions/sql", ResourceKind.SOURCE: "sources", } +_COLLECTION_KINDS: frozenset[SqlValueKind] = frozenset( + {SqlValueKind.LIST, SqlValueKind.SET, SqlValueKind.OBJECT} +) _SEED_ROOT: OwnershipRoot = OwnershipRoot("seeds", OwnershipRootKind.GLOBAL, ResourceKind.SEED) _PYTHON_FUNCTION_ROOT: OwnershipRoot = OwnershipRoot( "functions/python", OwnershipRootKind.GLOBAL, ResourceKind.FUNCTION @@ -528,10 +531,7 @@ def _enum_record( scope=scope, ownership_root=ownership_root, owning_path=normalize_path(path=owning_path) if owning_path is not None else None, - enum=EnumMetadata( - members=tuple(EnumMemberMetadata(item.name) for item in declaration.members), - scalar_type=declaration.scalar_type, - ), + enum=enum_metadata(declaration), ) @@ -544,12 +544,6 @@ def _constant_record( path: Path, owner: ResourceIdentity | None = None, ) -> DeclarationRecord: - collection_kinds: frozenset[SqlValueKind] = frozenset( - {SqlValueKind.LIST, SqlValueKind.SET, SqlValueKind.OBJECT} - ) - is_collection: bool = declaration.value.kind in collection_kinds - payload: object = declaration.value.value - item_count: int | None = len(payload) if is_collection and isinstance(payload, tuple) else None return DeclarationRecord( identity=DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner), path=normalize_path(path=path), @@ -558,13 +552,7 @@ def _constant_record( scope=scope, ownership_root=ownership_root, owning_path=normalize_path(path=owning_path) if owning_path is not None else None, - constant=ConstantMetadata( - logical_type=declaration.logical_type.display_name, - collection_kind=declaration.value.kind.value if is_collection else None, - item_count=item_count, - nullable=declaration.value.kind is SqlValueKind.NULL, - render_as=declaration.render_as.value if declaration.render_as is not None else None, - ), + constant=constant_metadata(declaration), ) @@ -583,21 +571,15 @@ def _macro_record( if discovered is not None and discovered.owning_path is not None else None ) - code: object = getattr(loaded.function, "__code__", None) - line: int = code.co_firstlineno if isinstance(code, CodeType) else 1 return DeclarationRecord( identity=DeclarationIdentity(DeclarationKind.MACRO, loaded.name), path=path, - line=line, + line=macro_line(loaded), column=1, scope=scope, ownership_root=ownership_root, owning_path=owning_path, - macro=MacroMetadata( - parameters=tuple(inspect.signature(loaded.function).parameters), - dependencies=loaded.dependencies, - source_digest=hashlib.sha256(loaded.raw_source.encode()).hexdigest(), - ), + macro=macro_metadata(loaded), ) @@ -734,3 +716,44 @@ def _declaration_key(record: DeclarationRecord) -> tuple[str, str, int, int]: def _diagnostic_key(item: ScopeDiagnostic) -> tuple[str, int, int, str, str]: return (item.path or "", item.line or 0, item.column or 0, item.code.value, item.message) + + +def enum_metadata(declaration: EnumDeclaration) -> EnumMetadata: + """Return the member names and scalar type of one enum.""" + + return EnumMetadata( + members=tuple(EnumMemberMetadata(item.name) for item in declaration.members), + scalar_type=declaration.scalar_type, + ) + + +def constant_metadata(declaration: ConstantDeclaration) -> ConstantMetadata: + """Return the logical type and collection shape of one constant.""" + + is_collection: bool = declaration.value.kind in _COLLECTION_KINDS + payload: object = declaration.value.value + item_count: int | None = len(payload) if is_collection and isinstance(payload, tuple) else None + return ConstantMetadata( + logical_type=declaration.logical_type.display_name, + collection_kind=declaration.value.kind.value if is_collection else None, + item_count=item_count, + nullable=declaration.value.kind is SqlValueKind.NULL, + render_as=declaration.render_as.value if declaration.render_as is not None else None, + ) + + +def macro_metadata(loaded: LoadedMacro) -> MacroMetadata: + """Return the parameters, dependencies and source digest of one loaded macro.""" + + return MacroMetadata( + parameters=tuple(inspect.signature(loaded.function).parameters), + dependencies=loaded.dependencies, + source_digest=hashlib.sha256(loaded.raw_source.encode()).hexdigest(), + ) + + +def macro_line(loaded: LoadedMacro) -> int: + """Return the first line of a loaded macro's function, or 1 without code.""" + + code: object = getattr(loaded.function, "__code__", None) + return code.co_firstlineno if isinstance(code, CodeType) else 1 diff --git a/src/sqlbuild/compiler/scopes/classes/__init__.py b/src/sqlbuild/compiler/scopes/classes/__init__.py new file mode 100644 index 0000000000..69ee566cdc --- /dev/null +++ b/src/sqlbuild/compiler/scopes/classes/__init__.py @@ -0,0 +1 @@ +"""Compiler scope runtime classes.""" diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py new file mode 100644 index 0000000000..b7d7957705 --- /dev/null +++ b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py @@ -0,0 +1,298 @@ +"""A natively built scope index, materialised as the Python scope models it replaces.""" + +from __future__ import annotations + +from collections.abc import Callable, Hashable, Sequence +from dataclasses import replace +from types import MappingProxyType + +import sqlbuild._native as _native +from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows +from sqlbuild.compiler.scopes.models import ( + DeclarationRecord, + DeclarationVisibilityIndex, + GrantRecord, + NativeDeclarationValues, + OwnershipRoot, + RelationshipFact, + ResourceIdentity, + ResourceRecord, + ScopeCompleteness, + ScopeDiagnostic, + ScopeIndex, + ScopeLookup, + UsageRecord, +) +from sqlbuild.compiler.scopes.types import ( + GrantKind, + OwnershipRootKind, + ResourceKind, + ScopeDiagnosticCode, + UsageKind, +) + +type _Groups = tuple[list[list[int]], bool] +type _IndexRows = tuple[ + list[str], + list[tuple[str, str, str, str | None, str | None]], + list[int], + list[int], + list[tuple[int, int]], + list[tuple[str, str, str | None, int | None, int | None, int | None, int | None]], + bool, +] +type _GrantRows = list[tuple[str, str, int, str | int, str]] +type _LookupRows = tuple[ + list[int], + list[int], + list[int], + list[int], + tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups], + tuple[ + list[int], + list[tuple[tuple[str, str], list[int]]], + list[tuple[str, list[int]]], + list[tuple[str, list[int]]], + ], +] + +_RESOURCE_ROOTS: dict[ResourceKind, str] = { + ResourceKind.MODEL: "models", + ResourceKind.TEST: "tests/unit", + ResourceKind.SCENARIO: "tests/scenarios", + ResourceKind.FUNCTION: "functions/sql", + ResourceKind.SOURCE: "sources", +} +_FIXED_ROOTS: dict[str, OwnershipRoot] = { + "seed": OwnershipRoot("seeds", OwnershipRootKind.GLOBAL, ResourceKind.SEED), + "python_function": OwnershipRoot( + "functions/python", OwnershipRootKind.GLOBAL, ResourceKind.FUNCTION + ), +} +_STATIC_COMPLETENESS: ScopeCompleteness = ScopeCompleteness( + runtime_usage=False, relationships=False, placement=False, promotion_impact=False +) + + +class NativeScopeIndex: + """One native scope index and the Python records materialised from it.""" + + def __init__(self, *, native: _native.NativeScopeIndex, rows: NativeScopeRows) -> None: + records: _IndexRows = native.records() + paths, declaration_rows, resource_order, declaration_order, usages, diagnostics, scoped = ( + records + ) + self._native: _native.NativeScopeIndex = native + self._resources: list[ResourceRecord] = [ + ResourceRecord( + identity=identity, + path=path, + ownership_root=_resource_root(identity=identity, root=root), + ) + for identity, path, root in zip( + rows.resource_identities, paths, rows.resource_roots, strict=True + ) + ] + self._declarations: list[DeclarationRecord] = [ + _declaration_record(values=values, row=row) + for values, row in zip(rows.declaration_values, declaration_rows, strict=True) + ] + self._usages: list[UsageRecord] = [ + UsageRecord( + consumer=self._declarations[consumer].identity, + declaration=rows.declaration_values[consumer].dependencies[dependency], + kind=UsageKind.DECLARATION_DEPENDENCY, + ) + for consumer, dependency in usages + ] + self._grants: list[GrantRecord] = [] + self.has_scoped_relationship_declarations: bool = scoped + self.index: ScopeIndex = ScopeIndex( + ownership_roots=tuple( + OwnershipRoot(path=path, resource_kind=kind) + for kind, path in sorted(_RESOURCE_ROOTS.items(), key=lambda item: item[1]) + ), + resources=tuple(self._resources[position] for position in resource_order), + declarations=tuple(self._declarations[position] for position in declaration_order), + usages=tuple(self._usages), + diagnostics=tuple( + ScopeDiagnostic( + ScopeDiagnosticCode(code), + message, + path=path, + line=line, + column=column, + declaration=( + None if declaration is None else self._declarations[declaration].identity + ), + resource=None if resource is None else self._resources[resource].identity, + ) + for code, message, path, line, column, declaration, resource in diagnostics + ), + completeness=_STATIC_COMPLETENESS, + ) + + def grant(self, facts: Sequence[RelationshipFact]) -> bool: + """Attach the relationship grants of `facts`; False when Python must resolve them.""" + + rows: _GrantRows | None = self._native.grant( + [ + ( + fact.resource.kind.value, + fact.resource.name, + list(fact.expected_models), + list(fact.called_macros), + list(fact.tested_macros), + ) + for fact in facts + ] + ) + if rows is None: + return False + identities: dict[tuple[str, str], ResourceIdentity] = {} + self._grants = [ + GrantRecord( + resource=identities.setdefault( + (kind, name), ResourceIdentity(ResourceKind(kind), name) + ), + declaration=self._declarations[declaration].identity, + through=( + ResourceIdentity(ResourceKind.MODEL, through) + if isinstance(through, str) + else self._declarations[through].identity + ), + kind=GrantKind(grant_kind), + ) + for kind, name, declaration, through, grant_kind in rows + ] + return True + + def index_with_relationships(self) -> ScopeIndex: + """Return the index with its grants attached and relationships complete.""" + + return replace( + self.index, + grants=tuple(self._grants), + completeness=replace(self.index.completeness, relationships=True), + ) + + def lookup(self, *, index: ScopeIndex) -> ScopeLookup | None: + """Return Python's `build_lookup(index)` for `index_with_relationships()`, or None.""" + + rows: _LookupRows | None = self._native.lookup() + if rows is None: + return None + resource_positions, declaration_positions, usage_positions, grant_positions = rows[:4] + groups: tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups] = rows[4] + global_positions, private_positions, local_positions, inherited_positions = rows[5] + declarations: tuple[DeclarationRecord, ...] = tuple( + self._declarations[position] for position in declaration_positions + ) + canonical_index: ScopeIndex = ScopeIndex( + ownership_roots=tuple( + sorted( + index.ownership_roots, + key=lambda item: ( + item.path, + item.resource_kind.value if item.resource_kind is not None else "", + ), + ) + ), + resources=tuple(self._resources[position] for position in resource_positions), + declarations=declarations, + usages=tuple(self._usages[position] for position in usage_positions), + grants=tuple(self._grants[position] for position in grant_positions), + visibility=(), + inaccessible=(), + diagnostics=tuple( + sorted( + index.diagnostics, + key=lambda item: ( + item.path or "", + item.line or 0, + item.column or 0, + item.code.value, + ), + ) + ), + completeness=index.completeness, + ) + return ScopeLookup( + index=canonical_index, + resources=_mapping( + groups=groups[0], records=self._resources, key=lambda item: item.identity + ), + resources_by_path=_mapping( + groups=groups[1], records=self._resources, key=lambda item: item.path + ), + declarations=_mapping( + groups=groups[2], records=self._declarations, key=lambda item: item.identity + ), + usages_by_consumer=_mapping( + groups=groups[3], records=self._usages, key=lambda item: item.consumer + ), + usages_by_declaration=_mapping( + groups=groups[4], records=self._usages, key=lambda item: item.declaration + ), + grants_by_resource=_mapping( + groups=groups[5], records=self._grants, key=lambda item: item.resource + ), + visibility_index=DeclarationVisibilityIndex( + identities=tuple(declaration.identity for declaration in declarations), + global_positions=tuple(global_positions), + private_positions=MappingProxyType( + { + ResourceIdentity(ResourceKind(kind), name): tuple(positions) + for (kind, name), positions in private_positions + } + ), + local_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in local_positions} + ), + inherited_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in inherited_positions} + ), + ), + ) + + +def _resource_root(*, identity: ResourceIdentity, root: str) -> OwnershipRoot: + fixed: OwnershipRoot | None = _FIXED_ROOTS.get(root) + if fixed is not None: + return fixed + return OwnershipRoot(path=_RESOURCE_ROOTS[identity.kind], resource_kind=identity.kind) + + +def _declaration_record( + *, values: NativeDeclarationValues, row: tuple[str, str, str, str | None, str | None] +) -> DeclarationRecord: + path, root_path, root_kind, root_resource_kind, owning_path = row + return DeclarationRecord( + identity=values.identity, + path=path, + line=values.line, + column=1, + scope=values.scope, + ownership_root=OwnershipRoot( + root_path, + OwnershipRootKind(root_kind), + None if root_resource_kind is None else ResourceKind(root_resource_kind), + ), + owning_path=owning_path, + macro=values.macro, + enum=values.enum, + constant=values.constant, + ) + + +def _mapping[Record, Key: Hashable]( + *, groups: _Groups, records: Sequence[Record], key: Callable[[Record], Key] +) -> MappingProxyType[Key, tuple[Record, ...]]: + members, repr_ordered = groups + items: list[tuple[Key, tuple[Record, ...]]] = [] + for group in members: + grouped: tuple[Record, ...] = tuple(records[member] for member in group) + items.append((key(grouped[0]), grouped)) + if not repr_ordered: + items.sort(key=lambda item: repr(item[0])) + return MappingProxyType(dict(items)) diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py b/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py new file mode 100644 index 0000000000..3e5d07295d --- /dev/null +++ b/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py @@ -0,0 +1,323 @@ +"""The builder's walk of discovered inputs as the plain rows the native scope index reads.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path + +from sqlbuild.compiler.compile.models import LoadedMacro +from sqlbuild.compiler.discovery.models import ( + ConstantDeclaration, + DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredMacroFile, + DiscoveredModelSchemaFile, + DiscoveredProjectInputs, + DiscoveredSqlHookFile, + EnumDeclaration, +) +from sqlbuild.compiler.scopes._helpers.builder import ( + constant_metadata, + enum_metadata, + macro_line, + macro_metadata, +) +from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + NativeDeclarationValues, + ResourceIdentity, +) +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + NativeDeclarationRow, + NativeIdentityRow, + NativeResourceRow, + ResourceKind, + ScopeKind, +) + +_KIND_ROOT: str = "kind" +_SEED_ROOT: str = "seed" +_PYTHON_FUNCTION_ROOT: str = "python_function" +_PRIVATE_ROOT: str = "models" +_MACRO_FALLBACK: str = "macros" +_ENUM_FALLBACK: str = "enums" +_CONSTANT_FALLBACK: str = "constants" + + +class NativeScopeRows: + """Resource and declaration rows in the Python builder's walk order, without records.""" + + def __init__( + self, + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro], + ) -> None: + self.resources: list[NativeResourceRow] = [] + self.resource_identities: list[ResourceIdentity] = [] + self.resource_roots: list[str] = [] + self.declarations: list[NativeDeclarationRow] = [] + self.declaration_values: list[NativeDeclarationValues] = [] + self._add_models(discovered_inputs=discovered_inputs) + self._add_resources(discovered_inputs=discovered_inputs) + self._add_declaration_files(discovered_inputs=discovered_inputs) + self._add_named_declarations(discovered_inputs=discovered_inputs) + self._add_macros(discovered_inputs=discovered_inputs, loaded_macros=loaded_macros) + + def _add_resource(self, *, kind: ResourceKind, name: str, path: Path, root: str) -> None: + self.resources.append((kind.value, name, str(path), root)) + self.resource_identities.append(ResourceIdentity(kind, name)) + self.resource_roots.append(root) + + def _add_declaration( + self, + *, + values: NativeDeclarationValues, + path: Path | str, + ownership_root: Path | str | None, + root_fallback: Path | str, + owning_path: Path | str | None, + ) -> None: + identity: DeclarationIdentity = values.identity + self.declarations.append( + ( + identity.kind.value, + identity.name, + _identity_row(identity.owner), + str(path), + values.line, + values.scope.value, + None if ownership_root is None else str(ownership_root), + str(root_fallback), + None if owning_path is None else str(owning_path), + [ + (item.kind.value, item.name, _identity_row(item.owner)) + for item in values.dependencies + ], + ) + ) + self.declaration_values.append(values) + + def _add_models(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: + for model_file in discovered_inputs.model_files: + model_name: str = _name_or_stem( + value=model_file.header_values.get("name"), path=model_file.relative_path + ) + self._add_resource( + kind=ResourceKind.MODEL, + name=model_name, + path=model_file.relative_path, + root=_KIND_ROOT, + ) + owner: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) + for enum in model_file.enum_declarations: + self._add_private(declaration=enum, owner=owner, path=model_file.relative_path) + for constant in model_file.constant_declarations: + self._add_private(declaration=constant, owner=owner, path=model_file.relative_path) + + def _add_private( + self, + *, + declaration: EnumDeclaration | ConstantDeclaration, + owner: ResourceIdentity, + path: Path, + ) -> None: + values: NativeDeclarationValues = ( + NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner), + line=1, + scope=ScopeKind.PRIVATE, + enum=enum_metadata(declaration), + ) + if isinstance(declaration, EnumDeclaration) + else NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner), + line=1, + scope=ScopeKind.PRIVATE, + constant=constant_metadata(declaration), + ) + ) + self._add_declaration( + values=values, + path=path, + ownership_root=_PRIVATE_ROOT, + root_fallback=_PRIVATE_ROOT, + owning_path=path.parent, + ) + + def _add_resources(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: + for test_file in discovered_inputs.test_files: + for block in test_file.blocks: + self._add_resource( + kind=ResourceKind.TEST, + name=block.name or test_file.relative_path.stem, + path=test_file.relative_path, + root=_KIND_ROOT, + ) + for scenario_file in discovered_inputs.scenario_files: + self._add_resource( + kind=ResourceKind.SCENARIO, + name=scenario_file.name, + path=scenario_file.relative_path, + root=_KIND_ROOT, + ) + for function_file in discovered_inputs.sql_function_files: + self._add_resource( + kind=ResourceKind.FUNCTION, + name=_name_or_stem( + value=function_file.header_values.get("name"), + path=function_file.relative_path, + ), + path=function_file.relative_path, + root=_KIND_ROOT, + ) + for python_function_file in discovered_inputs.python_function_files: + self._add_resource( + kind=ResourceKind.FUNCTION, + name=python_function_file.file_path.stem, + path=python_function_file.relative_path, + root=_PYTHON_FUNCTION_ROOT, + ) + for seed_file in discovered_inputs.seed_files: + self._add_resource( + kind=ResourceKind.SEED, + name=seed_file.file_path.stem, + path=seed_file.relative_path, + root=_SEED_ROOT, + ) + for source_file in discovered_inputs.source_files: + for source in source_file.source_entries: + self._add_resource( + kind=ResourceKind.SOURCE, + name=source.name, + path=source_file.relative_path, + root=_KIND_ROOT, + ) + + def _add_declaration_files(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: + for enum_file in discovered_inputs.enum_files: + for enum in enum_file.declarations: + self._add_declaration( + values=NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.ENUM, enum.name), + line=1, + scope=enum_file.scope_kind, + enum=enum_metadata(enum), + ), + path=enum_file.relative_path, + ownership_root=enum_file.ownership_root, + root_fallback=_ENUM_FALLBACK, + owning_path=enum_file.owning_path, + ) + for constant_file in discovered_inputs.constant_files: + for constant in constant_file.declarations: + self._add_declaration( + values=NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.CONSTANT, constant.name), + line=1, + scope=constant_file.scope_kind, + constant=constant_metadata(constant), + ), + path=constant_file.relative_path, + ownership_root=constant_file.ownership_root, + root_fallback=_CONSTANT_FALLBACK, + owning_path=constant_file.owning_path, + ) + + def _add_named_declarations(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: + for audit_file in discovered_inputs.audit_files: + if audit_file.declaration_kind is DeclarationKind.AUDIT: + self._add_named( + item=audit_file, + kind=DeclarationKind.AUDIT, + name=audit_file.relative_path.stem, + path=audit_file.relative_path, + ) + continue + for block in audit_file.blocks: + self._add_named( + item=audit_file, + kind=DeclarationKind.SINGULAR_AUDIT, + name=block.name or audit_file.relative_path.stem, + path=audit_file.relative_path, + ) + for schema_file in discovered_inputs.model_schema_files: + for declaration in schema_file.declarations: + self._add_named( + item=schema_file, + kind=DeclarationKind.SCHEMA, + name=declaration.name, + path=schema_file.relative_path, + ) + for hook_file in discovered_inputs.sql_hook_files: + self._add_named( + item=hook_file, + kind=DeclarationKind.SQL_HOOK, + name=hook_file.name, + path=hook_file.relative_path, + ) + for hook_function in discovered_inputs.hook_functions: + self._add_named( + item=hook_function, + kind=DeclarationKind.PYTHON_HOOK, + name=hook_function.name, + path=hook_function.relative_path, + ) + + def _add_named( + self, + *, + item: DiscoveredAuditFile + | DiscoveredModelSchemaFile + | DiscoveredSqlHookFile + | DiscoveredHookFunction, + kind: DeclarationKind, + name: str, + path: Path, + ) -> None: + self._add_declaration( + values=NativeDeclarationValues( + identity=DeclarationIdentity(kind, name), line=1, scope=item.scope_kind + ), + path=path, + ownership_root=item.ownership_root, + root_fallback=item.declaration_root or item.relative_path.parent, + owning_path=item.owning_path, + ) + + def _add_macros( + self, + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro], + ) -> None: + macro_files: dict[str, DiscoveredMacroFile] = { + normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files + } + for loaded in loaded_macros.values(): + discovered: DiscoveredMacroFile | None = macro_files.get( + normalize_path(path=loaded.relative_path) + ) + self._add_declaration( + values=NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.MACRO, loaded.name), + line=macro_line(loaded), + scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, + macro=macro_metadata(loaded), + dependencies=loaded.dependencies, + ), + path=loaded.relative_path, + ownership_root=discovered.ownership_root if discovered is not None else None, + root_fallback=_MACRO_FALLBACK, + owning_path=discovered.owning_path if discovered is not None else None, + ) + + +def _identity_row(identity: ResourceIdentity | None) -> NativeIdentityRow | None: + return None if identity is None else (identity.kind.value, identity.name) + + +def _name_or_stem(*, value: object, path: Path) -> str: + return value if isinstance(value, str) and value else path.stem diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py new file mode 100644 index 0000000000..bdc74e0f7c --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py @@ -0,0 +1,32 @@ +"""Build the static scope index natively for the compiler engine switch.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import LoadedMacro +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex +from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows +from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError + + +def open_native_scope_index( + *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] +) -> NativeScopeIndex | None: + """Return the natively built index, or None when the Python builder must run instead.""" + + try: + rows: NativeScopeRows = NativeScopeRows( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + ) + except (InvalidScopePathError, TypeError, ValueError): + return None + try: + native: _native.NativeScopeIndex | None = _native.build_native_scope_index( + rows.resources, rows.declarations + ) + except (TypeError, UnicodeError): + return None + return None if native is None else NativeScopeIndex(native=native, rows=rows) diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 97ee535dd8..d566889487 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -128,6 +128,29 @@ class GrantRecord: kind: GrantKind = GrantKind.EXPECTED_MODEL +@dataclass(frozen=True) +class NativeDeclarationValues: + """The Python-side parts of one declaration record the native scope index does not hold.""" + + identity: DeclarationIdentity + line: int + scope: ScopeKind + macro: MacroMetadata | None = None + enum: EnumMetadata | None = None + constant: ConstantMetadata | None = None + dependencies: tuple[DeclarationIdentity, ...] = () + + +@dataclass(frozen=True) +class RelationshipFact: + """Relationship names one test block or scenario authors, before grants are resolved.""" + + resource: ResourceIdentity + expected_models: tuple[str, ...] = field(default_factory=tuple) + called_macros: tuple[str, ...] = field(default_factory=tuple) + tested_macros: tuple[str, ...] = field(default_factory=tuple) + + @dataclass(frozen=True) class VisibilityRecord: """A resolved positive visibility fact with stable provenance.""" diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py index e49aafc858..4b059a3ff4 100644 --- a/src/sqlbuild/compiler/scopes/types.py +++ b/src/sqlbuild/compiler/scopes/types.py @@ -6,6 +6,21 @@ type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] type JsonObject = dict[str, object] +type NativeIdentityRow = tuple[str, str] +type NativeDeclarationIdentityRow = tuple[str, str, NativeIdentityRow | None] +type NativeResourceRow = tuple[str, str, str, str] +type NativeDeclarationRow = tuple[ + str, + str, + NativeIdentityRow | None, + str, + int, + str, + str | None, + str, + str | None, + list[NativeDeclarationIdentityRow], +] class ScopeCacheIdentityType(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/scopes/__init__.py b/tests/integration/src/sqlbuild/compiler/scopes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py new file mode 100644 index 0000000000..4af8c1e711 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ScopeEngineParityTestCase: + """A project whose declaration scope is built under each compiler engine.""" + + description: str + files: dict[str, str] + expected_error: str = "" + expected_minimum_grants: int = 0 + + +@dataclass(frozen=True) +class GeneratedScopeParityTestCase: + """Seeded scoped projects compared between engines, with minimum coverage of each outcome.""" + + description: str + seed: int + count: int + expected_minimum_valid: int + expected_minimum_invalid: int + expected_minimum_granting: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class ScopeCommandParityTestCase: + """A project whose `sqb scope` index is built offline under each compiler engine.""" + + description: str + files: dict[str, str] + expected_minimum_grants: int diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py new file mode 100644 index 0000000000..f71e36c379 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -0,0 +1,318 @@ +"""Scoped projects and engine-parity outcomes for the native declaration scope tests.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope +from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.discovery.exceptions import DiscoveryError +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index +from sqlbuild.compiler.scopes.main.load_or_build_scope_index import load_or_build_scope_index +from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + +PROJECT_CONFIG: str = ( + 'name = "scope_parity"\nadapter = "duckdb"\n\n[connection]\ndatabase = "scope.duckdb"\n' +) +RAW_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Orders feed.\n" + " expression: >-\n (SELECT 1 AS order_id)\n" +) +SCOPED_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": PROJECT_CONFIG, + "sources/raw.yml": RAW_SOURCES, + "macros/money.py": 'def cents(value: str) -> str:\n return f"{value} * 100"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "models/sales/_sqlbuild/macros/labels.py": ( + "def sales_label(value: str) -> str:\n return f\"'sales:' || {value}\"\n" + ), + "models/sales/_sqlbuild/constants/limits.sql": "CONSTANT (name sales_cap, value 3);\n", + "models/sales/eu/_macros/rates.py": ( + 'def eu_rate(value: str) -> str:\n return f"{value} * 2"\n' + ), + "models/sales/eu/_enums/region.sql": "ENUM (name eu_region, members [NORTH, SOUTH]);\n", + "models/sales/orders.sql": ( + 'MODEL (\n description "Orders",\n enums (\n _state [OPEN, CLOSED],\n ),\n);\n\n' + "SELECT\n order_id,\n @sales_label('1') AS label,\n" + ' @const("sales_cap") AS cap,\n @enum("_state").OPEN AS state,\n' + ' @enum("order_status").PLACED AS status\nFROM __source("raw_orders")\n' + ), + "models/sales/eu/eu_orders.sql": ( + 'MODEL (\n description "EU orders",\n);\n\n' + "SELECT\n order_id,\n @eu_rate('2') AS rate,\n" + " @enum(\"eu_region\").NORTH AS region,\n @cents('3') AS cents,\n" + " @sales_label('4') AS label,\n @const(\"sales_cap\") AS cap\n" + 'FROM __ref("orders")\n' + ), + "models/other/customers.sql": ( + 'MODEL (\n description "Customers",\n);\n\n' + "SELECT 1 AS customer_id, @enum(\"order_status\").SHIPPED AS status, @cents('5') AS cents\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "orders_label");\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id\n),\n" + "__expected__orders AS (\n" + " SELECT 1 AS order_id, @sales_label('1') AS label, @const(\"sales_cap\") AS cap\n)\n" + "SELECT 1\n" + ), + "tests/unit/test_eu.sql": ( + 'TEST (name "eu_rate");\n\nWITH\n__ref__orders AS (\n SELECT 1 AS order_id\n),\n' + "__expected__eu_orders AS (\n" + " SELECT 1 AS order_id, @eu_rate('2') AS rate, @enum(\"eu_region\").NORTH AS region\n)\n" + "SELECT 1\n" + ), + "tests/unit/test_label_macro.sql": ( + 'TEST (mode macro, name "labels_sales");\n\nWITH\n' + "__macro_actual__ AS (\n SELECT @sales_label('1') AS label\n),\n" + "__macro_expected__ AS (\n SELECT 'sales:' || 1 AS label\n)\nSELECT 1\n" + ), + "tests/scenarios/eu_flow.sql": ( + 'SCENARIO (\n description "EU flow"\n);\n\nWITH\n' + "__ref__orders AS (\n SELECT 1 AS order_id\n),\n" + "__expected__eu_orders AS (\n" + ' SELECT 1 AS order_id, @enum("eu_region").SOUTH AS region\n)\n' + "SELECT 1\n" + ), +} + + +@dataclass(frozen=True) +class ScopeOutcome: + """What building the declaration scope produced under one engine.""" + + kind: str + value: tuple[object, ...] + grants: int = 0 + error: str = "" + + +_FOLDERS: tuple[str, ...] = ( + "models/sales", + "models/sales/eu", + "models/sales/eu/north", + "models/other", + "models/other/archive", +) +_PLACEMENTS: tuple[str, ...] = ( + "{kind}s/f{index}", + "{folder}/{kind}s/f{index}", + "{folder}/_sqlbuild/{kind}s/f{index}", + "{folder}/_{kind}s/f{index}", + "{folder}/_sqlbuild/_{kind}s/f{index}", +) +_DECLARATION_KINDS: tuple[str, ...] = ("macro", "enum", "constant") + + +def write_project(*, project_dir: Path, files: dict[str, str]) -> None: + """Write `files` below `project_dir`.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +def generated_scope_files(*, rng: random.Random) -> dict[str, str]: + """Return a seeded project of scoped declarations, models, tests and scenarios.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": PROJECT_CONFIG, + "sources/raw.yml": RAW_SOURCES, + } + models: list[str] = [f"model_{index}" for index in range(rng.randint(1, 6))] + for index, model in enumerate(models): + private: str = rng.choice(("", "", f" constants (\n _limit_{index} 2,\n ),\n")) + files[f"{rng.choice(_FOLDERS)}/{model}.sql"] = ( + f'MODEL (\n description "Model {index}",\n{private});\n\n' + 'SELECT order_id\nFROM __source("raw_orders")\n' + ) + macros: list[str] = [] + for index in range(rng.randint(0, 8)): + kind: str = rng.choice(_DECLARATION_KINDS) + path: str = rng.choice(_PLACEMENTS).format( + kind=kind, folder=rng.choice(_FOLDERS), index=index + ) + declaration: tuple[str, str, tuple[str, ...]] = _DECLARATION_WRITERS[kind]( + path=path, index=index, variant=rng.randint(0, 4) + ) + files[declaration[0]] = declaration[1] + macros.extend(declaration[2]) + for index in range(rng.randint(0, 4)): + files[f"tests/unit/test_{index}.sql"] = _TEST_WRITERS[ + rng.choices(_TEST_KINDS, weights=_TEST_WEIGHTS)[0] + ]( + index=index, + calls=_macro_calls(rng=rng, macros=macros), + expected=rng.sample([*models, "missing_model"], k=rng.randint(1, 2)), + ) + scenario: dict[str, str] = { + "tests/scenarios/flow.sql": ( + 'SCENARIO (\n description "Flow"\n);\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id\n),\n" + f"__expected__{rng.choice([*models, 'missing_model'])} AS (\n" + " SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + } + files.update(rng.choice((scenario, scenario, {}, {}, {}))) + return files + + +def _macro_writer(*, path: str, index: int, variant: int) -> tuple[str, str, tuple[str, ...]]: + del variant + name: str = f"macro_{index}" + return f"{path}.py", f"def {name}(value: str) -> str:\n return value\n", (name,) + + +def _enum_writer(*, path: str, index: int, variant: int) -> tuple[str, str, tuple[str, ...]]: + del index + return f"{path}.sql", f"ENUM (name enum_{variant}, members [A, B]);\n", () + + +def _constant_writer(*, path: str, index: int, variant: int) -> tuple[str, str, tuple[str, ...]]: + del index + return f"{path}.sql", f"CONSTANT (name constant_{variant}, value 1);\n", () + + +_DECLARATION_WRITERS: dict[str, Callable[..., tuple[str, str, tuple[str, ...]]]] = { + "macro": _macro_writer, + "enum": _enum_writer, + "constant": _constant_writer, +} + + +def _macro_calls(*, rng: random.Random, macros: list[str]) -> str: + return "".join( + f", @{name}('{position}') AS value_{position}" + for position, name in enumerate(rng.sample(macros, k=min(len(macros), rng.randint(0, 2)))) + ) + + +def _macro_test(*, index: int, calls: str, expected: list[str]) -> str: + del expected + return ( + f'TEST (mode macro, name "macro_test_{index}");\n\nWITH\n' + f"__macro_actual__ AS (\n SELECT 1 AS id{calls}\n),\n" + f"__macro_expected__ AS (\n SELECT 1 AS id{calls}\n)\nSELECT 1\n" + ) + + +def _broken_test(*, index: int, calls: str, expected: list[str]) -> str: + del calls, expected + return ( + f'TEST (name "broken_{index}");\n\nWITH\n__source__raw_orders AS (\n SELECT 1\n),\n' + "__expected__ AS (\n SELECT 1\n)\nSELECT 1\n" + ) + + +def _expected_model_test(*, index: int, calls: str, expected: list[str]) -> str: + blocks: str = ",\n".join( + f"__expected__{model} AS (\n SELECT 1 AS order_id{calls}\n)" for model in expected + ) + return ( + f'TEST (name "test_{index}");\n\nWITH\n__source__raw_orders AS (\n' + f" SELECT 1 AS order_id\n),\n{blocks}\nSELECT 1\n" + ) + + +_TEST_KINDS: tuple[str, ...] = ("macro", "broken", "expected_model") +_TEST_WEIGHTS: tuple[int, ...] = (3, 1, 6) +_TEST_WRITERS: dict[str, Callable[..., str]] = { + "macro": _macro_test, + "broken": _broken_test, + "expected_model": _expected_model_test, +} + + +def scope_engine_outcomes( + *, project_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[ScopeOutcome, ScopeOutcome, bool]: + """Return the Python and native scope outcomes and whether the native index was built.""" + + try: + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) + except (DiscoveryError, CompileInputError) as error: + failure: ScopeOutcome = ScopeOutcome(kind="discovery", value=(), error=str(error)) + return failure, failure, True + syntax: SqlLexicalSyntax = DuckDbAdapter().sql_lexical_syntax + python: ScopeOutcome = _scope_outcome( + discovered=discovered, + macros=macros, + syntax=syntax, + engine="python", + monkeypatch=monkeypatch, + ) + native: ScopeOutcome = _scope_outcome( + discovered=discovered, + macros=macros, + syntax=syntax, + engine="native", + monkeypatch=monkeypatch, + ) + built: bool = ( + open_native_scope_index(discovered_inputs=discovered, loaded_macros=macros) is not None + ) + return python, native, built + + +def scope_command_indexes( + *, project_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[ScopeIndex, ScopeIndex]: + """Return the offline `sqb scope` index under the Python and native engines.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + python: ScopeIndex = load_or_build_scope_index(project_dir=project_dir, no_cache=True) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "native") + native: ScopeIndex = load_or_build_scope_index(project_dir=project_dir, no_cache=True) + return python, native + + +def _scope_outcome( + *, + discovered: DiscoveredProjectInputs, + macros: dict[str, LoadedMacro], + syntax: SqlLexicalSyntax, + engine: str, + monkeypatch: pytest.MonkeyPatch, +) -> ScopeOutcome: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + try: + scope: DeclarationScopeBuild = build_declaration_scope( + discovered_inputs=discovered, loaded_macros=macros, sql_lexical_syntax=syntax + ) + except CompileInputError as error: + return ScopeOutcome(kind="error", value=(type(error.__cause__).__name__,), error=str(error)) + return ScopeOutcome( + kind="ok", + value=(scope.index, _lookup_shape(scope.resolver.lookup), scope.resolver.resource_specific), + grants=len(scope.index.grants), + ) + + +def _lookup_shape(lookup: ScopeLookup) -> tuple[object, ...]: + return ( + lookup.index, + tuple(lookup.resources.items()), + tuple(lookup.resources_by_path.items()), + tuple(lookup.declarations.items()), + tuple(lookup.usages_by_consumer.items()), + tuple(lookup.usages_by_declaration.items()), + tuple(lookup.grants_by_resource.items()), + lookup.visibility_index.identities, + lookup.visibility_index.global_positions, + tuple(lookup.visibility_index.private_positions.items()), + tuple(lookup.visibility_index.local_positions.items()), + tuple(lookup.visibility_index.inherited_positions.items()), + ) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py new file mode 100644 index 0000000000..595595448b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -0,0 +1,165 @@ +"""Engine parity for the declaration scope the native engine builds: index, lookup and grants.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.scopes._test_types import ( + GeneratedScopeParityTestCase, + ScopeCommandParityTestCase, + ScopeEngineParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( + SCOPED_PROJECT, + ScopeOutcome, + generated_scope_files, + scope_command_indexes, + scope_engine_outcomes, + write_project, +) + +_BROKEN_TEST: str = ( + 'TEST (name "broken");\n\nWITH\n__source__raw_orders AS (\n SELECT 1\n),\n' + "__expected__ AS (\n SELECT 1\n)\nSELECT 1\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + ScopeEngineParityTestCase( + description="nested scoped roots, private values, expected-model and macro-test grants", + files=SCOPED_PROJECT, + expected_minimum_grants=5, + ), + ScopeEngineParityTestCase( + description="a global enum duplicating a scoped enum fails with Python's diagnostics", + files={ + **SCOPED_PROJECT, + "enums/region.sql": "ENUM (name eu_region, members [EAST]);\n", + }, + expected_error="Duplicate declaration 'enum:eu_region'", + ), + ScopeEngineParityTestCase( + description="a malformed expected-model CTE fails with Python's relationship fault", + files={**SCOPED_PROJECT, "tests/unit/test_broken.sql": _BROKEN_TEST}, + expected_error="__expected__", + ), + ScopeEngineParityTestCase( + description="non-ASCII folders keep Python's repr order of path keys", + files={ + **SCOPED_PROJECT, + "models/sales/r\u00e9gion/regional.sql": ( + 'MODEL (\n description "Regional",\n);\n\n' + 'SELECT order_id, @const("sales_cap") AS cap\nFROM __ref("orders")\n' + ), + }, + expected_minimum_grants=5, + ), + ScopeEngineParityTestCase( + description="a test granted a deeper macro's folder through a macro test", + files={ + **SCOPED_PROJECT, + "models/sales/eu/_macros/rates.py": ( + 'def eu_rate(value: str) -> str:\n return f"{value} * 2"\n\n\n' + "def eu_label(value: str) -> str:\n return eu_rate(value)\n" + ), + "tests/unit/test_eu_label.sql": ( + 'TEST (mode macro, name "eu_label");\n\nWITH\n' + "__macro_actual__ AS (\n SELECT @eu_label('1') AS label\n),\n" + "__macro_expected__ AS (\n SELECT 1 * 2 AS label\n)\nSELECT 1\n" + ), + }, + expected_minimum_grants=8, + ), + ], + ids=lambda case: case.description, +) +def test_given_scoped_project_when_building_scope_under_each_engine_then_outcomes_match( + test_case: ScopeEngineParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + python, native, built = scope_engine_outcomes(project_dir=tmp_path, monkeypatch=monkeypatch) + + assert ( + native == python, + built, + (python.kind == "error") == bool(test_case.expected_error), + test_case.expected_error in python.error, + native.grants >= test_case.expected_minimum_grants, + ) == (True, True, True, True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedScopeParityTestCase( + description="seeded scoped declarations, grants and relationship faults", + seed=20261007, + count=120, + expected_minimum_valid=40, + expected_minimum_invalid=10, + expected_minimum_granting=15, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_scoped_projects_when_building_scope_then_engines_match( + test_case: GeneratedScopeParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + project_dirs: list[Path] = [tmp_path / f"project_{index}" for index in range(test_case.count)] + for project_dir in project_dirs: + write_project(project_dir=project_dir, files=generated_scope_files(rng=rng)) + + outcomes: list[tuple[ScopeOutcome, ScopeOutcome, bool]] = [ + scope_engine_outcomes(project_dir=project_dir, monkeypatch=monkeypatch) + for project_dir in project_dirs + ] + + python: list[ScopeOutcome] = [outcome[0] for outcome in outcomes] + native: list[ScopeOutcome] = [outcome[1] for outcome in outcomes] + assert ( + mismatches( + inputs=[path.name for path in project_dirs], + expected=[*python], + actual=[*native], + ), + all(outcome[2] for outcome in outcomes), + sum(item.kind == "ok" for item in python) >= test_case.expected_minimum_valid, + sum(item.kind == "error" for item in python) >= test_case.expected_minimum_invalid, + sum(item.grants > 0 for item in python) >= test_case.expected_minimum_granting, + ) == (list(test_case.expected_mismatches), True, True, True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + ScopeCommandParityTestCase( + description="the offline scope command index with compile usages and grants", + files=SCOPED_PROJECT, + expected_minimum_grants=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_scoped_project_when_scope_command_builds_index_then_engines_match( + test_case: ScopeCommandParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + python, native = scope_command_indexes(project_dir=tmp_path, monkeypatch=monkeypatch) + + assert (native == python, len(native.grants) >= test_case.expected_minimum_grants) == ( + True, + True, + ), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 70fa90f4d8c91948b4818fe49d26ec72c1533f70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:38:50 +0100 Subject: [PATCH 3/7] feat(native): compare declaration scopes in the compiler differential The failure corpus gains scope cases: a duplicate scoped declaration, inaccessible local and scoped declarations, the first of two visibility errors, a test outside its relationship grants and a malformed expected model. A harness test compares a scoped project with expected-model and macro-test grants under both engines. --- .../_helpers/corpus/failure_cases.py | 10 ++- .../_helpers/corpus/scope_failure_cases.py | 84 ++++++++++++++++++ .../test_scope_differential.py | 86 +++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py create mode 100644 tests/e2e/scripts/compiler_differential/test_scope_differential.py diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index c15ed2a7fc..8953c5cfef 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -11,6 +11,9 @@ from scripts.compiler_differential._helpers.corpus.render_failure_cases import ( render_failure_cases, ) +from scripts.compiler_differential._helpers.corpus.scope_failure_cases import ( + scope_failure_cases, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, FAILURE_BASE_MART, @@ -33,7 +36,12 @@ def _rules(*codes: str) -> dict[str, str]: def all_failure_cases() -> tuple[FailureCase, ...]: """Return every failure case in a stable order.""" - return (*_compile_failure_cases(), *discovery_failure_cases(), *render_failure_cases()) + return ( + *_compile_failure_cases(), + *discovery_failure_cases(), + *render_failure_cases(), + *scope_failure_cases(), + ) def _compile_failure_cases() -> tuple[FailureCase, ...]: diff --git a/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py new file mode 100644 index 0000000000..035f9531d2 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py @@ -0,0 +1,84 @@ +"""Minimal projects that fail on declaration scopes: the index, visibility and relationships.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_FILES, + FAILURE_MART_PATH, + FAILURE_STAGING_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_STAGING_MACRO_PATH: str = "models/staging/_macros/amounts.py" +_STAGING_MACRO: str = "def staged_amount(value):\n return f'{value} * 1'\n" +_STAGING_ENUM_PATH: str = "models/staging/_enums/status.sql" +_STAGING_ENUM: str = "ENUM (name staged_status, members [PLACED]);\n" +_MART_HEADER: str = 'MODEL (\n description "Order totals per customer",\n);\n\n' +_STAGING_HEADER: str = 'MODEL (\n description "Staged orders",\n);\n\n' +_TEST_PATH: str = "tests/unit/test_customer_totals.sql" + + +def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code + ) + + +def scope_failure_cases() -> tuple[FailureCase, ...]: + """Return the declaration-scope failure cases in a stable order.""" + + return ( + _case( + name="scope-duplicate-scoped-enum", + expected_code="P001", + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + "enums/status.sql": "ENUM (name staged_status, members [SHIPPED]);\n", + }, + ), + _case( + name="scope-inaccessible-local-macro", + expected_code="P001", + files={ + _STAGING_MACRO_PATH: _STAGING_MACRO, + FAILURE_MART_PATH: _MART_HEADER + + "SELECT customer_id, SUM(@staged_amount('amount')) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n', + }, + ), + _case( + name="scope-first-of-two-visibility-errors", + expected_code="P001", + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + "models/marts/_macros/totals.py": "def mart_total(value):\n return value\n", + FAILURE_STAGING_PATH: _STAGING_HEADER + + "SELECT order_id, customer_id, @mart_total('amount') AS amount, status\n" + 'FROM __source("raw_orders")\n', + FAILURE_MART_PATH: _MART_HEADER + + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\n' + 'WHERE status = @enum("staged_status").PLACED\nGROUP BY customer_id\n', + }, + ), + _case( + name="scope-test-outside-relationship-grant", + expected_code="P001", + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + _TEST_PATH: 'TEST (name "totals_by_customer");\n\nWITH\n' + "__ref__stg_orders AS (\n SELECT 10 AS customer_id, 5 AS amount\n),\n" + "__expected__customer_totals AS (\n" + ' SELECT 10 AS customer_id, @enum("staged_status").PLACED AS status\n)\n' + "SELECT 1\n", + }, + ), + _case( + name="scope-malformed-expected-relationship", + expected_code="P001", + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + _TEST_PATH: 'TEST (name "totals_by_customer");\n\nWITH\n' + "__ref__stg_orders AS (\n SELECT 10 AS customer_id\n),\n" + "__expected__ AS (\n SELECT 10 AS customer_id\n)\nSELECT 1\n", + }, + ), + ) diff --git a/tests/e2e/scripts/compiler_differential/test_scope_differential.py b/tests/e2e/scripts/compiler_differential/test_scope_differential.py new file mode 100644 index 0000000000..f618764021 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_scope_differential.py @@ -0,0 +1,86 @@ +"""The harness finds a scoped project with relationship grants identical under both engines.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from scripts.compiler_differential.main.differential import run_compiler_differential +from tests.e2e.scripts.compiler_differential._test_types import HarnessRunTestCase +from tests.e2e.scripts.compiler_differential.helpers import harness_arguments + +_SCOPED_FILES: dict[str, str] = { + **FAILURE_BASE_FILES, + "macros/amounts.py": "def doubled(value):\n return f'{value} * 2'\n", + "models/staging/_enums/status.sql": "ENUM (name staged_status, members [PLACED]);\n", + "models/marts/_constants/limits.sql": "CONSTANT (name minimum_total, value 1);\n", + "models/staging/stg_orders.sql": ( + 'MODEL (\n description "Staged orders",\n);\n\n' + "SELECT order_id, customer_id, @doubled('amount') AS amount, status\n" + 'FROM __source("raw_orders")\nWHERE status = @enum("staged_status").PLACED\n' + ), + "models/marts/customer_totals.sql": ( + 'MODEL (\n description "Order totals per customer",\n);\n\n' + "SELECT customer_id, SUM(@doubled('amount')) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + 'HAVING SUM(amount) >= @const("minimum_total")\n' + ), + "tests/unit/test_stg_orders.sql": ( + 'TEST (name "staged_orders");\n\nWITH\n' + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'PLACED' AS status\n),\n" + "__expected__stg_orders AS (\n" + " SELECT 1 AS order_id, @doubled('5') AS amount," + ' @enum("staged_status").PLACED AS status\n)\nSELECT 1\n' + ), + "tests/unit/test_doubled.sql": ( + 'TEST (mode macro, name "doubles_amounts");\n\nWITH\n' + "__macro_actual__ AS (\n SELECT @doubled('2') AS value\n),\n" + "__macro_expected__ AS (\n SELECT 4 AS value\n)\nSELECT 1\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="scoped_declarations_with_relationship_grants", + extra_arguments=(), + expected_exit_code=0, + expected_lines=( + "OK project/scoped_orders", + "Compiler differential passed: 1 projects identical (python vs native)", + ), + expected_patterns=(), + expected_absent=("DIFF",), + ) + ], + ids=lambda case: case.description, +) +def test_given_scoped_project_when_comparing_engines_then_harness_passes( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project_dir: Path = tmp_path / "scoped_orders" + for relative_path, contents in _SCOPED_FILES.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8", newline="\n") + + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", extra=test_case.extra_arguments, project=project_dir + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert not any(text in output for text in test_case.expected_absent), output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From b14d0171ceebe57cf2792189dfc3e67581e4100d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:22:57 +0100 Subject: [PATCH 4/7] refactor(native): group the Python bindings by domain Move the PyO3 binding helpers into per-domain folders (analysis, boundary, discovery, rules, scopes, sqltext) and split the flat function module along the same lines. Function bodies are unchanged; only module paths and registration move. --- .../_helpers/{ => analysis}/catalog.rs | 6 +- .../_helpers/{ => analysis}/conversions.rs | 0 .../src/bindings/_helpers/analysis/mod.rs | 7 + .../_helpers/analysis/normalization.rs | 71 +++ .../analysis/normalization_results.rs | 19 + .../_helpers/{ => analysis}/positions.rs | 2 +- .../src/bindings/_helpers/analysis/queries.rs | 86 +++ .../bindings/_helpers/analysis/sql_tests.rs | 63 ++ .../_helpers/{ => boundary}/allocator.rs | 0 .../_helpers/{ => boundary}/config_values.rs | 0 .../src/bindings/_helpers/boundary/mod.rs | 4 + .../_helpers/{ => boundary}/oracles.rs | 0 .../_helpers/{ => boundary}/panics.rs | 19 +- .../src/bindings/_helpers/discovery/mod.rs | 1 + .../project_files.rs} | 14 +- .../src/bindings/_helpers/functions.rs | 599 ------------------ .../src/bindings/_helpers/mod.rs | 16 +- .../src/bindings/_helpers/rules/evaluation.rs | 105 +++ .../src/bindings/_helpers/rules/lint.rs | 104 +++ .../src/bindings/_helpers/rules/mod.rs | 3 + .../src/bindings/_helpers/rules/skills.rs | 40 ++ .../src/bindings/_helpers/scopes/mod.rs | 1 + .../{scopes.rs => scopes/scope_index.rs} | 2 +- .../_helpers/sqltext/authored_values.rs | 94 +++ .../src/bindings/_helpers/sqltext/mod.rs | 3 + .../_helpers/sqltext/model_headers.rs | 59 ++ .../bindings/_helpers/sqltext/static_sql.rs | 37 ++ .../src/bindings/main/register.rs | 30 +- crates/sqlbuild-python/src/bindings/models.rs | 2 +- 29 files changed, 748 insertions(+), 639 deletions(-) rename crates/sqlbuild-python/src/bindings/_helpers/{ => analysis}/catalog.rs (95%) rename crates/sqlbuild-python/src/bindings/_helpers/{ => analysis}/conversions.rs (100%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs rename crates/sqlbuild-python/src/bindings/_helpers/{ => analysis}/positions.rs (93%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs rename crates/sqlbuild-python/src/bindings/_helpers/{ => boundary}/allocator.rs (100%) rename crates/sqlbuild-python/src/bindings/_helpers/{ => boundary}/config_values.rs (100%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs rename crates/sqlbuild-python/src/bindings/_helpers/{ => boundary}/oracles.rs (100%) rename crates/sqlbuild-python/src/bindings/_helpers/{ => boundary}/panics.rs (70%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs rename crates/sqlbuild-python/src/bindings/_helpers/{discovery.rs => discovery/project_files.rs} (95%) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/functions.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs rename crates/sqlbuild-python/src/bindings/_helpers/{scopes.rs => scopes/scope_index.rs} (99%) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/catalog.rs rename to crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs index 7235d925b6..70ef2a70d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs @@ -1,8 +1,8 @@ //! Python methods of the compile-owned binding catalog and its compact analysis jobs. -use crate::bindings::_helpers::conversions; -use crate::bindings::_helpers::panics::compiler_error; -use crate::bindings::_helpers::panics::normalization_results; +use crate::bindings::_helpers::analysis::conversions; +use crate::bindings::_helpers::analysis::normalization_results::normalization_results; +use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; use crate::bindings::types::CompilerDetach; use pyo3::exceptions::PyValueError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/conversions.rs rename to crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs new file mode 100644 index 0000000000..762d17f3ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -0,0 +1,7 @@ +mod catalog; +pub(crate) mod conversions; +pub(crate) mod normalization; +pub(crate) mod normalization_results; +mod positions; +pub(crate) mod queries; +pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs new file mode 100644 index 0000000000..d5f3dda1be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs @@ -0,0 +1,71 @@ +//! Normalize SQL for analysis and recover binding diagnostics. + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::analysis::normalization_results::normalization_results; +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +fn normalize_analysis_sql( + py: Python<'_>, + request: crate::bindings::models::NormalizationInput, +) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( + request.into(), + ) + }) + .map_err(value_error) +} + +#[pyfunction] +fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql( + sql, dialect, + ) + }) + .map_err(value_error) +} + +/// Normalize a batch in one detached call; a failed member is returned as its exception. +#[pyfunction] +fn normalize_analysis_sqls( + py: Python<'_>, + dialect: &str, + requests: Vec, +) -> PyResult>> { + let results = py.compiler_detach(|| { + Ok( + sqlbuild_analysis::semantic_validation::main::normalize_batch::normalize_analysis_sqls( + dialect, requests, None, + ), + ) + }); + normalization_results(py, results.map_err(value_error)?) +} + +#[pyfunction] +fn binding_diagnostics( + py: Python<'_>, + sql: &str, + dialect: &str, + rows: Vec, +) -> PyResult> { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( + sql, dialect, rows, + ) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; + module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs new file mode 100644 index 0000000000..aa116ff1aa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs @@ -0,0 +1,19 @@ +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; + +/// Return normalized SQL in order, carrying each failure as the exception a single call raises. +pub(crate) fn normalization_results( + py: Python<'_>, + results: Vec>, +) -> PyResult>> { + results + .into_iter() + .map(|result| match result { + Ok(sql) => Ok(sql.into_pyobject(py)?.into_any().unbind()), + Err(error) => Ok( + crate::bindings::_helpers::boundary::panics::compiler_error(error) + .into_value(py) + .into_any(), + ), + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/positions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/positions.rs rename to crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs index fc3da95977..40b440c5bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/positions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs @@ -1,6 +1,6 @@ //! Python methods of authored binding positions. -use crate::bindings::_helpers::panics::compiler_guard; +use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::models::{BindingPositions, PositionInput}; use pyo3::exceptions::PyValueError; use pyo3::{PyResult, pymethods}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs new file mode 100644 index 0000000000..c824541f2f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -0,0 +1,86 @@ +//! Validate, fingerprint and analyze SQL queries for the Python compiler. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) + }) + .map_err(value_error) +} + +#[pyfunction(name = "validate_sql_with_schema_json")] +fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::validation_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction(name = "validate_sql_with_schemas_json")] +fn schema_validations_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::validations_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| sqlbuild_analysis::semantic_usage::main::analyze_json(request_json)) + .map_err(value_error) +} + +#[pyfunction] +fn analyze_column_references_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::column_references::main::analyze::analyze_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::query_analysis::main::analyze::analyze_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::query_analysis::main::analyze_project::analyze_project_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::query_analysis::main::analyze_project_compact::analyze_project_compact_json( + request_json, + ) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; + module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_project_queries_json, module)?)?; + module.add_function(wrap_pyfunction!( + analyze_project_queries_compact_json, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs new file mode 100644 index 0000000000..5d39624fe3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -0,0 +1,63 @@ +//! Extract, plan and render SQL tests natively. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_rendering::render_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn plan_and_render_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_planning::plan_and_render_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_chain_resolution::resolve_chains_json( + request_json, + ) + }) + .map_err(value_error) +} + +#[pyfunction] +fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json( + request_json, + ) + }) + .map_err(value_error) +} + +#[pyfunction] +fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; + module.add_function(wrap_pyfunction!(plan_and_render_sql_tests_json, module)?)?; + module.add_function(wrap_pyfunction!(resolve_sql_test_chains_json, module)?)?; + module.add_function(wrap_pyfunction!( + render_sql_test_difference_sample_json, + module + )?)?; + module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/allocator.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/allocator.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/allocator.rs rename to crates/sqlbuild-python/src/bindings/_helpers/boundary/allocator.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/config_values.rs rename to crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs new file mode 100644 index 0000000000..0a21932dc9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -0,0 +1,4 @@ +mod allocator; +pub(crate) mod config_values; +pub(crate) mod oracles; +pub(crate) mod panics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/oracles.rs rename to crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/panics.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/panics.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/panics.rs rename to crates/sqlbuild-python/src/bindings/_helpers/boundary/panics.rs index 50e7e2f9de..b1f1fbcdcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/panics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/panics.rs @@ -2,7 +2,6 @@ use pyo3::Python; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult}; use pyo3::{PyErr, create_exception}; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; @@ -22,6 +21,10 @@ pub(crate) fn compiler_error(error: impl std::fmt::Display) -> PyErr { } } +pub(crate) fn value_error(error: impl std::fmt::Display) -> PyErr { + compiler_error(error) +} + pub(crate) fn compiler_guard( operation: impl FnOnce() -> pyo3::PyResult, ) -> pyo3::PyResult { @@ -39,17 +42,3 @@ impl CompilerDetach for Python<'_> { self.detach(|| catch_compiler_panic(operation)) } } - -/// Return normalized SQL in order, carrying each failure as the exception a single call raises. -pub(crate) fn normalization_results( - py: Python<'_>, - results: Vec>, -) -> PyResult>> { - results - .into_iter() - .map(|result| match result { - Ok(sql) => Ok(sql.into_pyobject(py)?.into_any().unbind()), - Err(error) => Ok(compiler_error(error).into_value(py).into_any()), - }) - .collect() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs new file mode 100644 index 0000000000..6ff8145be2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -0,0 +1 @@ +pub(crate) mod project_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery.rs rename to crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 0879b08b11..8826d44899 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -27,8 +27,8 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Mutex; -use crate::bindings::_helpers::config_values::config_value_to_python; -use crate::bindings::_helpers::functions::map_to_python; +use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree, SqlTestDiscoveryRequest}; use crate::bindings::types::CompilerDetach; @@ -178,7 +178,7 @@ fn discover_model_files( }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::panics::compiler_error)?; + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match discovered { Ok(files) => Ok(Some(files_object(py, files, model_object)?)), Err(_deferral) => Ok(None), @@ -193,7 +193,7 @@ fn discover_declaration_layout( ) -> PyResult> { let layout: Result = py .compiler_detach(|| Ok(declaration_layout(&tree.inner))) - .map_err(crate::bindings::_helpers::panics::compiler_error)?; + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match layout { Ok(layout) => Ok(Some(( valid_rows(layout.file_facts, fact_row), @@ -286,7 +286,7 @@ fn discover_sql_test_files( }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::panics::compiler_error)?; + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match discovered { Ok(files) => Ok(Some(files_object(py, files, test_file_object)?)), Err(_deferral) => Ok(None), @@ -305,7 +305,7 @@ fn discover_scenario_files( }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::panics::compiler_error)?; + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match discovered { Ok(files) => Ok(Some(files_object(py, files, scenario_object)?)), Err(_deferral) => Ok(None), @@ -337,7 +337,7 @@ fn load_yaml_files( ) -> PyResult>> { let loaded: Result, StageDeferral> = py .compiler_detach(|| Ok(load_yaml(&tree.inner, &relative_paths))) - .map_err(crate::bindings::_helpers::panics::compiler_error)?; + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match loaded { Ok(outcomes) => Ok(Some( outcomes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs deleted file mode 100644 index 6bfc7c7c37..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs +++ /dev/null @@ -1,599 +0,0 @@ -//! Register the versioned Python boundary for the native Rules engine. - -use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; - -use crate::bindings::models::ParsedRulesRequest; -use sqlbuild_rules::configuration::main::load; -use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; -use sqlbuild_rules::engine::main::{evaluate, evaluate_parsed, parse_parts}; -use sqlbuild_rules::models::CatalogueResponse; -use sqlbuild_rules::rules::main::{catalogue, selected_codes}; -use std::sync::Mutex; - -const SKILL_OWNER: &str = "sqlbuild"; -use crate::bindings::_helpers::panics::compiler_guard; -use crate::bindings::types::CompilerDetach; -const SKILL_IDENTITY: &str = "sqlbuild-rules"; - -#[pyfunction] -fn normalize_analysis_sql( - py: Python<'_>, - request: crate::bindings::models::NormalizationInput, -) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( - request.into(), - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql( - sql, dialect, - ) - }) - .map_err(value_error) -} - -/// Normalize a batch in one detached call; a failed member is returned as its exception. -#[pyfunction] -fn normalize_analysis_sqls( - py: Python<'_>, - dialect: &str, - requests: Vec, -) -> PyResult>> { - let results = py.compiler_detach(|| { - Ok( - sqlbuild_analysis::semantic_validation::main::normalize_batch::normalize_analysis_sqls( - dialect, requests, None, - ), - ) - }); - crate::bindings::_helpers::panics::normalization_results(py, results.map_err(value_error)?) -} - -#[pyfunction] -fn binding_diagnostics( - py: Python<'_>, - sql: &str, - dialect: &str, - rows: Vec, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( - sql, dialect, rows, - ) - }) - .map_err(value_error) -} - -fn value_error(error: impl std::fmt::Display) -> PyErr { - crate::bindings::_helpers::panics::compiler_error(error) -} - -#[pyfunction] -fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| evaluate::evaluate_json(request_json)) - .map_err(value_error) -} - -#[pyfunction] -fn parse_rules_parts( - py: Python<'_>, - request_json: Bound<'_, PyBytes>, - model_jsons: Vec>, - model_digests: Vec, -) -> PyResult { - let request: &[u8] = request_json.as_bytes(); - let models: Vec<&[u8]> = model_jsons.iter().map(|model| model.as_bytes()).collect(); - let parsed = py - .compiler_detach(|| parse_parts::parse_parts(request, &models, &model_digests)) - .map_err(value_error)?; - Ok(ParsedRulesRequest { - request: Mutex::new(Some(parsed)), - }) -} - -#[pyfunction] -fn evaluate_parsed_rules( - py: Python<'_>, - parsed: &Bound<'_, ParsedRulesRequest>, -) -> PyResult { - let request = parsed - .borrow() - .request - .lock() - .map_err(|_| value_error("rules request lock is poisoned"))? - .take() - .ok_or_else(|| value_error("rules request was already evaluated"))?; - py.compiler_detach(|| evaluate_parsed::evaluate_parsed(request)) - .map_err(value_error) -} - -#[pyfunction] -fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::engine::main::custom_host::run_custom_host_json(spec_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| sqlbuild_rules::sql_lint::main::engine::lint_json(request_json)) - .map_err(value_error) -} - -#[pyfunction] -fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::engine::main::finalize::finalize_findings_json(request_json) - }) - .map_err(value_error) -} - -/// One SQL lint preparation request, read from a Python mapping. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct LintPreparationRequest { - expanded: String, - before_expansion: String, - prior_sites: Vec, - dialect: String, -} - -#[pyfunction] -fn prepare_lint_sql( - py: Python<'_>, - request: LintPreparationRequest, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, - ) - }) - .map_err(value_error) -} - -/// Prepare many bodies in one detached call; `false` marks a body to prepare again individually. -#[pyfunction] -fn prepare_lint_sql_batch( - py: Python<'_>, - requests: Vec, -) -> PyResult)>> { - py.compiler_detach(|| { - Ok(requests - .iter() - .map(|request| { - sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, - ) - }) - .map_or((false, None), |prepared| (true, prepared)) - }) - .collect()) - }) - .map_err(value_error) -} - -#[pyfunction] -fn lint_backtick_identifiers(dialect: &str) -> PyResult { - compiler_guard(|| { - Ok(sqlbuild_rules::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect)) - }) -} - -#[pyfunction] -fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::batch_engine::lint_batch_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| sqlbuild_rules::sql_lint::main::formatter::format_json(request_json)) - .map_err(value_error) -} - -#[pyfunction] -fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::batch_formatter::format_batch_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) - }) - .map_err(value_error) -} - -#[pyfunction(name = "validate_sql_with_schema_json")] -fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::validation_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction(name = "validate_sql_with_schemas_json")] -fn schema_validations_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::validations_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| sqlbuild_analysis::semantic_usage::main::analyze_json(request_json)) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_column_references_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::column_references::main::analyze::analyze_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze::analyze_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze_project::analyze_project_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze_project_compact::analyze_project_compact_json( - request_json, - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_rendering::render_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn plan_and_render_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_planning::plan_and_render_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_chain_resolution::resolve_chains_json( - request_json, - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json( - request_json, - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json) - }) - .map_err(value_error) -} - -fn authored_value_to_python( - py: Python<'_>, - value: sqlbuild_sqltext::compiler::models::AuthoredValue, -) -> PyResult> { - use sqlbuild_sqltext::compiler::models::AuthoredValue; - - match value { - AuthoredValue::Null => Ok(py.None()), - AuthoredValue::Boolean(value) => { - Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) - } - AuthoredValue::BareWord(value) => { - marker_to_python(py, "word", value.into_pyobject(py)?.unbind().into_any()) - } - AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), - AuthoredValue::List(values) => { - let projected = values - .into_iter() - .map(|item| authored_value_to_python(py, item)) - .collect::>>()?; - Ok(PyList::new(py, projected)?.unbind().into_any()) - } - AuthoredValue::Map(values) => map_to_python(py, values), - AuthoredValue::Set(values) => marker_to_python( - py, - "set", - authored_value_to_python(py, AuthoredValue::List(values))?, - ), - AuthoredValue::Tuple(values) => marker_to_python( - py, - "tuple", - authored_value_to_python(py, AuthoredValue::List(values))?, - ), - AuthoredValue::TypedConstant(values) => { - marker_to_python(py, "constant", map_to_python(py, values)?) - } - AuthoredValue::InlineSqlHook(statement) => marker_to_python( - py, - "inline_sql", - statement.into_pyobject(py)?.unbind().into_any(), - ), - AuthoredValue::NamedSqlHook(name, kwargs) => hook_marker(py, "sql", name, kwargs), - AuthoredValue::PythonHook(name, kwargs) => hook_marker(py, "python", name, kwargs), - } -} - -pub(crate) fn map_to_python( - py: Python<'_>, - values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, -) -> PyResult> { - let result = PyDict::new(py); - for (key, value) in values { - result.set_item(key, authored_value_to_python(py, value)?)?; - } - Ok(result.unbind().into_any()) -} - -fn marker_to_python(py: Python<'_>, kind: &str, value: Py) -> PyResult> { - Ok( - PyTuple::new(py, [kind.into_pyobject(py)?.unbind().into_any(), value])? - .unbind() - .into_any(), - ) -} - -fn hook_marker( - py: Python<'_>, - kind: &str, - name: String, - kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, -) -> PyResult> { - let payload = PyTuple::new( - py, - [ - name.into_pyobject(py)?.unbind().into_any(), - map_to_python(py, kwargs)?, - ], - )?; - marker_to_python(py, kind, payload.unbind().into_any()) -} - -fn optional_authored_value_to_python( - py: Python<'_>, - value: Option, -) -> PyResult>> { - value - .map(|item| authored_value_to_python(py, item)) - .transpose() -} - -type ParsedModelHeader = ( - Option>, - Option>, - Option, -); - -#[pyfunction] -fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { - let parsed = py - .compiler_detach(|| { - sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) - }) - .map_err(value_error)?; - parsed - .into_iter() - .map(|(value, offsets, error)| { - Ok(( - optional_authored_value_to_python(py, value)?, - offsets, - error, - )) - }) - .collect() -} - -#[pyfunction] -fn match_model_headers( - py: Python<'_>, - contents: Vec, -) -> PyResult>> { - py.compiler_detach(|| { - Ok(sqlbuild_sqltext::compiler::main::model_header_matching::match_batch(&contents)) - }) - .map_err(value_error) -} - -#[pyfunction] -fn tokenize_model_header(header: &str) -> PyResult> { - compiler_guard(|| { - sqlbuild_sqltext::compiler::main::model_header_tokenizing::tokenize_one(header) - .map_err(value_error) - }) -} - -#[pyfunction] -fn substitute_static_project_vars( - sqls: Vec, - variables: Vec<(String, String)>, -) -> PyResult)>> { - compiler_guard(|| { - Ok( - sqlbuild_sqltext::compiler::main::sql_interpolation::substitute_batch( - &sqls, &variables, - ), - ) - }) -} - -#[pyfunction] -fn extract_static_sql_references( - sql: &str, -) -> PyResult>> { - compiler_guard(|| { - Ok(sqlbuild_sqltext::compiler::main::sql_references::extract( - sql, - )) - }) -} - -#[pyfunction] -fn load_config_json(project_dir: &str) -> PyResult { - compiler_guard(|| { - load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) - }) -} - -#[pyfunction] -fn catalogue_json() -> PyResult { - compiler_guard(|| { - serde_json::to_string(&CatalogueResponse { - version: API_VERSION, - rules: catalogue::catalogue(), - }) - .map_err(value_error) - }) -} - -#[pyfunction] -fn selected_codes_json(request_json: &str) -> PyResult { - compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) -} - -#[pyfunction] -fn render_owned_skill(content: &str, input_fingerprint: &str) -> PyResult { - compiler_guard(|| { - fensu_policy::render_owned_skill( - SKILL_OWNER, - SKILL_IDENTITY, - input_fingerprint, - content.as_bytes(), - ) - .map_err(value_error) - .and_then(|value| String::from_utf8(value).map_err(value_error)) - }) -} - -#[pyfunction] -fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { - let freshness = fensu_policy::skill_freshness( - content.map(str::as_bytes), - SKILL_OWNER, - SKILL_IDENTITY, - input_fingerprint, - ); - format!("{freshness:?}").to_lowercase() -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_json, module)?)?; - module.add_function(wrap_pyfunction!(parse_rules_parts, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_parsed_rules, module)?)?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; - module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; - module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; - module.add_function(wrap_pyfunction!(prepare_lint_sql_batch, module)?)?; - module.add( - "NativeCompilerError", - module - .py() - .get_type::(), - )?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; - module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; - module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; - module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; - module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; - module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; - module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; - module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; - module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; - module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; - module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; - module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; - module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; - module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; - module.add_function(wrap_pyfunction!(analyze_project_queries_json, module)?)?; - module.add_function(wrap_pyfunction!( - analyze_project_queries_compact_json, - module - )?)?; - module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; - module.add_function(wrap_pyfunction!(plan_and_render_sql_tests_json, module)?)?; - module.add_function(wrap_pyfunction!(resolve_sql_test_chains_json, module)?)?; - module.add_function(wrap_pyfunction!( - render_sql_test_difference_sample_json, - module - )?)?; - module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; - module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; - module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; - module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?; - module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?; - module.add_function(wrap_pyfunction!(extract_static_sql_references, module)?)?; - module.add_function(wrap_pyfunction!(load_config_json, module)?)?; - module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; - module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; - module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; - module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; - crate::bindings::_helpers::oracles::register(module)?; - crate::bindings::_helpers::discovery::register(module)?; - crate::bindings::_helpers::scopes::register(module)?; - module.add("API_VERSION", API_VERSION)?; - module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 4bb1f63809..9c489926c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,6 @@ -mod allocator; -mod catalog; -mod config_values; -pub(crate) mod conversions; -mod discovery; -pub(crate) mod functions; -mod oracles; -pub(crate) mod panics; -mod positions; -mod scopes; +pub(crate) mod analysis; +pub(crate) mod boundary; +pub(crate) mod discovery; +pub(crate) mod rules; +pub(crate) mod scopes; +pub(crate) mod sqltext; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs new file mode 100644 index 0000000000..d421144bd3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -0,0 +1,105 @@ +//! Register the versioned Python boundary for the native Rules engine. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyBytesMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::configuration::main::load; +use sqlbuild_rules::constants::API_VERSION; +use sqlbuild_rules::engine::main::{evaluate, evaluate_parsed, parse_parts}; +use sqlbuild_rules::models::CatalogueResponse; +use sqlbuild_rules::rules::main::{catalogue, selected_codes}; +use std::sync::Mutex; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::models::ParsedRulesRequest; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| evaluate::evaluate_json(request_json)) + .map_err(value_error) +} + +#[pyfunction] +fn parse_rules_parts( + py: Python<'_>, + request_json: Bound<'_, PyBytes>, + model_jsons: Vec>, + model_digests: Vec, +) -> PyResult { + let request: &[u8] = request_json.as_bytes(); + let models: Vec<&[u8]> = model_jsons.iter().map(|model| model.as_bytes()).collect(); + let parsed = py + .compiler_detach(|| parse_parts::parse_parts(request, &models, &model_digests)) + .map_err(value_error)?; + Ok(ParsedRulesRequest { + request: Mutex::new(Some(parsed)), + }) +} + +#[pyfunction] +fn evaluate_parsed_rules( + py: Python<'_>, + parsed: &Bound<'_, ParsedRulesRequest>, +) -> PyResult { + let request = parsed + .borrow() + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))? + .take() + .ok_or_else(|| value_error("rules request was already evaluated"))?; + py.compiler_detach(|| evaluate_parsed::evaluate_parsed(request)) + .map_err(value_error) +} + +#[pyfunction] +fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_rules::engine::main::custom_host::run_custom_host_json(spec_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_rules::engine::main::finalize::finalize_findings_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn load_config_json(project_dir: &str) -> PyResult { + compiler_guard(|| { + load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) + }) +} + +#[pyfunction] +fn catalogue_json() -> PyResult { + compiler_guard(|| { + serde_json::to_string(&CatalogueResponse { + version: API_VERSION, + rules: catalogue::catalogue(), + }) + .map_err(value_error) + }) +} + +#[pyfunction] +fn selected_codes_json(request_json: &str) -> PyResult { + compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(evaluate_json, module)?)?; + module.add_function(wrap_pyfunction!(parse_rules_parts, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_parsed_rules, module)?)?; + module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; + module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; + module.add_function(wrap_pyfunction!(load_config_json, module)?)?; + module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; + module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs new file mode 100644 index 0000000000..5b3f06f034 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -0,0 +1,104 @@ +//! Lint and format SQL with the native rules engine. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| sqlbuild_rules::sql_lint::main::engine::lint_json(request_json)) + .map_err(value_error) +} + +/// One SQL lint preparation request, read from a Python mapping. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct LintPreparationRequest { + expanded: String, + before_expansion: String, + prior_sites: Vec, + dialect: String, +} + +#[pyfunction] +fn prepare_lint_sql( + py: Python<'_>, + request: LintPreparationRequest, +) -> PyResult> { + py.compiler_detach(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) + }) + .map_err(value_error) +} + +/// Prepare many bodies in one detached call; `false` marks a body to prepare again individually. +#[pyfunction] +fn prepare_lint_sql_batch( + py: Python<'_>, + requests: Vec, +) -> PyResult)>> { + py.compiler_detach(|| { + Ok(requests + .iter() + .map(|request| { + sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) + }) + .map_or((false, None), |prepared| (true, prepared)) + }) + .collect()) + }) + .map_err(value_error) +} + +#[pyfunction] +fn lint_backtick_identifiers(dialect: &str) -> PyResult { + compiler_guard(|| { + Ok(sqlbuild_rules::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect)) + }) +} + +#[pyfunction] +fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_rules::sql_lint::main::batch_engine::lint_batch_json(request_json) + }) + .map_err(value_error) +} + +#[pyfunction] +fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| sqlbuild_rules::sql_lint::main::formatter::format_json(request_json)) + .map_err(value_error) +} + +#[pyfunction] +fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_rules::sql_lint::main::batch_formatter::format_batch_json(request_json) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; + module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; + module.add_function(wrap_pyfunction!(prepare_lint_sql_batch, module)?)?; + module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; + module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; + module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; + module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs new file mode 100644 index 0000000000..0d2a1be28f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod evaluation; +pub(crate) mod lint; +pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs new file mode 100644 index 0000000000..bb39d0f9c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs @@ -0,0 +1,40 @@ +//! Render and check the generated rules skill. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; + +const SKILL_OWNER: &str = "sqlbuild"; +const SKILL_IDENTITY: &str = "sqlbuild-rules"; + +#[pyfunction] +fn render_owned_skill(content: &str, input_fingerprint: &str) -> PyResult { + compiler_guard(|| { + fensu_policy::render_owned_skill( + SKILL_OWNER, + SKILL_IDENTITY, + input_fingerprint, + content.as_bytes(), + ) + .map_err(value_error) + .and_then(|value| String::from_utf8(value).map_err(value_error)) + }) +} + +#[pyfunction] +fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { + let freshness = fensu_policy::skill_freshness( + content.map(str::as_bytes), + SKILL_OWNER, + SKILL_IDENTITY, + input_fingerprint, + ); + format!("{freshness:?}").to_lowercase() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; + module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs new file mode 100644 index 0000000000..2aec50a4ca --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -0,0 +1 @@ +pub(crate) mod scope_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes.rs rename to crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 30c9aab065..4df342b0ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -14,7 +14,7 @@ use sqlbuild_scopes::scope_index::models::{ DeclarationKind, ResourceKind, ResourceRoot, ScopeKind, }; -use crate::bindings::_helpers::panics::compiler_error; +use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; type IdentityRow = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs new file mode 100644 index 0000000000..381c3d0ab8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -0,0 +1,94 @@ +//! Authored header values as the Python objects the model header parser builds. + +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; + +fn authored_value_to_python( + py: Python<'_>, + value: sqlbuild_sqltext::compiler::models::AuthoredValue, +) -> PyResult> { + use sqlbuild_sqltext::compiler::models::AuthoredValue; + + match value { + AuthoredValue::Null => Ok(py.None()), + AuthoredValue::Boolean(value) => { + Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) + } + AuthoredValue::BareWord(value) => { + marker_to_python(py, "word", value.into_pyobject(py)?.unbind().into_any()) + } + AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), + AuthoredValue::List(values) => { + let projected = values + .into_iter() + .map(|item| authored_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, projected)?.unbind().into_any()) + } + AuthoredValue::Map(values) => map_to_python(py, values), + AuthoredValue::Set(values) => marker_to_python( + py, + "set", + authored_value_to_python(py, AuthoredValue::List(values))?, + ), + AuthoredValue::Tuple(values) => marker_to_python( + py, + "tuple", + authored_value_to_python(py, AuthoredValue::List(values))?, + ), + AuthoredValue::TypedConstant(values) => { + marker_to_python(py, "constant", map_to_python(py, values)?) + } + AuthoredValue::InlineSqlHook(statement) => marker_to_python( + py, + "inline_sql", + statement.into_pyobject(py)?.unbind().into_any(), + ), + AuthoredValue::NamedSqlHook(name, kwargs) => hook_marker(py, "sql", name, kwargs), + AuthoredValue::PythonHook(name, kwargs) => hook_marker(py, "python", name, kwargs), + } +} + +pub(crate) fn map_to_python( + py: Python<'_>, + values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, +) -> PyResult> { + let result = PyDict::new(py); + for (key, value) in values { + result.set_item(key, authored_value_to_python(py, value)?)?; + } + Ok(result.unbind().into_any()) +} + +fn marker_to_python(py: Python<'_>, kind: &str, value: Py) -> PyResult> { + Ok( + PyTuple::new(py, [kind.into_pyobject(py)?.unbind().into_any(), value])? + .unbind() + .into_any(), + ) +} + +fn hook_marker( + py: Python<'_>, + kind: &str, + name: String, + kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, +) -> PyResult> { + let payload = PyTuple::new( + py, + [ + name.into_pyobject(py)?.unbind().into_any(), + map_to_python(py, kwargs)?, + ], + )?; + marker_to_python(py, kind, payload.unbind().into_any()) +} + +pub(crate) fn optional_authored_value_to_python( + py: Python<'_>, + value: Option, +) -> PyResult>> { + value + .map(|item| authored_value_to_python(py, item)) + .transpose() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs new file mode 100644 index 0000000000..b02161ba3e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod authored_values; +pub(crate) mod model_headers; +pub(crate) mod static_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs new file mode 100644 index 0000000000..7a6740be79 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -0,0 +1,59 @@ +//! Match, tokenize and parse model headers natively. + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::sqltext::authored_values::optional_authored_value_to_python; +use crate::bindings::types::CompilerDetach; + +type ParsedModelHeader = ( + Option>, + Option>, + Option, +); + +#[pyfunction] +fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { + let parsed = py + .compiler_detach(|| { + sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) + }) + .map_err(value_error)?; + parsed + .into_iter() + .map(|(value, offsets, error)| { + Ok(( + optional_authored_value_to_python(py, value)?, + offsets, + error, + )) + }) + .collect() +} + +#[pyfunction] +fn match_model_headers( + py: Python<'_>, + contents: Vec, +) -> PyResult>> { + py.compiler_detach(|| { + Ok(sqlbuild_sqltext::compiler::main::model_header_matching::match_batch(&contents)) + }) + .map_err(value_error) +} + +#[pyfunction] +fn tokenize_model_header(header: &str) -> PyResult> { + compiler_guard(|| { + sqlbuild_sqltext::compiler::main::model_header_tokenizing::tokenize_one(header) + .map_err(value_error) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; + module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; + module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs new file mode 100644 index 0000000000..caefa71b05 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -0,0 +1,37 @@ +//! Substitute static project variables and extract static SQL references. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +#[pyfunction] +fn substitute_static_project_vars( + sqls: Vec, + variables: Vec<(String, String)>, +) -> PyResult)>> { + compiler_guard(|| { + Ok( + sqlbuild_sqltext::compiler::main::sql_interpolation::substitute_batch( + &sqls, &variables, + ), + ) + }) +} + +#[pyfunction] +fn extract_static_sql_references( + sql: &str, +) -> PyResult>> { + compiler_guard(|| { + Ok(sqlbuild_sqltext::compiler::main::sql_references::extract( + sql, + )) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?; + module.add_function(wrap_pyfunction!(extract_static_sql_references, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 2eb16ea645..3bdbd5fd32 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -1,7 +1,33 @@ -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::pymodule; +use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::{analysis, boundary, discovery, rules, scopes, sqltext}; +use crate::bindings::models; #[pymodule] pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::functions::register(module) + rules::evaluation::register(module)?; + rules::lint::register(module)?; + rules::skills::register(module)?; + module.add_class::()?; + module.add( + "NativeCompilerError", + module.py().get_type::(), + )?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + analysis::normalization::register(module)?; + analysis::queries::register(module)?; + analysis::sql_tests::register(module)?; + sqltext::model_headers::register(module)?; + sqltext::static_sql::register(module)?; + boundary::oracles::register(module)?; + discovery::project_files::register(module)?; + scopes::scope_index::register(module)?; + module.add("API_VERSION", API_VERSION)?; + module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; + Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index bf31ede3ef..cc5d970dbe 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -21,7 +21,7 @@ pub(crate) struct CatalogInput { pub(crate) quoted_ignore_case: bool, pub(crate) known_functions: Vec, pub(crate) known_types: Vec, - #[pyo3(from_py_with = crate::bindings::_helpers::conversions::relations)] + #[pyo3(from_py_with = crate::bindings::_helpers::analysis::conversions::relations)] pub(crate) relations: Relations, } From 56537ee120178b8b73b9e1c1af845ce54c322329 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:39:19 +0100 Subject: [PATCH 5/7] feat(native): add a native-preview compiler engine tier Each native stage declares a shipped or preview tier in one registry. The native engine runs shipped stages only (discovery); native-preview adds stages still in development (declaration scopes). Preview keeps its own compiler, Rules and reuse stores. The per-PR differential compares python with native-preview on the full corpus and python with native on the seeds and failure corpus. --- .github/workflows/verify.yml | 7 +- Makefile | 11 +- crates/README.md | 22 ++++ scripts/compiler_differential/constants.py | 3 +- .../main/differential.py | 13 ++- .../_helpers/attachment/declaration_scope.py | 6 +- .../compiler/discovery/main/discover.py | 23 ++-- src/sqlbuild/compiler/frontier/constants.py | 19 ++- .../frontier/main/engine_cache_name.py | 7 +- .../frontier/main/native_stage_enabled.py | 11 ++ src/sqlbuild/compiler/frontier/types.py | 15 +++ .../scripts/compiler_differential/helpers.py | 2 +- .../test_compiler_differential.py | 6 +- .../test_scope_differential.py | 13 ++- .../cli/commands/main/compile/_test_types.py | 11 ++ .../cli/commands/main/compile/helpers.py | 2 +- .../compile/test_compiler_engine_tiers.py | 109 ++++++++++++++++++ .../sqlbuild/compiler/scopes/_test_types.py | 9 ++ .../src/sqlbuild/compiler/scopes/helpers.py | 37 +++++- .../scopes/test_native_scope_parity.py | 29 +++++ .../_helpers/comparing/test_normalize.py | 5 + .../entry/test_compiler_engine_option.py | 11 +- .../sqlbuild/compiler/frontier/_test_types.py | 12 +- .../compiler/frontier/test_cache_namespace.py | 18 ++- .../frontier/test_native_stage_tiers.py | 64 ++++++++++ .../frontier/test_resolve_compiler_engine.py | 24 +++- 26 files changed, 450 insertions(+), 39 deletions(-) create mode 100644 src/sqlbuild/compiler/frontier/main/native_stage_enabled.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_tiers.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index f950f7e307..426162c215 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -290,11 +290,13 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras - - name: Compare fixtures, examples, generated seeds and the failure corpus + - name: Compare fixtures, examples, generated seeds and the failure corpus (python vs native-preview) run: >- make compiler-differential COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage --require-render-coverage" + - name: Compare generated seeds and the failure corpus (python vs shipped native) + run: make compiler-differential-shipped compiler_differential_python_versions: name: Native discovery on Python ${{ matrix.python }} @@ -333,11 +335,12 @@ jobs: uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py -n auto --dist loadfile - - name: Compare engines on fixtures, examples, the failure corpus and a few seeds + - name: Compare python with the shipped native default on fixtures, examples, the failure corpus and a few seeds env: PYTHONUNBUFFERED: "1" run: >- uv run --no-sync python -m scripts.run_compiler_differential + --engines python native --corpus fixtures examples failures seeds --seeds 4 --jobs 4 format_performance: diff --git a/Makefile b/Makefile index 9e4a3f80ff..c5e1fd2f08 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ SHELL := /bin/bash test-e2e-duckdb-integrations test-e2e-performance \ test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ test-e2e-dense-compile-performance test-e2e-varied-cache-performance \ - compiler-differential compiler-differential-dense + compiler-differential compiler-differential-shipped compiler-differential-dense format: uv run ruff format . @@ -289,6 +289,7 @@ test-e2e-varied-cache-performance: COMPILER_DIFFERENTIAL_JOBS ?= 4 COMPILER_DIFFERENTIAL_SEEDS ?= 12 +COMPILER_DIFFERENTIAL_SHIPPED_SEEDS ?= 12 COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000 COMPILER_DIFFERENTIAL_ARGS ?= @@ -297,6 +298,14 @@ compiler-differential: --jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \ $(COMPILER_DIFFERENTIAL_ARGS) +# python vs native-preview above covers every native stage; this keeps the shipped `native` default +# covered on a bounded corpus. +compiler-differential-shipped: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ + --engines python native --corpus seeds failures \ + --jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SHIPPED_SEEDS) \ + $(COMPILER_DIFFERENTIAL_ARGS) + compiler-differential-dense: env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ --corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \ diff --git a/crates/README.md b/crates/README.md index c6775425b4..2eec4615b8 100644 --- a/crates/README.md +++ b/crates/README.md @@ -44,6 +44,28 @@ The same check fails when the workspace version, the `pyproject.toml` version an `fensu check` applies the Rust structure rules inside each crate: other crates use items through `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`. +## Compiler engines + +`SQLBUILD_COMPILER_ENGINE` (or the hidden `--compiler-engine` flag) selects which compiler +stages run natively: + +| Engine | Runs | +|---|---| +| `python` | The Python compiler only. It is the oracle every native stage must match byte for byte. | +| `native` | The default: native stages that passed their flip gate (`shipped` tier). | +| `native-preview` | Opt-in: shipped stages plus stages still in development (`preview` tier). | + +Each native stage declares its tier once, in `NATIVE_STAGE_TIERS` in +`src/sqlbuild/compiler/frontier/constants.py`. A stage moves from `preview` to `shipped` by +changing that line, after its flip gate passes: a byte-identical real project, a green +differential, and no measured slowdown on cold, edit and no-change compiles. Every engine keeps +its own compiler, Rules and compile-reuse stores, so preview output is never reused by `native`. + +`make compiler-differential` compares `python` with `native-preview` on the full per-PR corpus. +`make compiler-differential-shipped` compares `python` with `native` on the generated seeds and +the failure corpus, so the shipped default stays covered on its own. CI runs both, and compares +`python` with `native` on Python 3.13 and 3.14 as well. + ## Working on the crates - Tests live next to the module they cover; run them for one crate with diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 8b72d0b5c7..05f3f8cdac 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -7,7 +7,8 @@ ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" STAGE_CAPTURE_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" -DEFAULT_ENGINES: tuple[str, str] = ("python", "native") +ENGINE_NAMES: tuple[str, ...] = ("python", "native", "native-preview") +DEFAULT_ENGINES: tuple[str, str] = ("python", "native-preview") SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index fdb49af5c1..314fc3ceab 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -28,6 +28,7 @@ DEFAULT_DENSE_MODELS, DEFAULT_ENGINES, DEFAULT_SEED_COUNT, + ENGINE_NAMES, EXPECT_SUCCESS, PER_PULL_REQUEST_CORPORA, ) @@ -113,7 +114,17 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--seeds", type=int, default=DEFAULT_SEED_COUNT) parser.add_argument("--seed-start", type=int, default=0) parser.add_argument("--dense-models", type=int, default=DEFAULT_DENSE_MODELS) - parser.add_argument("--engines", nargs=2, default=list(DEFAULT_ENGINES), metavar="ENGINE") + parser.add_argument( + "--engines", + nargs=2, + choices=ENGINE_NAMES, + default=list(DEFAULT_ENGINES), + metavar="ENGINE", + help=( + "oracle and candidate engines (default: python native-preview, every native stage; " + "pass python native to cover only the shipped native stages)" + ), + ) parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1) parser.add_argument( "--stage-captures", diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 331bccd8ed..5e510999bd 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -22,8 +22,8 @@ ScopeRelationshipBuild, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine -from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index @@ -43,7 +43,7 @@ def build_declaration_scope( ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" - if resolve_compiler_engine() is CompilerEngine.NATIVE: + if native_stage_enabled(NativeStage.DECLARATION_SCOPES): native_scope: DeclarationScopeBuild | None = _build_native_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index e298c16593..394aaa28bd 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -31,7 +31,8 @@ from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier -from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import CompilerStage, NativeStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -58,14 +59,18 @@ def discover_project_inputs( declaration_reuse=declaration_reuse, native=False, ), - native_stage=partial( - _discover_project_inputs, - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, - declaration_reuse=declaration_reuse, - native=True, + native_stage=( + partial( + _discover_project_inputs, + project_dir=project_dir, + sql_analysis_enabled_override=sql_analysis_enabled_override, + extract_output_column_locations=extract_output_column_locations, + cache_request=cache_request, + declaration_reuse=declaration_reuse, + native=True, + ) + if native_stage_enabled(NativeStage.DISCOVERY) + else None ), ) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 09445fd0de..cfc1e9fd6d 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -2,13 +2,28 @@ import re -from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.frontier.types import CompilerEngine, NativeStage, NativeStageTier COMPILER_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" COMPILER_ENGINE_OPTION: str = "--compiler-engine" DEFAULT_COMPILER_ENGINE: CompilerEngine = CompilerEngine.NATIVE COMPILER_ENGINE_VALUES: tuple[str, ...] = tuple(engine.value for engine in CompilerEngine) NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" +NATIVE_PREVIEW_CACHE_NAMESPACE_SUFFIX: str = "-native-preview-v1" +ENGINE_CACHE_NAMESPACE_SUFFIXES: dict[CompilerEngine, str] = { + CompilerEngine.PYTHON: "", + CompilerEngine.NATIVE: NATIVE_CACHE_NAMESPACE_SUFFIX, + CompilerEngine.NATIVE_PREVIEW: NATIVE_PREVIEW_CACHE_NAMESPACE_SUFFIX, +} +NATIVE_STAGE_TIERS: dict[NativeStage, NativeStageTier] = { + NativeStage.DISCOVERY: NativeStageTier.SHIPPED, + NativeStage.DECLARATION_SCOPES: NativeStageTier.PREVIEW, +} +ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { + CompilerEngine.PYTHON: frozenset(), + CompilerEngine.NATIVE: frozenset({NativeStageTier.SHIPPED}), + CompilerEngine.NATIVE_PREVIEW: frozenset({NativeStageTier.SHIPPED, NativeStageTier.PREVIEW}), +} TARGET_DIRECTORY_NAME: str = "target" CACHE_DIRECTORY_NAME: str = "cache" COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" @@ -23,7 +38,7 @@ ) STAGE_CAPTURE_INVOCATION_ID_PATTERN: re.Pattern[str] = re.compile(r"\b\d{8}T\d{6}Z_[0-9a-f]{12}\b") STAGE_CAPTURE_INVOCATION_ID_MASK: str = "" -STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native-v\d+\b") +STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native(?:-preview)?-v\d+\b") STAGE_CAPTURE_SKIPPED_SLOTS: frozenset[str] = frozenset({"__dict__", "__weakref__"}) STAGE_CAPTURE_UNORDERED_ATTRIBUTES: dict[str, frozenset[str]] = { "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( diff --git a/src/sqlbuild/compiler/frontier/main/engine_cache_name.py b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py index da1eca742c..5bcefb17b6 100644 --- a/src/sqlbuild/compiler/frontier/main/engine_cache_name.py +++ b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py @@ -1,13 +1,10 @@ """Engine namespacing for on-disk compiler stores.""" from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine -from sqlbuild.compiler.frontier.constants import NATIVE_CACHE_NAMESPACE_SUFFIX -from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.frontier.constants import ENGINE_CACHE_NAMESPACE_SUFFIXES def engine_cache_name(base: str) -> str: """Name an on-disk store so engines never read each other's entries.""" - if active_compiler_engine() is CompilerEngine.PYTHON: - return base - return f"{base}{NATIVE_CACHE_NAMESPACE_SUFFIX}" + return f"{base}{ENGINE_CACHE_NAMESPACE_SUFFIXES[active_compiler_engine()]}" diff --git a/src/sqlbuild/compiler/frontier/main/native_stage_enabled.py b/src/sqlbuild/compiler/frontier/main/native_stage_enabled.py new file mode 100644 index 0000000000..c03089cd62 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/native_stage_enabled.py @@ -0,0 +1,11 @@ +"""Whether the active compiler engine runs one native stage.""" + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier.constants import ENGINE_NATIVE_STAGE_TIERS, NATIVE_STAGE_TIERS +from sqlbuild.compiler.frontier.types import NativeStage + + +def native_stage_enabled(stage: NativeStage) -> bool: + """Return whether the active engine includes the tier `stage` is registered at.""" + + return NATIVE_STAGE_TIERS[stage] in ENGINE_NATIVE_STAGE_TIERS[active_compiler_engine()] diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index f7d4daa79d..dbeb1fe7d3 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -8,6 +8,21 @@ class CompilerEngine(StrEnum): PYTHON = "python" NATIVE = "native" + NATIVE_PREVIEW = "native-preview" + + +class NativeStage(StrEnum): + """A native implementation that replaces one Python compiler stage when its tier is active.""" + + DISCOVERY = "discovery" + DECLARATION_SCOPES = "declaration_scopes" + + +class NativeStageTier(StrEnum): + """Which engines run a native stage: shipped stages passed their flip gate.""" + + SHIPPED = "shipped" + PREVIEW = "preview" class CompilerStage(StrEnum): diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index ea3af574c8..059fbbedba 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -135,5 +135,5 @@ def perturbation_arguments(directory: Path, *, source: str = _SITECUSTOMIZE) -> return ( "--stage-captures", "--engine-env", - f"native:PYTHONPATH={write_native_perturbation(directory, source=source)}", + f"native-preview:PYTHONPATH={write_native_perturbation(directory, source=source)}", ) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index f66275611d..ba415b93f9 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -33,7 +33,7 @@ expected_exit_code=0, expected_lines=( "OK project/waffle_shop", - "Compiler differential passed: 1 projects identical (python vs native)", + "Compiler differential passed: 1 projects identical (python vs native-preview)", ), expected_patterns=(), expected_absent=("DIFF",), @@ -64,9 +64,9 @@ def test_given_identical_engines_when_comparing_fixture_then_harness_passes( expected_lines=( "DIFF project/waffle_shop", "- `compile` stdout at /resources/models/0/query_sql", - 'native: "SELECT * FROM (', + 'native-preview: "SELECT * FROM (', "- `compile` stderr at line 1", - f'native: "{NATIVE_ONLY_STDERR_LINE}"', + f'native-preview: "{NATIVE_ONLY_STDERR_LINE}"', "- stage capture 0-compile/003-compiled_project.json at /models/0/query_sql", "First difference: project/waffle_shop: `compile` stdout at", ), diff --git a/tests/e2e/scripts/compiler_differential/test_scope_differential.py b/tests/e2e/scripts/compiler_differential/test_scope_differential.py index f618764021..5a3032b21d 100644 --- a/tests/e2e/scripts/compiler_differential/test_scope_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_scope_differential.py @@ -51,13 +51,24 @@ description="scoped_declarations_with_relationship_grants", extra_arguments=(), expected_exit_code=0, + expected_lines=( + "OK project/scoped_orders", + "Compiler differential passed: 1 projects identical (python vs native-preview)", + ), + expected_patterns=(), + expected_absent=("DIFF",), + ), + HarnessRunTestCase( + description="shipped_native_stages_only", + extra_arguments=("--engines", "python", "native"), + expected_exit_code=0, expected_lines=( "OK project/scoped_orders", "Compiler differential passed: 1 projects identical (python vs native)", ), expected_patterns=(), expected_absent=("DIFF",), - ) + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index e1a9dba792..fe4db75a86 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -829,3 +829,14 @@ class CompilerEngineRulesStoreTestCase: native_marker: str expected_python_files: tuple[str, ...] expected_native_files: tuple[str, ...] + + +@dataclass(frozen=True) +class CompilerEngineTierStoreTestCase: + """Alternating compiles under two native tiers and the stores each tier must own alone.""" + + description: str + engines: tuple[str, ...] + expected_reused: tuple[bool, ...] + expected_first_engine_stores: tuple[str, ...] + expected_second_engine_stores: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 4e793077a0..f389e87a07 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4151,7 +4151,7 @@ def json_report_keys(stdout: str) -> tuple[str, ...]: return tuple(cast(dict[str, object], json.loads(stdout))) -_COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z]+)",') +_COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z-]+)",') def copy_compile_project(*, source: Path, destination: Path) -> Path: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_tiers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_tiers.py new file mode 100644 index 0000000000..519651f2d4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_tiers.py @@ -0,0 +1,109 @@ +"""The native-preview tier matches Python and never shares stores with the shipped native tier.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompilerEngineParityTestCase, + CompilerEngineTierStoreTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + engine_reuse_compile, + prepare_compile_reuse_project, + report_engine, + report_without_engine, + store_digests, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineParityTestCase( + description="python_and_native_preview", + left_engine="python", + right_engine="native-preview", + expected_engines=("python", "native-preview"), + expected_exit_codes=(0, 0), + ) + ], + ids=lambda case: case.description, +) +def test_given_same_project_when_compiling_with_preview_tier_then_outputs_match_python( + test_case: CompilerEngineParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + left: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "left"), + engine=test_case.left_engine, + ) + right: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "right"), + engine=test_case.right_engine, + ) + + assert (left.returncode, right.returncode) == test_case.expected_exit_codes + assert (report_engine(left), report_engine(right)) == test_case.expected_engines + assert report_without_engine(right) == report_without_engine(left) + assert right.compiled == left.compiled + assert left.compiled + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineTierStoreTestCase( + description="native_preview_preview_native", + engines=("native", "native-preview", "native-preview", "native"), + expected_reused=(False, False, True, True), + expected_first_engine_stores=( + "target/cache/compiler-native-v1", + "target/rules-cache-native-v1", + ), + expected_second_engine_stores=( + "target/cache/compiler-native-preview-v1", + "target/rules-cache-native-preview-v1", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_shipped_native_compile_when_preview_compiles_then_no_store_is_shared( + test_case: CompilerEngineTierStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=project_dir) + + first: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[0] + ) + first_stores: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_first_engine_stores + ) + later: list[CompileReuseRun] = [ + engine_reuse_compile(project_dir=project_dir, engine=engine) + for engine in test_case.engines[1:-1] + ] + first_stores_after_preview: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_first_engine_stores + ) + last: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[-1] + ) + runs: list[CompileReuseRun] = [first, *later, last] + + assert tuple(run.reused for run in runs) == test_case.expected_reused + assert first_stores + assert first_stores_after_preview == first_stores + assert store_digests(project_dir=project_dir, stores=test_case.expected_second_engine_stores) + assert {report_without_engine(run) for run in runs[2:]} == {report_without_engine(last)} + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index 4af8c1e711..d1c3179396 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -33,3 +33,12 @@ class ScopeCommandParityTestCase: description: str files: dict[str, str] expected_minimum_grants: int + + +@dataclass(frozen=True) +class ScopeEngineTierTestCase: + """One compiler engine and whether it runs the preview-tier native declaration scopes.""" + + description: str + engine: str + expected_native_attempts: int diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index f71e36c379..a86da39dc4 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -3,13 +3,14 @@ from __future__ import annotations import random -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -18,6 +19,8 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main.load_or_build_scope_index import load_or_build_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup @@ -258,7 +261,7 @@ def scope_engine_outcomes( discovered=discovered, macros=macros, syntax=syntax, - engine="native", + engine=CompilerEngine.NATIVE_PREVIEW.value, monkeypatch=monkeypatch, ) built: bool = ( @@ -274,11 +277,39 @@ def scope_command_indexes( monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") python: ScopeIndex = load_or_build_scope_index(project_dir=project_dir, no_cache=True) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "native") + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) native: ScopeIndex = load_or_build_scope_index(project_dir=project_dir, no_cache=True) return python, native +def native_scope_attempts( + *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[int, ScopeOutcome]: + """Build the declaration scope under `engine`; count how often the native index was opened.""" + + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) + attempts: list[None] = [] + + def counted( + *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] + ) -> NativeScopeIndex | None: + attempts.append(None) + return open_native_scope_index( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + ) + + monkeypatch.setattr(declaration_scope, "open_native_scope_index", counted) + outcome: ScopeOutcome = _scope_outcome( + discovered=discovered, + macros=macros, + syntax=DuckDbAdapter().sql_lexical_syntax, + engine=engine, + monkeypatch=monkeypatch, + ) + return len(attempts), outcome + + def _scope_outcome( *, discovered: DiscoveredProjectInputs, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index 595595448b..db93b64bc9 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -12,11 +12,13 @@ GeneratedScopeParityTestCase, ScopeCommandParityTestCase, ScopeEngineParityTestCase, + ScopeEngineTierTestCase, ) from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( SCOPED_PROJECT, ScopeOutcome, generated_scope_files, + native_scope_attempts, scope_command_indexes, scope_engine_outcomes, write_project, @@ -161,5 +163,32 @@ def test_given_scoped_project_when_scope_command_builds_index_then_engines_match ), test_case.description +@pytest.mark.parametrize( + "test_case", + [ + ScopeEngineTierTestCase( + description="python oracle", engine="python", expected_native_attempts=0 + ), + ScopeEngineTierTestCase( + description="shipped native stages only", engine="native", expected_native_attempts=0 + ), + ScopeEngineTierTestCase( + description="native preview", engine="native-preview", expected_native_attempts=1 + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_tier_when_building_scope_then_native_scopes_run_only_in_preview( + test_case: ScopeEngineTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=SCOPED_PROJECT) + + attempts, outcome = native_scope_attempts( + project_dir=tmp_path, engine=test_case.engine, monkeypatch=monkeypatch + ) + + assert (attempts, outcome.kind) == (test_case.expected_native_attempts, "ok") + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py index 05f84df19b..71d3d265aa 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py +++ b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py @@ -34,6 +34,11 @@ raw="wrote target/cache/compiler-native-v1/facts", expected_text="wrote target/cache/compiler/facts", ), + NormalizationTestCase( + description="preview_engine_store_path", + raw="wrote target/rules-cache-native-preview-v1/bulk", + expected_text="wrote target/rules-cache/bulk", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py index 273ba2609d..fc4a0629f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py @@ -30,6 +30,12 @@ environment_value="", expected_engine="native", ), + CompilerEngineOptionTestCase( + description="preview_flag", + argv=("--compiler-engine", "native-preview", "compile"), + environment_value="", + expected_engine="native-preview", + ), CompilerEngineOptionTestCase( description="flag_after_subcommand", argv=("plan", "--compiler-engine", "python"), @@ -73,7 +79,10 @@ def test_given_engine_selection_when_parsing_then_flag_value_is_recorded( argv=("compile",), environment_value="rust", expected_exit_code=2, - expected_error="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + expected_error=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native, native-preview " + "(got 'rust')" + ), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py index b35c5513e4..10a3546659 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -5,7 +5,7 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage, NativeStage @dataclass(frozen=True) @@ -105,3 +105,13 @@ class SharedCaptureTestCase: description: str value: Callable[[], object] expected_shared_nodes: int + + +@dataclass(frozen=True) +class NativeStageTierTestCase: + """One engine, one native stage, and whether that engine runs it.""" + + description: str + engine: CompilerEngine + stage: NativeStage + expected_enabled: bool diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py index cf5b751b56..8f284670f9 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py @@ -37,10 +37,16 @@ base="rules-cache", expected_name="rules-cache-native-v1", ), + EngineCacheNameTestCase( + description="native_preview_gets_its_own_suffix", + engine=CompilerEngine.NATIVE_PREVIEW, + base="compiler", + expected_name="compiler-native-preview-v1", + ), ], ids=lambda case: case.description, ) -def test_given_engine_when_naming_store_then_only_native_is_suffixed( +def test_given_engine_when_naming_store_then_only_native_engines_are_suffixed( test_case: EngineCacheNameTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) @@ -71,6 +77,16 @@ def test_given_engine_when_naming_store_then_only_native_is_suffixed( "target/rules-cache-native-v1/bulk/sql.json", ), ), + EngineStorePathsTestCase( + description="native_preview_paths_separate", + engine=CompilerEngine.NATIVE_PREVIEW, + expected_paths=( + "target/cache/compiler-native-preview-v1", + "target/cache/compiler-native-preview-v1", + "target/cache/compiler-native-preview-v1/declaration-scopes-v2", + "target/rules-cache-native-preview-v1/bulk/sql.json", + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py new file mode 100644 index 0000000000..5cb419510d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -0,0 +1,64 @@ +"""Each engine runs exactly the native stages whose registered tier it includes.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import CompilerEngine, NativeStage +from tests.unit.src.sqlbuild.compiler.frontier._test_types import NativeStageTierTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + NativeStageTierTestCase( + description="python_discovery", + engine=CompilerEngine.PYTHON, + stage=NativeStage.DISCOVERY, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="python_declaration_scopes", + engine=CompilerEngine.PYTHON, + stage=NativeStage.DECLARATION_SCOPES, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_discovery", + engine=CompilerEngine.NATIVE, + stage=NativeStage.DISCOVERY, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="native_declaration_scopes", + engine=CompilerEngine.NATIVE, + stage=NativeStage.DECLARATION_SCOPES, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_discovery", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.DISCOVERY, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="native_preview_declaration_scopes", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.DECLARATION_SCOPES, + expected_enabled=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_checking_native_stage_then_only_its_tiers_run( + test_case: NativeStageTierTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + + assert native_stage_enabled(test_case.stage) is test_case.expected_enabled + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py index ea3f6a405d..3180d6d89e 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py @@ -30,6 +30,11 @@ EngineResolutionTestCase( description="native", raw_value="native", expected_engine=CompilerEngine.NATIVE ), + EngineResolutionTestCase( + description="native_preview", + raw_value="native-preview", + expected_engine=CompilerEngine.NATIVE_PREVIEW, + ), ], ids=lambda case: case.description, ) @@ -64,20 +69,33 @@ def test_given_no_engine_variable_when_resolving_then_native_is_the_default( EngineErrorTestCase( description="unknown", raw_value="rust", - expected_message="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native, native-preview " + "(got 'rust')" + ), ), EngineErrorTestCase( description="wrong_case", raw_value="Native", expected_message=( - "SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'Native')" + "SQLBUILD_COMPILER_ENGINE must be one of python, native, native-preview " + "(got 'Native')" + ), + ), + EngineErrorTestCase( + description="underscored_preview", + raw_value="native_preview", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native, native-preview " + "(got 'native_preview')" ), ), EngineErrorTestCase( description="padded", raw_value=" python", expected_message=( - "SQLBUILD_COMPILER_ENGINE must be one of python, native (got ' python')" + "SQLBUILD_COMPILER_ENGINE must be one of python, native, native-preview " + "(got ' python')" ), ), ], From 3d9c2ca0c1690258e80e76b7f07bef5e25dab9d6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:58:53 +0100 Subject: [PATCH 6/7] feat(native): scan SQL tests natively for expected-model relationships Under native-preview the declaration scope reads each model-mode SQL test's and scenario's __expected__ names with a native port of the Python top-level CTE scan, using the adapter's lexical rules. Any text Python would reject, or a non-ASCII character at a code position, defers that body to the Python extraction, so names and fault messages stay byte-identical. --- Cargo.lock | 3 + crates/README.md | 9 +- .../src/bindings/_helpers/scopes/mod.rs | 1 + .../_helpers/scopes/relationship_names.rs | 60 ++++++ .../src/bindings/main/register.rs | 1 + crates/sqlbuild-scopes/Cargo.toml | 3 + crates/sqlbuild-scopes/src/lib.rs | 1 + .../relationship_names/_helpers/cte_scan.rs | 189 ++++++++++++++++++ .../src/relationship_names/_helpers/mod.rs | 1 + .../main/expected_model_names.rs | 16 ++ .../src/relationship_names/main/mod.rs | 1 + .../src/relationship_names/mod.rs | 6 + .../src/relationship_names/models.rs | 10 + .../src/relationship_names/tests.rs | 6 + .../src/relationship_names/tests/helpers.rs | 39 ++++ .../tests/test_expected_model_names.rs | 157 +++++++++++++++ .../relationship_names/tests/test_types.rs | 16 ++ .../src/sql_scan/main/dialect_non_code_end.rs | 13 ++ .../sqlbuild-sqltext/src/sql_scan/main/mod.rs | 1 + src/sqlbuild/_native.pyi | 3 + .../attachment/scope_relationships.py | 67 ++++++- .../main/_native_expected_model_names.py | 32 +++ .../sqlbuild/compiler/scopes/_test_types.py | 13 ++ .../src/sqlbuild/compiler/scopes/helpers.py | 136 +++++++++++++ .../scopes/test_native_relationship_names.py | 54 +++++ 25 files changed, 828 insertions(+), 10 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/main/mod.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/mod.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/models.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/tests.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs create mode 100644 crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_end.rs create mode 100644 src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py create mode 100644 tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py diff --git a/Cargo.lock b/Cargo.lock index c8be4c3ca6..ab2ed994fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -712,6 +712,9 @@ dependencies = [ [[package]] name = "sqlbuild-scopes" version = "0.139.0" +dependencies = [ + "sqlbuild-sqltext", +] [[package]] name = "sqlbuild-sqltext" diff --git a/crates/README.md b/crates/README.md index 2eec4615b8..9b52dca374 100644 --- a/crates/README.md +++ b/crates/README.md @@ -10,7 +10,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. | -| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants and the scope lookup groups. Anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -25,9 +25,10 @@ sqlbuild-python -> sqlbuild-sqltext -> sqlbuild-core ``` -`sqlbuild-config` and `sqlbuild-scopes` do not depend on the crates below them today; their -place in the order only fixes which crates may use them. The JSON emitter lives in -`sqlbuild-core` so that any layer can produce text that must equal Python's `json.dumps` output. +`sqlbuild-config` does not depend on the crates below it today, and `sqlbuild-scopes` uses only +`sqlbuild-sqltext`; their place in the order fixes which crates may use them. The JSON emitter +lives in `sqlbuild-core` so that any layer can produce text that must equal Python's +`json.dumps` output. Each crate may depend only on crates below it, and may also skip layers. The order is declared in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 2aec50a4ca..a2c248dbbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -1 +1,2 @@ +pub(crate) mod relationship_names; pub(crate) mod scope_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs new file mode 100644 index 0000000000..6f32b4c81f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -0,0 +1,60 @@ +//! Expected-model relationship names of SQL tests and scenarios, scanned natively. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; +use sqlbuild_scopes::relationship_names::models::ExpectedNames; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +/// One adapter's `SqlLexicalSyntax`, read from a Python mapping. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct LexicalSyntaxInput { + backslash_escape_quotes: Vec, + escape_string_prefix: bool, + raw_string_prefix: bool, + triple_quoted_strings: bool, + nested_block_comments: bool, + line_comment_prefixes: Vec, +} + +impl From for LexicalSyntax { + fn from(input: LexicalSyntaxInput) -> Self { + Self { + backslash_escape_quotes: input.backslash_escape_quotes, + escape_string_prefix: input.escape_string_prefix, + raw_string_prefix: input.raw_string_prefix, + triple_quoted_strings: input.triple_quoted_strings, + nested_block_comments: input.nested_block_comments, + line_comment_prefixes: input.line_comment_prefixes, + } + } +} + +/// Return each body's expected-model names, or `None` where Python must extract them. +#[pyfunction] +fn scope_expected_model_names( + py: Python<'_>, + sqls: Vec, + syntax: LexicalSyntaxInput, +) -> PyResult>>> { + let syntax: LexicalSyntax = syntax.into(); + let outcomes: Vec = py + .compiler_detach(|| Ok(expected_model_names(&sqls, &syntax))) + .map_err(compiler_error)?; + Ok(outcomes + .into_iter() + .map(|outcome| match outcome { + ExpectedNames::Scanned(names) => Some(names), + ExpectedNames::Deferred => None, + }) + .collect()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 3bdbd5fd32..924a4d638c 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -27,6 +27,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { boundary::oracles::register(module)?; discovery::project_files::register(module)?; scopes::scope_index::register(module)?; + scopes::relationship_names::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-scopes/Cargo.toml b/crates/sqlbuild-scopes/Cargo.toml index acb059c867..90a67d20a5 100644 --- a/crates/sqlbuild-scopes/Cargo.toml +++ b/crates/sqlbuild-scopes/Cargo.toml @@ -7,5 +7,8 @@ license.workspace = true publish.workspace = true rust-version.workspace = true +[dependencies] +sqlbuild-sqltext.workspace = true + [lints] workspace = true diff --git a/crates/sqlbuild-scopes/src/lib.rs b/crates/sqlbuild-scopes/src/lib.rs index 42fe53ff2b..ea218f103e 100644 --- a/crates/sqlbuild-scopes/src/lib.rs +++ b/crates/sqlbuild-scopes/src/lib.rs @@ -2,4 +2,5 @@ #![forbid(unsafe_code)] +pub mod relationship_names; pub mod scope_index; diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs new file mode 100644 index 0000000000..d43bf704e9 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs @@ -0,0 +1,189 @@ +//! The top-level CTE scan of Python `extract_sql_test_expected_model_names`, reproduced exactly. + +use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashSet; + +/// Python raises here, or a non-ASCII code character needs Python's `str` classification. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Deferred; + +type Scan = Result; + +const EXPECTED_PREFIX: &[u8] = b"__expected__"; +const QUOTE_TOKENS: &[u8] = b"'\"`$"; +/// Python's parenthesis matcher only looks for comments at these characters. +const PAREN_SCAN_SPECIALS: &[u8] = b"-/#'\"`$()"; + +/// Return the expected-model names of one SQL test or scenario body. +pub(crate) fn expected_names(sql: &[u8], syntax: &LexicalSyntax) -> Scan> { + if !syntax.line_comment_prefixes.iter().all(|prefix| { + prefix.is_ascii() + && prefix + .as_bytes() + .first() + .is_some_and(|first| PAREN_SCAN_SPECIALS.contains(first)) + }) { + return Err(Deferred); + } + let start = skip_ignorable(sql, 0, syntax)?; + if try_consume_keyword(sql, start, b"WITH")?.is_none() { + return Ok(Vec::new()); + } + let (names, end) = scan_top_level_ctes(sql, syntax)?; + if !(trailing_statement_end(sql, end, syntax)? || trailing_ceremonial_select(sql, end, syntax)?) + { + return Err(Deferred); + } + names + .into_iter() + .filter_map(|name| name.strip_prefix(EXPECTED_PREFIX).map(<[u8]>::to_vec)) + .map(|model| { + if model.is_empty() { + Err(Deferred) + } else { + String::from_utf8(model).map_err(|_| Deferred) + } + }) + .collect() +} + +fn scan_top_level_ctes<'sql>( + sql: &'sql [u8], + syntax: &LexicalSyntax, +) -> Scan<(Vec<&'sql [u8]>, usize)> { + let with_end = + try_consume_keyword(sql, skip_ignorable(sql, 0, syntax)?, b"WITH")?.ok_or(Deferred)?; + let mut index = skip_ignorable(sql, with_end, syntax)?; + if let Some(recursive_end) = try_consume_keyword(sql, index, b"RECURSIVE")? { + index = skip_ignorable(sql, recursive_end, syntax)?; + } + let mut names: Vec<&[u8]> = Vec::new(); + let mut seen: HashSet<&[u8]> = HashSet::new(); + loop { + let (name, name_end) = read_identifier(sql, index)?; + if !seen.insert(name) { + return Err(Deferred); + } + names.push(name); + index = skip_ignorable(sql, name_end, syntax)?; + if sql.get(index) == Some(&b'(') { + index = matching_paren(sql, index, syntax)? + 1; + index = skip_ignorable(sql, index, syntax)?; + } + index = try_consume_keyword(sql, index, b"AS")?.ok_or(Deferred)?; + index = skip_ignorable(sql, index, syntax)?; + if sql.get(index) != Some(&b'(') { + return Err(Deferred); + } + index = skip_ignorable(sql, matching_paren(sql, index, syntax)? + 1, syntax)?; + if sql.get(index) == Some(&b',') { + index = skip_ignorable(sql, index + 1, syntax)?; + continue; + } + return Ok((names, index)); + } +} + +fn trailing_ceremonial_select(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { + let index = skip_ignorable(sql, start, syntax)?; + let Some(select_end) = try_consume_keyword(sql, index, b"SELECT")? else { + return Ok(false); + }; + let index = skip_ignorable(sql, select_end, syntax)?; + if sql.get(index) != Some(&b'1') { + return Ok(false); + } + let index = skip_ignorable(sql, index + 1, syntax)?; + trailing_statement_end(sql, index, syntax) +} + +fn trailing_statement_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { + let mut index = start; + if sql.get(index) == Some(&b';') { + index = skip_ignorable(sql, index + 1, syntax)?; + } + Ok(index == sql.len()) +} + +fn skip_ignorable(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { + let mut index = start; + while let Some(&byte) = sql.get(index) { + if !byte.is_ascii() { + return Err(Deferred); + } + if is_python_space(byte) { + index += 1; + continue; + } + if QUOTE_TOKENS.contains(&byte) { + return Ok(index); + } + match dialect_non_code_end(sql, index, syntax) { + Ok(Some(end)) => index = end, + Ok(None) => return Ok(index), + Err(_) => return Err(Deferred), + } + } + Ok(index) +} + +fn try_consume_keyword(sql: &[u8], start: usize, keyword: &[u8]) -> Scan> { + let end = start + keyword.len(); + let window = &sql[start.min(sql.len())..end.min(sql.len())]; + if !window.is_ascii() { + return Err(Deferred); + } + if !window.eq_ignore_ascii_case(keyword) { + return Ok(None); + } + if let Some(&next) = sql.get(end) { + if !next.is_ascii() { + return Err(Deferred); + } + if is_identifier_character(next) { + return Ok(None); + } + } + if let Some(&previous) = start.checked_sub(1).and_then(|index| sql.get(index)) { + if !previous.is_ascii() { + return Err(Deferred); + } + if is_identifier_character(previous) { + return Ok(None); + } + } + Ok(Some(end)) +} + +fn read_identifier(sql: &[u8], start: usize) -> Scan<(&[u8], usize)> { + let first = *sql.get(start).ok_or(Deferred)?; + if !(first.is_ascii_alphabetic() || first == b'_') { + return Err(Deferred); + } + let mut index = start + 1; + while let Some(&byte) = sql.get(index) { + if !byte.is_ascii() { + return Err(Deferred); + } + if !is_identifier_character(byte) { + break; + } + index += 1; + } + Ok((&sql[start..index], index)) +} + +fn matching_paren(sql: &[u8], open: usize, syntax: &LexicalSyntax) -> Scan { + dialect_matching_paren(sql, open, syntax).map_err(|_| Deferred) +} + +fn is_identifier_character(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + +/// ASCII characters Python's `str.isspace` accepts, including the information separators. +fn is_python_space(byte: u8) -> bool { + matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs new file mode 100644 index 0000000000..7671fab062 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod cte_scan; diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs b/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs new file mode 100644 index 0000000000..46c4be07c1 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs @@ -0,0 +1,16 @@ +//! Expected-model relationship names of SQL tests and scenarios, or a deferral to Python. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relationship_names::_helpers::cte_scan::{Deferred, expected_names}; +use crate::relationship_names::models::ExpectedNames; + +/// Return each SQL text's `__expected__` names in CTE order, or a deferral to Python. +pub fn expected_model_names(sqls: &[String], syntax: &LexicalSyntax) -> Vec { + sqls.iter() + .map(|sql| match expected_names(sql.as_bytes(), syntax) { + Ok(names) => ExpectedNames::Scanned(names), + Err(Deferred) => ExpectedNames::Deferred, + }) + .collect() +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs new file mode 100644 index 0000000000..9ec2fdbfe4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs @@ -0,0 +1 @@ +pub mod expected_model_names; diff --git a/crates/sqlbuild-scopes/src/relationship_names/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/relationship_names/models.rs b/crates/sqlbuild-scopes/src/relationship_names/models.rs new file mode 100644 index 0000000000..d25b107563 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/models.rs @@ -0,0 +1,10 @@ +//! Outcomes of the native relationship-name scan. + +/// The expected-model names of one SQL text, or a deferral to Python's extraction. +#[derive(Debug, PartialEq, Eq)] +pub enum ExpectedNames { + /// The `__expected__` names in CTE order, exactly as Python extracts them. + Scanned(Vec), + /// Python raises for this text, or classifies a character the scan reached differently. + Deferred, +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests.rs b/crates/sqlbuild-scopes/src/relationship_names/tests.rs new file mode 100644 index 0000000000..12cfec4f9b --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_expected_model_names.rs"] +mod expected_model_names; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs new file mode 100644 index 0000000000..04f167aa10 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs @@ -0,0 +1,39 @@ +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relationship_names::main::expected_model_names::expected_model_names; +use crate::relationship_names::models::ExpectedNames; +use crate::relationship_names::tests::test_types::SyntaxTestCase; + +/// The generic lexical rules of the base adapter. +pub(super) fn generic_syntax() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + } +} + +/// The lexical rules one dialect case declares. +pub(super) fn dialect_syntax(test_case: &SyntaxTestCase) -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: owned(test_case.backslash_escape_quotes), + nested_block_comments: test_case.nested_block_comments, + line_comment_prefixes: owned(test_case.line_comment_prefixes), + ..LexicalSyntax::default() + } +} + +/// The native outcome for one SQL text. +pub(super) fn names(sql: &str, syntax: &LexicalSyntax) -> ExpectedNames { + expected_model_names(&[sql.to_owned()], syntax).remove(0) +} + +/// The outcome a case expects: its names, or a deferral when it has none. +pub(super) fn expected(names: Option<&[&str]>) -> ExpectedNames { + names.map_or(ExpectedNames::Deferred, |names| { + ExpectedNames::Scanned(owned(names)) + }) +} + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs new file mode 100644 index 0000000000..f9fa4eb5f1 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs @@ -0,0 +1,157 @@ +use crate::relationship_names::tests::helpers::{dialect_syntax, expected, generic_syntax, names}; +use crate::relationship_names::tests::test_types::{ExpectedNamesTestCase, SyntaxTestCase}; + +#[test] +fn given_sql_test_when_scanning_expected_models_then_names_match_python_or_defer() { + let test_cases = [ + ExpectedNamesTestCase { + description: "expected CTEs in authored order after mocks", + sql: "WITH\n__source__raw AS (SELECT 1),\n__expected__orders AS (SELECT 2),\n\ + __expected__customers AS (SELECT 3)\nSELECT 1\n", + expected_names: Some(&["orders", "customers"]), + }, + ExpectedNamesTestCase { + description: "comments, column lists, RECURSIVE and an omitted SELECT 1", + sql: "-- header\n/* note */ with recursive __expected__a (x, y) AS (\n\ + SELECT ')' AS x, 1 AS y -- )\n)", + expected_names: Some(&["a"]), + }, + ExpectedNamesTestCase { + description: "a terminated ceremonial select with trailing comments", + sql: "WITH __expected__a AS (SELECT 1) select 1 ; -- done\n", + expected_names: Some(&["a"]), + }, + ExpectedNamesTestCase { + description: "no top-level WITH has no expected models", + sql: "SELECT 1", + expected_names: Some(&[]), + }, + ExpectedNamesTestCase { + description: "a keyword prefix of an identifier is not WITH", + sql: "WITHIN AS (SELECT 1)", + expected_names: Some(&[]), + }, + ExpectedNamesTestCase { + description: "non-ASCII inside bodies, strings and comments is opaque", + sql: "WITH __expected__a AS (SELECT 'caf\u{e9}' AS \"\u{e9}\") -- \u{e9}\nSELECT 1", + expected_names: Some(&["a"]), + }, + ExpectedNamesTestCase { + description: "non-ASCII at a code position defers", + sql: "WITH __expected__caf\u{e9} AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "non-ASCII whitespace defers", + sql: "WITH\u{a0}__expected__a AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a non-ASCII character Python upper-cases into a keyword defers", + sql: "w\u{131}th __expected__a AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "Python separators count as whitespace", + sql: "WITH\u{1f}__expected__a AS (SELECT 1)\u{b}SELECT 1", + expected_names: Some(&["a"]), + }, + ExpectedNamesTestCase { + description: "a bare expected prefix defers to Python's error", + sql: "WITH __expected__ AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a duplicate CTE defers to Python's error", + sql: "WITH a AS (SELECT 1), a AS (SELECT 2) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a final statement other than SELECT 1 defers", + sql: "WITH __expected__a AS (SELECT 1) SELECT 2", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a missing AS defers", + sql: "WITH __expected__a (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "an unclosed parenthesis defers", + sql: "WITH __expected__a AS (SELECT (1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "an unclosed block comment defers", + sql: "/* open\nWITH __expected__a AS (SELECT 1)", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a quoted CTE name defers to Python's error", + sql: "WITH \"__expected__a\" AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ExpectedNamesTestCase { + description: "a dollar sign at a code position defers", + sql: "WITH __expected__a AS (SELECT 1) $x", + expected_names: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + names(test_case.sql, &generic_syntax()), + expected(test_case.expected_names), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_follow_them() { + let test_cases = [ + SyntaxTestCase { + description: "hash line comments hide a closing parenthesis", + backslash_escape_quotes: &[], + nested_block_comments: false, + line_comment_prefixes: &["--", "#"], + sql: "WITH __expected__a AS (SELECT 1 # )\n) SELECT 1", + expected_names: Some(&["a"]), + }, + SyntaxTestCase { + description: "nested block comments close at the outer delimiter", + backslash_escape_quotes: &[], + nested_block_comments: true, + line_comment_prefixes: &["--"], + sql: "/* a /* b */ ) */ WITH __expected__a AS (SELECT 1) SELECT 1", + expected_names: Some(&["a"]), + }, + SyntaxTestCase { + description: "backslash-escaped quotes keep the string open", + backslash_escape_quotes: &["'"], + nested_block_comments: false, + line_comment_prefixes: &["--"], + sql: "WITH __expected__a AS (SELECT 'it\\')' AS x) SELECT 1", + expected_names: Some(&["a"]), + }, + SyntaxTestCase { + description: "a line comment prefix Python's matcher never looks for defers", + backslash_escape_quotes: &[], + nested_block_comments: false, + line_comment_prefixes: &["--", ";;"], + sql: "WITH __expected__a AS (SELECT 1) SELECT 1", + expected_names: None, + }, + ]; + + for test_case in test_cases { + let syntax = dialect_syntax(&test_case); + assert_eq!( + names(test_case.sql, &syntax), + expected(test_case.expected_names), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs new file mode 100644 index 0000000000..2f43ea29cb --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs @@ -0,0 +1,16 @@ +/// One SQL text and the expected-model names the native scan returns, or `None` for a deferral. +pub(super) struct ExpectedNamesTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_names: Option<&'static [&'static str]>, +} + +/// One dialect's lexical rules as the Python adapter declares them. +pub(super) struct SyntaxTestCase { + pub(super) description: &'static str, + pub(super) backslash_escape_quotes: &'static [&'static str], + pub(super) nested_block_comments: bool, + pub(super) line_comment_prefixes: &'static [&'static str], + pub(super) sql: &'static str, + pub(super) expected_names: Option<&'static [&'static str]>, +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_end.rs new file mode 100644 index 0000000000..128b5261d2 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_end.rs @@ -0,0 +1,13 @@ +//! Dialect-aware boundaries of the comment or quoted text starting at one offset. + +use crate::sql_scan::_helpers::dialect::non_code_end; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +/// Return the end of a comment or quoted text starting at `index` under the dialect's rules. +pub fn dialect_non_code_end( + sql: &[u8], + index: usize, + syntax: &LexicalSyntax, +) -> Result, Unclosed> { + non_code_end(sql, index, syntax) +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs index dcd98888be..2b5b0542f4 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -1,5 +1,6 @@ pub mod comment_end; pub mod dialect_matching_paren; +pub mod dialect_non_code_end; pub mod dialect_non_code_ranges; pub mod matching_paren; pub mod non_code_end; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ee7a3037fd..acb25ca011 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -127,6 +127,9 @@ def build_native_scope_index( ] ], ) -> NativeScopeIndex | None: ... +def scope_expected_model_names( + sqls: list[str], syntax: dict[str, object] +) -> list[list[str] | None]: ... def substitute_static_project_vars( sqls: list[str], variables: list[tuple[str, str]] ) -> list[tuple[int, str | None]]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index cc93264a23..46758adddc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path from typing import cast @@ -29,6 +30,9 @@ from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.scopes.main._native_expected_model_names import ( + native_expected_model_names, +) from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( resolve_scope_declaration_visibility, ) @@ -119,18 +123,32 @@ def extract_scope_relationship_facts( for file_index, cache_key in cache_keys.items() ) ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - cache_key: str | None = cache_keys.get(file_index) - cached_file_names: tuple[tuple[str, ...], ...] | None = _cached_expected_names( - value=None if cache_key is None else cached_names.get(cache_key), + cached_by_file: list[tuple[tuple[str, ...], ...] | None] = [ + _cached_expected_names( + value=cached_names.get(cache_keys[file_index]) + if file_index in cache_keys + else None, block_count=len(test_file.blocks), ) + for file_index, test_file in enumerate(discovered_inputs.test_files) + ] + scanned_names: dict[tuple[int, int], tuple[str, ...]] = _scanned_expected_names( + test_files=discovered_inputs.test_files, + cached_by_file=cached_by_file, + syntax=sql_lexical_syntax, + ) + for file_index, test_file in enumerate(discovered_inputs.test_files): + cache_key: str | None = cache_keys.get(file_index) + cached_file_names: tuple[tuple[str, ...], ...] | None = cached_by_file[file_index] extracted_names: list[tuple[str, ...]] = [] for block_index, block in enumerate(test_file.blocks): + scanned: tuple[str, ...] | None = scanned_names.get((file_index, block_index)) try: expected_names: tuple[str, ...] = ( cached_file_names[block_index] if cached_file_names is not None + else scanned + if scanned is not None else extract_sql_test_expected_model_names( sql=block.sql_body, file_label=str(test_file.relative_path), @@ -170,12 +188,18 @@ def extract_scope_relationship_facts( slot=_expected_models_fact_slot(test_file), value=tuple(extracted_names), ) - for scenario in discovered_inputs.scenario_files: + scenario_names: list[tuple[str, ...] | None] = native_expected_model_names( + sqls=[scenario.sql_body for scenario in discovered_inputs.scenario_files], + syntax=sql_lexical_syntax, + ) + for scenario, scanned in zip(discovered_inputs.scenario_files, scenario_names, strict=True): try: facts.append( RelationshipFact( resource=ResourceIdentity(ResourceKind.SCENARIO, scenario.name), - expected_models=extract_sql_scenario_expected_model_names( + expected_models=scanned + if scanned is not None + else extract_sql_scenario_expected_model_names( sql=scenario.sql_body, file_label=str(scenario.relative_path), syntax=sql_lexical_syntax, @@ -187,6 +211,37 @@ def extract_scope_relationship_facts( return tuple(facts), faults[0].message if faults else None +def _scanned_expected_names( + *, + test_files: Sequence[DiscoveredSqlTestFile], + cached_by_file: Sequence[tuple[tuple[str, ...], ...] | None], + syntax: SqlLexicalSyntax, +) -> dict[tuple[int, int], tuple[str, ...]]: + """Scan uncached model-mode blocks natively, keeping only the names the scan reproduces.""" + + pending: list[tuple[int, int]] = [] + for file_index, test_file in enumerate(test_files): + if cached_by_file[file_index] is not None: + continue + pending.extend( + (file_index, block_index) + for block_index, block in enumerate(test_file.blocks) + if block.mode is SqlTestMode.MODEL + ) + scanned: list[tuple[str, ...] | None] = native_expected_model_names( + sqls=[ + test_files[file_index].blocks[block_index].sql_body + for file_index, block_index in pending + ], + syntax=syntax, + ) + return { + position: names + for position, names in zip(pending, scanned, strict=True) + if names is not None + } + + def _tested_macro_names(*, sql: str, file_label: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( sql=sql, file_label=file_label, syntax=syntax diff --git a/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py b/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py new file mode 100644 index 0000000000..0f37b7504f --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py @@ -0,0 +1,32 @@ +"""Expected-model relationship names scanned natively for the native declaration scope.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + + +def native_expected_model_names( + *, sqls: Sequence[str], syntax: SqlLexicalSyntax +) -> list[tuple[str, ...] | None]: + """Return each body's expected-model names, or None where Python must extract them.""" + + if not sqls: + return [] + try: + names: list[list[str] | None] = _native.scope_expected_model_names( + list(sqls), + { + "backslash_escape_quotes": sorted(syntax.backslash_escape_quotes), + "escape_string_prefix": syntax.escape_string_prefix, + "raw_string_prefix": syntax.raw_string_prefix, + "triple_quoted_strings": syntax.triple_quoted_strings, + "nested_block_comments": syntax.nested_block_comments, + "line_comment_prefixes": sorted(syntax.line_comment_prefixes), + }, + ) + except (TypeError, UnicodeError): + return [None] * len(sqls) + return [None if model_names is None else tuple(model_names) for model_names in names] diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index d1c3179396..846f45dbfe 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -42,3 +42,16 @@ class ScopeEngineTierTestCase: description: str engine: str expected_native_attempts: int + + +@dataclass(frozen=True) +class ExpectedNameScanTestCase: + """Seeded SQL test bodies scanned for expected models natively and in Python.""" + + description: str + syntax: str + seed: int + count: int + expected_minimum_scanned: int + expected_minimum_deferred: int + expected_minimum_python_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index a86da39dc4..8a18d8df9c 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -5,6 +5,7 @@ import random from collections.abc import Callable, Mapping from dataclasses import dataclass +from itertools import compress from pathlib import Path import pytest @@ -13,18 +14,24 @@ from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_expected_model_names from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex +from sqlbuild.compiler.scopes.main._native_expected_model_names import ( + native_expected_model_names, +) from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main.load_or_build_scope_index import load_or_build_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from tests.integration.src.sqlbuild.compiler.helpers import mismatches PROJECT_CONFIG: str = ( 'name = "scope_parity"\nadapter = "duckdb"\n\n[connection]\ndatabase = "scope.duckdb"\n' @@ -347,3 +354,132 @@ def _lookup_shape(lookup: ScopeLookup) -> tuple[object, ...]: tuple(lookup.visibility_index.local_positions.items()), tuple(lookup.visibility_index.inherited_positions.items()), ) + + +_LEADING: tuple[str, ...] = ( + "", + "-- header\n", + "/* note ) */ ", + "# hash comment\n", + "// slash comment\n", + "/* outer /* inner */ ) */ ", + "\n\t ", + "\x1c", + "\u00a0", + "'quoted' ", +) +_WITH: tuple[str, ...] = ( + "WITH ", + "with\n", + "WITH RECURSIVE ", + "With/*c*/", + "w\u0131th ", + "WITHIN ", +) +_CTE_NAMES: tuple[str, ...] = ( + "__expected__orders", + "__expected__customers", + "__source__raw_orders", + "__ref__stg_orders", + "__assert__totals", + "__expected__", + "__expected__caf\u00e9", + '"__expected__quoted"', + "helper", +) +_COLUMN_LISTS: tuple[str, ...] = ("", " (order_id, amount)", "(x)") +_AS: tuple[str, ...] = (" AS ", " as\n", "AS", " ") +_BODIES: tuple[str, ...] = ( + "(SELECT 1 AS order_id)", + "(SELECT ')' AS x, '(' AS y)", + "(SELECT 'it\\')' AS x)", + "(SELECT $$ ) $$ AS x)", + "(SELECT $tag$ ( $tag$ AS x)", + "(SELECT '''a ) b''' AS x)", + "(SELECT r'\\' AS x) ", + "(SELECT E'\\')' AS x)", + "(SELECT 1 -- )\n)", + "(SELECT 1 # )\n)", + '(SELECT "caf\u00e9" FROM t)', + "(SELECT (1)", +) +_TRAILING: tuple[str, ...] = ( + "", + "\nSELECT 1\n", + " select 1 ;", + " SELECT 1; -- done\n", + " SELECT 2", + " SELECT 1 /* open", + "\u00a0SELECT 1", + " $x", + " ; ", +) +LEXICAL_SYNTAXES: dict[str, SqlLexicalSyntax] = { + "generic": SqlLexicalSyntax(), + "backslash_triple_raw": SqlLexicalSyntax( + backslash_escape_quotes=frozenset({"'", '"'}), + raw_string_prefix=True, + triple_quoted_strings=True, + line_comment_prefixes=frozenset({"--", "#"}), + ), + "escape_prefix": SqlLexicalSyntax(escape_string_prefix=True), + "nested_comments": SqlLexicalSyntax(nested_block_comments=True), + "slash_comments": SqlLexicalSyntax(line_comment_prefixes=frozenset({"--", "//"})), +} + + +@dataclass(frozen=True) +class ExpectedNameScanParity: + """How the native expected-model scan compared with Python over one corpus.""" + + mismatches: list[tuple[object, object, object]] + scanned: int + deferred: int + python_errors: int + + +def generated_expected_model_sqls(*, rng: random.Random, count: int) -> list[str]: + """Return seeded SQL test bodies mixing valid, dialect-sensitive and invalid CTE layouts.""" + + return [_generated_expected_model_sql(rng=rng) for _ in range(count)] + + +def _generated_expected_model_sql(*, rng: random.Random) -> str: + ctes: list[str] = [ + f"{rng.choice(_CTE_NAMES)}{rng.choice(_COLUMN_LISTS)}{rng.choice(_AS)}{rng.choice(_BODIES)}" + for _ in range(rng.randint(1, 4)) + ] + return f"{rng.choice(_LEADING)}{rng.choice(_WITH)}{','.join(ctes)}{rng.choice(_TRAILING)}" + + +def expected_name_scan_parity( + *, sqls: list[str], syntax: SqlLexicalSyntax +) -> ExpectedNameScanParity: + """Compare the native scan with Python wherever the native scan does not defer.""" + + native: list[tuple[str, ...] | None] = native_expected_model_names(sqls=sqls, syntax=syntax) + python: list[tuple[str, ...] | str] = [ + _python_expected_names(sql=sql, syntax=syntax) for sql in sqls + ] + scanned: list[tuple[str, tuple[str, ...] | str, tuple[str, ...] | None]] = list( + compress(zip(sqls, python, native, strict=True), [names is not None for names in native]) + ) + return ExpectedNameScanParity( + mismatches=mismatches( + inputs=[sql for sql, _, _ in scanned], + expected=[expected for _, expected, _ in scanned], + actual=[names for _, _, names in scanned], + ), + scanned=sum(names is not None for names in native), + deferred=sum(names is None for names in native), + python_errors=sum(isinstance(expected, str) for expected in python), + ) + + +def _python_expected_names(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[str, ...] | str: + try: + return extract_sql_test_expected_model_names( + sql=sql, file_label="tests/unit/test.sql", syntax=syntax, mode=SqlTestMode.MODEL + ) + except CompileInputError as error: + return str(error) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py new file mode 100644 index 0000000000..8c35db8819 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py @@ -0,0 +1,54 @@ +"""The native expected-model scan returns Python's names exactly or defers to Python.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.compiler.scopes._test_types import ExpectedNameScanTestCase +from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( + LEXICAL_SYNTAXES, + ExpectedNameScanParity, + expected_name_scan_parity, + generated_expected_model_sqls, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedNameScanTestCase( + description=f"{syntax} dialect", + syntax=syntax, + seed=20261007 + offset, + count=3000, + expected_minimum_scanned=300, + expected_minimum_deferred=300, + expected_minimum_python_errors=300, + ) + for offset, syntax in enumerate(LEXICAL_SYNTAXES) + ], + ids=lambda case: case.description, +) +def test_given_generated_sql_tests_when_scanning_expected_models_then_native_matches_or_defers( + test_case: ExpectedNameScanTestCase, +) -> None: + sqls: list[str] = generated_expected_model_sqls( + rng=random.Random(test_case.seed), count=test_case.count + ) + + parity: ExpectedNameScanParity = expected_name_scan_parity( + sqls=sqls, syntax=LEXICAL_SYNTAXES[test_case.syntax] + ) + + assert ( + parity.mismatches, + parity.scanned >= test_case.expected_minimum_scanned, + parity.deferred >= test_case.expected_minimum_deferred, + parity.python_errors >= test_case.expected_minimum_python_errors, + ) == ([], True, True, True), (parity.scanned, parity.deferred, parity.python_errors) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 29583e350907e6bb1506d0de2fb9ebad72b5a013 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:31:06 +0100 Subject: [PATCH 7/7] ci: cover the shipped engine in the dense differential and gate on the Python version job --- .github/workflows/compiler-differential.yml | 10 ++++++++-- .github/workflows/verify.yml | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compiler-differential.yml b/.github/workflows/compiler-differential.yml index 67604d4878..140548c944 100644 --- a/.github/workflows/compiler-differential.yml +++ b/.github/workflows/compiler-differential.yml @@ -16,7 +16,7 @@ jobs: dense: name: Dense compiler differential (${{ matrix.models }} models) runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 150 strategy: fail-fast: false matrix: @@ -37,8 +37,14 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras - - name: Compare dense projects with and without custom rules + - name: Compare dense projects with and without custom rules on the preview engine env: MODELS: ${{ matrix.models }} run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}" + - name: Compare dense projects with and without custom rules on the shipped engine + env: + MODELS: ${{ matrix.models }} + run: >- + make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}" + COMPILER_DIFFERENTIAL_ARGS="--engines python native" diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 426162c215..fe41979fd2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -761,6 +761,7 @@ jobs: checks, e2e_shards, compiler_differential, + compiler_differential_python_versions, format_performance, performance, cold_compile_performance, @@ -781,6 +782,7 @@ jobs: CHECKS_RESULT: ${{ needs.checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} + COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }} FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }} PERFORMANCE_RESULT: ${{ needs.performance.result }} COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} @@ -803,6 +805,7 @@ jobs: for result in \ "${E2E_SHARDS_RESULT}" \ "${COMPILER_DIFFERENTIAL_RESULT}" \ + "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \ "${FORMAT_PERFORMANCE_RESULT}" \ "${PERFORMANCE_RESULT}" \ "${COLD_COMPILE_PERFORMANCE_RESULT}" \