diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index d1d34b11ae..37c0084a2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod declaration_files; pub(crate) mod project_files; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs new file mode 100644 index 0000000000..557b064ebe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -0,0 +1,11 @@ +//! Register the native project discovery functions and classes. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::discovery::{declaration_files, project_files}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_files::register(module)?; + declaration_files::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index d3bfe54b86..464ecd1be5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod authored_nodes; pub(crate) mod config_presence; pub(crate) mod config_templates; pub(crate) mod header_metadata; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs new file mode 100644 index 0000000000..5c67efdfdd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -0,0 +1,12 @@ +//! Register the native model configuration functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::model_config::{config_presence, config_templates, header_metadata}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + header_metadata::register(module)?; + config_presence::register(module)?; + config_templates::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs new file mode 100644 index 0000000000..ffd8bc16ee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs @@ -0,0 +1,294 @@ +//! Declaration resolution contexts built natively from a Python scope lookup's positions. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{FromPyObject, intern, pyclass, pymethods}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use sqlbuild_scopes::scope_index::main::classify_resource::classify_resource; +use sqlbuild_scopes::scope_index::models::{ + ConsumerResource, GrantEntry, VisibilityReason, VisibilityTable, VisibleEntry, +}; + +const ENUM_KIND: u8 = 0; +const CONSTANT_KIND: u8 = 1; +const MACRO_KIND: u8 = 2; + +type TableRow = ( + Vec, + HashMap>, + HashMap>, +); +type VisibleRow<'py> = (Bound<'py, PyAny>, VisibleEntry, Option>); + +/// One declaration position: identity, first record, name, kinds, runtime value and key. +#[derive(FromPyObject)] +struct PositionRow(Py, Py, Py, u8, Py, u8, u32); + +/// The Python classes and enum members the contexts are built from. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ContextClasses { + visibility_record: Py, + resolution_context: Py, + allocate: Py, + reasons: Py, +} + +/// Positions, visibility table and classes of one scope lookup. +#[pyclass(frozen, module = "sqlbuild._native")] +struct NativeDeclarationContexts { + positions: Vec, + table: VisibilityTable, + classes: ContextClasses, + lookup: LookupMappings, +} + +/// The Python lookup mappings a matching resource's private positions and grants come from. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct LookupMappings { + identity_keys: Py, + private_positions: Py, + grants_by_resource: Py, +} + +/// Dictionaries one context is assembled into, in Python's insertion order. +struct ContextParts<'py> { + enums: Bound<'py, PyDict>, + constants: Bound<'py, PyDict>, + macros: Bound<'py, PyDict>, + macro_records: Bound<'py, PyDict>, + inaccessible: [Bound<'py, PyDict>; 3], + visibility: [Bound<'py, PyDict>; 3], +} + +#[pymethods] +impl NativeDeclarationContexts { + #[new] + fn new( + positions: Vec, + table: TableRow, + classes: ContextClasses, + lookup: LookupMappings, + ) -> Self { + let (global, local, inherited) = table; + let identity_keys: Vec = positions.iter().map(|row| row.6).collect(); + Self { + table: VisibilityTable { + identity_keys, + global, + local, + inherited, + }, + positions, + classes, + lookup, + } + } + + /// Return the context Python's resolver builds for these matching resources, or None. + fn context<'py>( + &self, + py: Python<'py>, + matches: Vec>, + consumer: Bound<'py, PyAny>, + ) -> PyResult>> { + compiler_guard(|| self.classified_context(py, matches, consumer)) + } +} + +impl NativeDeclarationContexts { + fn classified_context<'py>( + &self, + py: Python<'py>, + matches: Vec>, + consumer: Bound<'py, PyAny>, + ) -> PyResult>> { + let mut visible: Vec> = Vec::new(); + let mut inaccessible: Vec = Vec::new(); + for record in matches { + let identity: Bound<'py, PyAny> = record.getattr(intern!(py, "identity"))?; + let path: String = record.getattr(intern!(py, "path"))?.extract()?; + let private: Vec = self + .lookup + .private_positions + .bind(py) + .call_method1(intern!(py, "get"), (&identity, PyTuple::empty(py)))? + .extract()?; + if private + .iter() + .any(|position| *position >= self.positions.len()) + { + return Ok(None); + } + let (grants, throughs) = self.grants(py, &identity)?; + let Ok(classified) = classify_resource( + &self.table, + &ConsumerResource { + private, + path, + grants, + }, + ) else { + return Ok(None); + }; + for entry in classified.visible { + let through: Option> = + entry.through.map(|slot| throughs[slot].clone()); + visible.push((identity.clone(), entry, through)); + } + inaccessible.extend(classified.inaccessible); + } + self.assemble(py, &visible, &inaccessible, consumer) + .map(Some) + } + + /// The resource's grants of indexed declarations, with each grant's `through` value. + fn grants<'py>( + &self, + py: Python<'py>, + identity: &Bound<'py, PyAny>, + ) -> PyResult<(Vec, Vec>)> { + let records: Bound<'py, PyAny> = self + .lookup + .grants_by_resource + .bind(py) + .call_method1(intern!(py, "get"), (identity, PyTuple::empty(py)))?; + let identity_keys: &Bound<'py, PyDict> = self.lookup.identity_keys.bind(py); + let mut grants: Vec = Vec::new(); + let mut throughs: Vec> = Vec::new(); + for grant in records.try_iter()? { + let grant: Bound<'py, PyAny> = grant?; + let Some(identity_key) = + identity_keys.get_item(grant.getattr(intern!(py, "declaration"))?)? + else { + continue; + }; + let kind: String = grant + .getattr(intern!(py, "kind"))? + .getattr(intern!(py, "value"))? + .extract()?; + grants.push(GrantEntry { + identity_key: identity_key.extract()?, + reason: if kind == VisibilityReason::TestedMacro.as_str() { + VisibilityReason::TestedMacro + } else { + VisibilityReason::ExpectedModel + }, + through: throughs.len(), + }); + throughs.push(grant.getattr(intern!(py, "through"))?); + } + Ok((grants, throughs)) + } + + fn assemble<'py>( + &self, + py: Python<'py>, + visible: &[VisibleRow<'py>], + inaccessible: &[usize], + consumer: Bound<'py, PyAny>, + ) -> PyResult> { + let parts: ContextParts<'py> = ContextParts { + enums: PyDict::new(py), + constants: PyDict::new(py), + macros: PyDict::new(py), + macro_records: PyDict::new(py), + inaccessible: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], + visibility: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], + }; + let mut records_by_identity: HashMap>> = HashMap::new(); + for (resource, entry, through) in visible { + let record: Bound<'py, PyAny> = self.visibility_record(py, resource, entry, through)?; + records_by_identity + .entry(self.positions[entry.position].6) + .or_default() + .push(record); + } + for (_, entry, _) in visible { + self.add_visible_value(py, &parts, entry.position)?; + } + for position in inaccessible { + let row: &PositionRow = &self.positions[*position]; + if let Some(target) = parts.inaccessible.get(usize::from(row.3)) { + target.set_item(row.2.bind(py), row.1.bind(py))?; + } + } + for (_, entry, _) in visible { + let row: &PositionRow = &self.positions[entry.position]; + if let Some(target) = parts.visibility.get(usize::from(row.3)) { + let records: &[Bound<'py, PyAny>] = + records_by_identity.get(&row.6).map_or(&[], Vec::as_slice); + target.set_item(row.2.bind(py), PyTuple::new(py, records)?)?; + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("enums", &parts.enums)?; + fields.set_item("constants", &parts.constants)?; + fields.set_item("inaccessible_enums", &parts.inaccessible[0])?; + fields.set_item("inaccessible_constants", &parts.inaccessible[1])?; + fields.set_item("enum_visibility", &parts.visibility[0])?; + fields.set_item("constant_visibility", &parts.visibility[1])?; + fields.set_item("macros", &parts.macros)?; + fields.set_item("macro_records", &parts.macro_records)?; + fields.set_item("macro_visibility", &parts.visibility[2])?; + fields.set_item("inaccessible_macros", &parts.inaccessible[2])?; + fields.set_item("consumer", consumer)?; + self.classes + .resolution_context + .bind(py) + .call((), Some(&fields)) + } + + fn add_visible_value<'py>( + &self, + py: Python<'py>, + parts: &ContextParts<'py>, + position: usize, + ) -> PyResult<()> { + let row: &PositionRow = &self.positions[position]; + let name: &Bound<'py, PyAny> = row.2.bind(py); + let value: &Bound<'py, PyAny> = row.4.bind(py); + match row.5 { + ENUM_KIND => parts.enums.set_item(name, value), + CONSTANT_KIND => parts.constants.set_item(name, value), + MACRO_KIND => { + parts.macros.set_item(name, value)?; + parts.macro_records.set_item(name, row.1.bind(py)) + } + _ => Ok(()), + } + } + + fn visibility_record<'py>( + &self, + py: Python<'py>, + resource: &Bound<'py, PyAny>, + entry: &VisibleEntry, + through: &Option>, + ) -> PyResult> { + let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); + let record: Bound<'py, PyAny> = self.classes.allocate.bind(py).call1((class,))?; + let attributes: Bound<'py, PyDict> = + record.getattr("__dict__")?.downcast_into::()?; + attributes.set_item("resource", resource)?; + attributes.set_item("declaration", self.positions[entry.position].0.bind(py))?; + attributes.set_item( + "reason", + self.classes + .reasons + .bind(py) + .get_item(entry.reason.as_str())?, + )?; + attributes.set_item("through", through)?; + Ok(record) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index a2c248dbbf..01543fdf62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -1,2 +1,4 @@ +pub(crate) mod declaration_contexts; +pub(crate) mod registration; pub(crate) mod relationship_names; pub(crate) mod scope_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs new file mode 100644 index 0000000000..fbac057e45 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -0,0 +1,12 @@ +//! Register the native declaration scope functions and classes. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::scopes::{declaration_contexts, relationship_names, scope_index}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_index::register(module)?; + relationship_names::register(module)?; + declaration_contexts::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs new file mode 100644 index 0000000000..3cb89d7c60 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -0,0 +1,47 @@ +//! `@enum`/`@const` reference offsets scanned natively for the preview model loop. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; +use sqlbuild_sqltext::compiler::models::{DeclarationReference, DeclarationReferenceKind}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +/// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. +type ReferenceRow = (u8, String, Option, usize, usize); + +/// Return each SQL string's references in order, or `None` where Python must expand it. +#[pyfunction] +fn scan_sql_declaration_references( + py: Python<'_>, + sqls: Vec, +) -> PyResult>>> { + let scanned: Vec>> = py + .compiler_detach(|| Ok(scan_declaration_references(&sqls))) + .map_err(compiler_error)?; + Ok(scanned.into_iter().map(reference_rows).collect()) +} + +fn reference_rows(references: Option>) -> Option> { + Some(references?.into_iter().map(reference_row).collect()) +} + +fn reference_row(reference: DeclarationReference) -> ReferenceRow { + let kind: u8 = match reference.kind { + DeclarationReferenceKind::Enum => 0, + DeclarationReferenceKind::Constant => 1, + }; + ( + kind, + reference.name, + reference.member, + reference.start, + reference.end, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_sql_declaration_references, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 2f5ab47c7e..13533443f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod authored_values; +pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; pub(crate) mod model_headers; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 8182effbbf..0e0a165845 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::sqltext::{model_headers, sql_references, static_sql}; +use crate::bindings::_helpers::sqltext::{ + declaration_references, model_headers, sql_references, static_sql, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_headers::register(module)?; static_sql::register(module)?; sql_references::register(module)?; + declaration_references::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 7600d563b7..6e226b6956 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -26,13 +26,9 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; boundary::oracles::register(module)?; - discovery::project_files::register(module)?; - discovery::declaration_files::register(module)?; - scopes::scope_index::register(module)?; - scopes::relationship_names::register(module)?; - model_config::header_metadata::register(module)?; - model_config::config_presence::register(module)?; - model_config::config_templates::register(module)?; + discovery::registration::register(module)?; + scopes::registration::register(module)?; + model_config::registration::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/consumer_positions.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/consumer_positions.rs new file mode 100644 index 0000000000..1d2821b49c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/consumer_positions.rs @@ -0,0 +1,49 @@ +//! Python's `_visible_positions`: positions a resource sees by scope, sorted. + +use crate::scope_index::_helpers::paths::{canonical_ancestors, parent}; +use crate::scope_index::models::{ + ConsumerResource, ScopeDeferral, VisibilityReason, VisibilityTable, +}; + +/// Return `(position, reason)` pairs sorted as Python sorts its tuples. +pub(crate) fn visible_positions( + table: &VisibilityTable, + resource: &ConsumerResource, +) -> Result, ScopeDeferral> { + let mut positive: Vec<(usize, VisibilityReason)> = table + .global + .iter() + .map(|position| (*position, VisibilityReason::Global)) + .collect(); + positive.extend( + resource + .private + .iter() + .map(|position| (*position, VisibilityReason::PrivateOwner)), + ); + if !table.local.is_empty() || !table.inherited.is_empty() { + let parent: String = parent(&resource.path)?; + if let Some(positions) = table.local.get(&parent) { + positive.extend( + positions + .iter() + .map(|position| (*position, VisibilityReason::LocalOwner)), + ); + } + for ancestor in canonical_ancestors(&parent) { + if let Some(positions) = table.inherited.get(ancestor) { + positive.extend( + positions + .iter() + .map(|position| (*position, VisibilityReason::InheritedAncestor)), + ); + } + } + } + positive.sort_by(|left, right| { + left.0 + .cmp(&right.0) + .then_with(|| left.1.as_str().cmp(right.1.as_str())) + }); + Ok(positive) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs index 3e418ab8e3..3e98d19382 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod consumer_positions; pub(crate) mod diagnostics; pub(crate) mod grants; pub(crate) mod identities; diff --git a/crates/sqlbuild-scopes/src/scope_index/main/classify_resource.rs b/crates/sqlbuild-scopes/src/scope_index/main/classify_resource.rs new file mode 100644 index 0000000000..ed23617b29 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/classify_resource.rs @@ -0,0 +1,70 @@ +//! Python's `_classify_resource`: ordered visible records and inaccessible positions. + +use std::collections::HashMap; + +use crate::scope_index::_helpers::consumer_positions::visible_positions; +use crate::scope_index::models::{ + ConsumerResource, GrantEntry, ResourceVisibility, ScopeDeferral, VisibilityReason, + VisibilityTable, VisibleEntry, +}; + +/// Classify every declaration for one resource as Python does; rejected paths defer. +pub fn classify_resource( + table: &VisibilityTable, + resource: &ConsumerResource, +) -> Result { + let positive: Vec<(usize, VisibilityReason)> = visible_positions(table, resource)?; + if resource.grants.is_empty() { + let mut visible_positions: Vec = vec![false; table.identity_keys.len()]; + for (position, _) in &positive { + visible_positions[*position] = true; + } + return Ok(ResourceVisibility { + visible: positive + .iter() + .map(|(position, reason)| VisibleEntry { + position: *position, + reason: *reason, + through: None, + }) + .collect(), + inaccessible: (0..table.identity_keys.len()) + .filter(|position| !visible_positions[*position]) + .collect(), + }); + } + let mut reasons: HashMap = HashMap::new(); + for (position, reason) in &positive { + reasons.insert(*position, *reason); + } + let mut grants_by_identity: HashMap> = HashMap::new(); + for grant in &resource.grants { + grants_by_identity + .entry(grant.identity_key) + .or_default() + .push(*grant); + } + let mut classified: ResourceVisibility = ResourceVisibility::default(); + for (position, identity_key) in table.identity_keys.iter().enumerate() { + let reason: Option = reasons.get(&position).copied(); + if let Some(reason) = reason { + classified.visible.push(VisibleEntry { + position, + reason, + through: None, + }); + } + match grants_by_identity.get(identity_key) { + Some(grants) => classified + .visible + .extend(grants.iter().map(|grant| VisibleEntry { + position, + reason: grant.reason, + through: Some(grant.through), + })), + None if reason.is_none() => classified.inaccessible.push(position), + None => {} + } + } + Ok(classified) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs index 2d04f761b5..7989fe667c 100644 --- a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -1,3 +1,4 @@ pub mod build_scope_index; +pub mod classify_resource; pub mod relationship_grants; pub mod scope_lookup; diff --git a/crates/sqlbuild-scopes/src/scope_index/models.rs b/crates/sqlbuild-scopes/src/scope_index/models.rs index ac712864bf..cf0a49dde0 100644 --- a/crates/sqlbuild-scopes/src/scope_index/models.rs +++ b/crates/sqlbuild-scopes/src/scope_index/models.rs @@ -1,5 +1,7 @@ //! Scope enums, inputs, records and lookup groups exchanged with the Python scope facade. +use std::collections::HashMap; + /// Python's `ResourceKind`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ResourceKind { @@ -386,3 +388,70 @@ pub struct ScopeLookupGroups { pub struct ScopeDeferral { pub reason: String, } + +/// Python's `VisibilityReason`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VisibilityReason { + Global, + InheritedAncestor, + LocalOwner, + PrivateOwner, + ExpectedModel, + TestedMacro, +} + +impl VisibilityReason { + /// Python's enum value text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Global => "global", + Self::InheritedAncestor => "inherited_ancestor", + Self::LocalOwner => "local_owner", + Self::PrivateOwner => "private_owner", + Self::ExpectedModel => "expected_model", + Self::TestedMacro => "tested_macro", + } + } +} + +/// Python's `DeclarationVisibilityIndex` without its private owners, which callers resolve. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct VisibilityTable { + /// One key per canonical declaration position; equal keys mean equal identities. + pub identity_keys: Vec, + pub global: Vec, + pub local: HashMap>, + pub inherited: HashMap>, +} + +/// One relationship grant: the declaration it opens, why, and the caller's `through` slot. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GrantEntry { + pub identity_key: u32, + pub reason: VisibilityReason, + pub through: usize, +} + +/// One resource matching the consumer: its private positions, authored path and grants. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsumerResource { + pub private: Vec, + pub path: String, + pub grants: Vec, +} + +/// One visible fact: a declaration position, the reason, and the grant's `through` slot. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VisibleEntry { + pub position: usize, + pub reason: VisibilityReason, + pub through: Option, +} + +/// What one matching resource can see, in Python's record order, and what stays inaccessible. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResourceVisibility { + pub visible: Vec, + pub inaccessible: Vec, +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs index 6ee4a5e471..eff62179a5 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_build_scope_index.rs"] mod build_scope_index; +#[path = "tests/test_classify_resource.rs"] +mod classify_resource; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_relationship_grants.rs"] diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs index ebc938bdfb..7d85257420 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs @@ -1,16 +1,21 @@ use crate::scope_index::_helpers::identities::{ declaration_text, grant_through_text, resource_text, }; +use std::collections::HashMap; + use crate::scope_index::main::build_scope_index::build_scope_index; +use crate::scope_index::main::classify_resource::classify_resource; use crate::scope_index::main::relationship_grants::relationship_grants; use crate::scope_index::main::scope_lookup::scope_lookup_groups; use crate::scope_index::models::{ - DeclarationIdentity, DeclarationInput, DeclarationKind, RelationshipFact, ResourceIdentity, - ResourceInput, ResourceKind, ResourceRoot, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, - ScopeLookupGroups, + ConsumerResource, DeclarationIdentity, DeclarationInput, DeclarationKind, GrantEntry, + RelationshipFact, ResourceIdentity, ResourceInput, ResourceKind, ResourceRoot, + ResourceVisibility, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, ScopeLookupGroups, + VisibilityTable, }; use crate::scope_index::tests::test_types::{ - DeclarationRow, FactRow, GrantTestCase, IndexFactsTestCase, LookupTestCase, ResourceRow, + ClassifyTestCase, DeclarationRow, FactRow, GrantTestCase, IndexFactsTestCase, LookupTestCase, + ResourceRow, VisibleRow, }; const GLOBAL_ROOT_FALLBACKS: [(DeclarationKind, &str); 3] = [ @@ -226,3 +231,44 @@ fn resource(kind: &str, name: &str) -> ResourceIdentity { fn owned(values: &[&str]) -> Vec { values.iter().map(|value| (*value).to_owned()).collect() } + +/// Six positions: global 0 and 4, local 2 under `models/marts`, inherited 3 and 5. +fn consumer_table() -> VisibilityTable { + VisibilityTable { + identity_keys: vec![0, 1, 2, 3, 4, 5], + global: vec![0, 4], + local: HashMap::from([("models/marts".to_owned(), vec![2])]), + inherited: HashMap::from([ + ("models".to_owned(), vec![3]), + ("models/staging".to_owned(), vec![5]), + ]), + } +} + +/// Classify one case's consumer against the shared table, or None when it defers. +pub(super) fn classified_rows( + test_case: &ClassifyTestCase, +) -> Option<(Vec, Vec)> { + let resource: ConsumerResource = ConsumerResource { + private: test_case.private.to_vec(), + path: test_case.path.to_owned(), + grants: test_case + .grants + .iter() + .map(|(identity_key, reason, through)| GrantEntry { + identity_key: *identity_key, + reason: *reason, + through: *through, + }) + .collect(), + }; + let classified: ResourceVisibility = classify_resource(&consumer_table(), &resource).ok()?; + Some(( + classified + .visible + .iter() + .map(|entry| (entry.position, entry.reason.as_str(), entry.through)) + .collect(), + classified.inaccessible, + )) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_classify_resource.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_classify_resource.rs new file mode 100644 index 0000000000..a8a479087b --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_classify_resource.rs @@ -0,0 +1,67 @@ +use crate::scope_index::models::VisibilityReason; +use crate::scope_index::tests::helpers::classified_rows; +use crate::scope_index::tests::test_types::ClassifyTestCase; + +#[test] +fn given_consumers_when_classifying_then_records_follow_python_order() { + let test_cases = [ + ClassifyTestCase { + description: "global, private, local and inherited positions sort by position", + path: "models/marts/orders.sql", + private: &[1], + grants: &[], + expected_classified: Some(( + &[ + (0, "global", None), + (1, "private_owner", None), + (2, "local_owner", None), + (3, "inherited_ancestor", None), + (4, "global", None), + ], + &[5], + )), + }, + ClassifyTestCase { + description: "grants follow the position's own reason and keep their order", + path: "models/marts/orders.sql", + private: &[], + grants: &[ + (5, VisibilityReason::TestedMacro, 7), + (0, VisibilityReason::ExpectedModel, 8), + (5, VisibilityReason::ExpectedModel, 9), + ], + expected_classified: Some(( + &[ + (0, "global", None), + (0, "expected_model", Some(8)), + (2, "local_owner", None), + (3, "inherited_ancestor", None), + (4, "global", None), + (5, "tested_macro", Some(7)), + (5, "expected_model", Some(9)), + ], + &[1], + )), + }, + ClassifyTestCase { + description: "a path escaping the project defers to Python", + path: "../orders.sql", + private: &[], + grants: &[], + expected_classified: None, + }, + ]; + + for test_case in test_cases { + let classified = classified_rows(&test_case); + + assert_eq!( + classified + .as_ref() + .map(|(visible, inaccessible)| (visible.as_slice(), inaccessible.as_slice())), + test_case.expected_classified, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs index e5d4b76d04..38fa9c8b75 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -1,3 +1,5 @@ +use crate::scope_index::models::VisibilityReason; + /// `(kind, name, path)` of one resource in walk order. pub(super) type ResourceRow = (&'static str, &'static str, &'static str); @@ -83,3 +85,15 @@ pub(super) struct ReprTestCase { pub(super) value: &'static str, pub(super) expected_repr: Option<&'static str>, } + +/// `(position, reason, through)` of one visible record. +pub(super) type VisibleRow = (usize, &'static str, Option); + +/// One consumer path, its private positions and `(identity key, reason, through)` grants. +pub(super) struct ClassifyTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) private: &'static [usize], + pub(super) grants: &'static [(u32, VisibilityReason, usize)], + pub(super) expected_classified: Option<(&'static [VisibleRow], &'static [usize])>, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/mod.rs new file mode 100644 index 0000000000..fbfe5de859 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/mod.rs @@ -0,0 +1,2 @@ +pub mod reference_syntax; +pub mod scanner; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs new file mode 100644 index 0000000000..cf18fe4765 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs @@ -0,0 +1,109 @@ +//! Python's `@enum("name").MEMBER` and `@const("name")` patterns at one reference start. + +use crate::compiler::models::DeclarationReferenceKind; + +/// One matched reference: kind, name, member and the byte offset after it. +pub(crate) struct MatchedReference<'sql> { + pub(crate) kind: DeclarationReferenceKind, + pub(crate) name: &'sql str, + pub(crate) member: Option<&'sql str>, + pub(crate) end: usize, +} + +/// Why a reference start cannot be matched natively. +pub(crate) enum ReferenceSyntax<'sql> { + /// `@enum`/`@const` followed by a word character: not a reference, keep scanning. + NotReference, + Matched(MatchedReference<'sql>), + /// Malformed or Unicode-dependent text: Python raises or decides. + Deferred, +} + +/// Match Python's `@(enum|const)\b` start and the full reference pattern at `start`. +pub(crate) fn match_reference(sql: &str, start: usize) -> ReferenceSyntax<'_> { + let bytes: &[u8] = sql.as_bytes(); + let (kind, keyword_end) = if bytes[start..].starts_with(b"@enum") { + (DeclarationReferenceKind::Enum, start + 5) + } else if bytes[start..].starts_with(b"@const") { + (DeclarationReferenceKind::Constant, start + 6) + } else { + return ReferenceSyntax::NotReference; + }; + match bytes.get(keyword_end) { + Some(byte) if !byte.is_ascii() => return ReferenceSyntax::Deferred, + Some(byte) if is_word(*byte) => return ReferenceSyntax::NotReference, + _ => {} + } + let Some(matched) = match_arguments(sql, kind, keyword_end) else { + return ReferenceSyntax::Deferred; + }; + ReferenceSyntax::Matched(matched) +} + +fn match_arguments( + sql: &str, + kind: DeclarationReferenceKind, + keyword_end: usize, +) -> Option> { + let bytes: &[u8] = sql.as_bytes(); + let mut index: usize = consume(bytes, skip_whitespace(bytes, keyword_end)?, b'(')?; + index = skip_whitespace(bytes, index)?; + let quote: u8 = *bytes + .get(index) + .filter(|byte| matches!(byte, b'\'' | b'"'))?; + let name_start: usize = index + 1; + let name_end: usize = identifier_end(bytes, name_start)?; + index = consume(bytes, name_end, quote)?; + index = consume(bytes, skip_whitespace(bytes, index)?, b')')?; + let mut member: Option<&str> = None; + if matches!(kind, DeclarationReferenceKind::Enum) { + index = consume(bytes, skip_whitespace(bytes, index)?, b'.')?; + let member_start: usize = skip_whitespace(bytes, index)?; + index = identifier_end(bytes, member_start)?; + member = Some(&sql[member_start..index]); + } + Some(MatchedReference { + kind, + name: &sql[name_start..name_end], + member, + end: index, + }) +} + +/// Python's `\s*` over ASCII; a non-ASCII byte may be Unicode whitespace, so it defers. +fn skip_whitespace(bytes: &[u8], mut index: usize) -> Option { + while let Some(byte) = bytes.get(index) { + if !byte.is_ascii() { + return None; + } + if !matches!( + byte, + b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c | 0x1c..=0x1f + ) { + break; + } + index += 1; + } + Some(index) +} + +fn consume(bytes: &[u8], index: usize, expected: u8) -> Option { + (bytes.get(index) == Some(&expected)).then_some(index + 1) +} + +/// The end of `[A-Za-z_][A-Za-z0-9_]*` starting at `start`. +fn identifier_end(bytes: &[u8], start: usize) -> Option { + let first: u8 = *bytes.get(start)?; + if !(first.is_ascii_alphabetic() || first == b'_') { + return None; + } + let mut end: usize = start + 1; + while bytes.get(end).copied().is_some_and(is_word) { + end += 1; + } + Some(end) +} + +fn is_word(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs new file mode 100644 index 0000000000..f44a3fda6a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs @@ -0,0 +1,129 @@ +//! Python's `_find_next_reference_start` walk over quoted text and comments. + +use crate::compiler::_helpers::declaration_references::reference_syntax::{ + ReferenceSyntax, match_reference, +}; +use crate::compiler::models::DeclarationReference; + +const SPECIAL_BYTES: &[u8] = b"'\"`$@/-"; + +/// Every `@enum`/`@const` reference in `sql` with code-point offsets, or None when Python decides. +pub(crate) fn scan_references(sql: &str) -> Option> { + let mut references: Vec = Vec::new(); + let mut offsets: CharOffsets = CharOffsets::default(); + let mut cursor: usize = 0; + while cursor < sql.len() { + let Some(start) = next_reference_start(sql, cursor)? else { + break; + }; + let ReferenceSyntax::Matched(matched) = match_reference(sql, start) else { + return None; + }; + let start_char: usize = offsets.advance(sql, start); + let end_char: usize = offsets.advance(sql, matched.end); + references.push(DeclarationReference { + kind: matched.kind, + name: matched.name.to_owned(), + member: matched.member.map(str::to_owned), + start: start_char, + end: end_char, + }); + cursor = matched.end; + } + Some(references) +} + +/// The next reference start at or after `start`; the outer None means Python decides. +fn next_reference_start(sql: &str, start: usize) -> Option> { + let rest: &str = &sql[start..]; + if !rest.contains("@enum") && !rest.contains("@const") { + return Some(None); + } + let bytes: &[u8] = sql.as_bytes(); + let mut index: usize = start; + while let Some(offset) = bytes[index..] + .iter() + .position(|byte| SPECIAL_BYTES.contains(byte)) + { + index += offset; + index = match bytes[index] { + b'\'' | b'"' => quoted_end(bytes, index, true)?, + b'`' => quoted_end(bytes, index, false)?, + b'$' => dollar_end(bytes, index)?.unwrap_or(index + 1), + b'-' if bytes.get(index + 1) == Some(&b'-') => bytes[index..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(bytes.len(), |newline| index + newline + 1), + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index + 2 + find(&bytes[index + 2..], b"*/")? + 2 + } + b'@' => match match_reference(sql, index) { + ReferenceSyntax::NotReference => index + 1, + ReferenceSyntax::Matched(_) | ReferenceSyntax::Deferred => { + return Some(Some(index)); + } + }, + _ => index + 1, + }; + } + Some(None) +} + +/// Python's `quoted_text_end_impl` for `'`, `"` (doubled quotes escape) and backticks. +fn quoted_end(bytes: &[u8], start: usize, doubled_escapes: bool) -> Option { + let quote: u8 = bytes[start]; + let mut index: usize = start + 1; + loop { + index += bytes[index..].iter().position(|byte| *byte == quote)?; + if doubled_escapes && bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Some(index + 1); + } +} + +/// Python's `_dollar_quoted_text_end`: Some(None) when `$` opens no literal. +fn dollar_end(bytes: &[u8], start: usize) -> Option> { + if start + .checked_sub(1) + .is_some_and(|previous| continues_dollar_word(bytes[previous])) + { + return Some(None); + } + let tag_end: usize = bytes[start + 1..] + .iter() + .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) + .map_or(bytes.len(), |offset| start + 1 + offset); + if bytes.get(tag_end) != Some(&b'$') || bytes.get(start + 1).is_some_and(u8::is_ascii_digit) { + return Some(None); + } + let delimiter: &[u8] = &bytes[start..=tag_end]; + let closing: usize = find(&bytes[tag_end + 1..], delimiter)?; + Some(Some(tag_end + 1 + closing + delimiter.len())) +} + +fn continues_dollar_word(byte: u8) -> bool { + !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// Converts increasing byte offsets into Python code-point offsets. +#[derive(Default)] +struct CharOffsets { + byte: usize, + char: usize, +} + +impl CharOffsets { + fn advance(&mut self, sql: &str, byte: usize) -> usize { + self.char += sql[self.byte..byte].chars().count(); + self.byte = byte; + self.char + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs index efcfa91e8b..934c174aee 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub mod declaration_references; pub mod model_headers; pub mod sql_interpolation; pub mod sql_references; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs new file mode 100644 index 0000000000..f6fe439602 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs @@ -0,0 +1,12 @@ +//! Public compiler entry point for scanning `@enum`/`@const` references. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; + +use crate::compiler::_helpers::declaration_references::scanner::scan_references; +use crate::compiler::models::DeclarationReference; + +/// Each SQL string's declaration references in order, or None where Python must expand it. +#[must_use] +pub fn scan_declaration_references(sqls: &[String]) -> Vec>> { + sqls.par_iter().map(|sql| scan_references(sql)).collect() +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index 0050ddac66..a9be565a74 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -1,3 +1,4 @@ +pub mod declaration_references; pub mod model_header_byte_matching; pub mod model_header_matching; pub mod model_header_parsing; diff --git a/crates/sqlbuild-sqltext/src/compiler/models.rs b/crates/sqlbuild-sqltext/src/compiler/models.rs index 3950629e32..4a6f0fd7ad 100644 --- a/crates/sqlbuild-sqltext/src/compiler/models.rs +++ b/crates/sqlbuild-sqltext/src/compiler/models.rs @@ -13,3 +13,31 @@ pub enum AuthoredValue { NamedSqlHook(String, Vec<(String, AuthoredValue)>), PythonHook(String, Vec<(String, AuthoredValue)>), } + +/// Whether a declaration reference names an enum member or a constant. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarationReferenceKind { + Enum, + Constant, +} + +impl DeclarationReferenceKind { + /// The reference keyword after `@`. + #[must_use] + pub const fn keyword(self) -> &'static str { + match self { + Self::Enum => "enum", + Self::Constant => "const", + } + } +} + +/// One `@enum("name").MEMBER` or `@const("name")` reference, with code-point offsets. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclarationReference { + pub kind: DeclarationReferenceKind, + pub name: String, + pub member: Option, + pub start: usize, + pub end: usize, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests.rs b/crates/sqlbuild-sqltext/src/compiler/tests.rs index 6b14cd92fe..97a4fcc4b8 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_declaration_references.rs"] +mod declaration_references; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_model_header_matching.rs"] diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index 8f13e1b58e..f0abc3e0dc 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -5,7 +5,8 @@ use crate::compiler::_helpers::sql_interpolation::substitution::{ FALLBACK, SUBSTITUTED, UNCHANGED, substitute_batch, }; use crate::compiler::_helpers::sql_references::extraction::extract; -use crate::compiler::models::AuthoredValue; +use crate::compiler::main::declaration_references::scan_declaration_references; +use crate::compiler::models::{AuthoredValue, DeclarationReference}; pub(crate) fn scalar_variables_preserve_lexical_boundaries() -> bool { let sqls = vec![ @@ -212,3 +213,29 @@ pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { assert_eq!(TOKENIZER_WORKER_STACK_BYTES, 16 * 1024 * 1024); true } + +/// Scan one SQL string and spell its references as `kind:name[.member]@start..end, ...`. +pub(crate) fn scanned_references(sql: &str) -> Option { + let references: Vec = scan_declaration_references(&[sql.to_owned()]) + .pop() + .flatten()?; + Some( + references + .iter() + .map(|reference| { + let member: String = reference + .member + .as_ref() + .map_or_else(String::new, |member| format!(".{member}")); + format!( + "{}:{}{member}@{}..{}", + reference.kind.keyword(), + reference.name, + reference.start, + reference.end + ) + }) + .collect::>() + .join(", "), + ) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs new file mode 100644 index 0000000000..4ab940beda --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs @@ -0,0 +1,69 @@ +use crate::compiler::tests::helpers::scanned_references; +use crate::compiler::tests::test_types::DeclarationReferenceScanTestCase; + +#[test] +fn given_sql_when_scanning_declaration_references_then_python_offsets_are_returned() { + let test_cases = [ + DeclarationReferenceScanTestCase { + description: "enum members and constants in order with code-point offsets", + sql: "SELECT 'é', @enum(\"order_status\").PLACED, @const( 'sales_cap' ) FROM t", + expected_references: Some("enum:order_status.PLACED@12..40, const:sales_cap@42..63"), + }, + DeclarationReferenceScanTestCase { + description: "references inside quotes, comments and dollar literals are text", + sql: "SELECT '@const(\"a\")', `@const(\"b\")`, $tag$@const(\"c\")$tag$ -- @const(\"d\")\n/* @enum(\"e\").X */ @const(\"f\")", + expected_references: Some("const:f@93..104"), + }, + DeclarationReferenceScanTestCase { + description: "a word after the keyword is not a reference and a $ inside a word is code", + sql: "SELECT @constant, price$1, @const(\"cap\")", + expected_references: Some("const:cap@27..40"), + }, + DeclarationReferenceScanTestCase { + description: "doubled single quotes stay inside the string", + sql: "SELECT 'it''s @const(\"a\")', @const(\"b\")", + expected_references: Some("const:b@28..39"), + }, + DeclarationReferenceScanTestCase { + description: "doubled backticks close and reopen quoted text, leaving it unclosed", + sql: "SELECT 'it''s @const(\"a\")', `x``@const(\"b\")", + expected_references: None, + }, + DeclarationReferenceScanTestCase { + description: "SQL without references is not scanned for unclosed quotes", + sql: "SELECT 'unterminated", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "an unclosed quote before a reference defers to Python", + sql: "SELECT 'unterminated @const(\"a\")", + expected_references: None, + }, + DeclarationReferenceScanTestCase { + description: "a malformed reference defers to Python", + sql: "SELECT @enum(\"order_status\")", + expected_references: None, + }, + DeclarationReferenceScanTestCase { + description: "a non-ASCII character after the keyword defers its word boundary to Python", + sql: "SELECT @consté", + expected_references: None, + }, + DeclarationReferenceScanTestCase { + description: "an unclosed block comment before a reference defers to Python", + sql: "SELECT /* open @const(\"a\")", + expected_references: None, + }, + ]; + + for test_case in test_cases { + let scanned = scanned_references(test_case.sql); + + assert_eq!( + scanned.as_deref(), + test_case.expected_references, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 269da1d49a..34c31f6162 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -15,3 +15,10 @@ pub(crate) struct ModelHeaderMatchTestCase { pub(crate) contents: &'static str, pub(crate) expected_offsets: Option<(usize, usize, usize)>, } + +pub(crate) struct DeclarationReferenceScanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + /// References spelled `kind:name[.member]@start..end, ...` in code points. + pub(crate) expected_references: Option<&'static str>, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 61e786b7aa..5b64d00c95 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,6 @@ """Private SQLBuild native engine bindings.""" +from collections.abc import Sequence from typing import Any, TypedDict BUILD_IDENTITY: str @@ -86,6 +87,16 @@ def parse_model_contents(request: dict[str, object], contents: str) -> tuple[obj def parse_sql_test_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... def parse_scenario_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... +class NativeDeclarationContexts: + def __init__( + self, + positions: list[tuple[object, ...]], + table: tuple[list[int], dict[str, list[int]], dict[str, list[int]]], + classes: dict[str, object], + lookup: dict[str, object], + ) -> None: ... + def context(self, matches: Sequence[Any], consumer: object) -> Any: ... + class NativeScopeIndex: def records( self, @@ -164,6 +175,9 @@ class SqlReferenceScanner: def scope_expected_model_names( sqls: list[str], syntax: dict[str, object] ) -> list[list[str] | None]: ... +def scan_sql_declaration_references( + sqls: list[str], +) -> list[list[tuple[int, str, str | None, int, int]] | None]: ... def substitute_static_project_vars( sqls: list[str], variables: list[tuple[str, str]] ) -> list[tuple[int, str | None]]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index f16faa7ce7..c63e030c05 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -80,6 +80,7 @@ from sqlbuild.compiler.compile._helpers.render.declarations import ( build_model_declaration_indexes, expand_declaration_references_result, + expand_scanned_declaration_references, resolve_declaration_context, resolve_enum_contract_columns, ) @@ -148,6 +149,10 @@ parse_native_header_metadata, ) from sqlbuild.compiler.model_config.models import NativeHeaderMetadata +from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( + scan_native_declaration_references, +) +from sqlbuild.compiler.model_loop.types import NativeDeclarationReference from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -278,6 +283,7 @@ class _ModelInputLoop: reusable_config_cache: _ReusableModelConfigCache declaration_cache: _VisibleModelDeclarationCache native_header_metadata: dict[Path, NativeHeaderMetadata] + native_declaration_references: dict[Path, tuple[NativeDeclarationReference, ...] | None] @dataclass(frozen=True) @@ -519,6 +525,15 @@ def _build_model_inputs( ) ) native_model_config: bool = native_stage_enabled(NativeStage.MODEL_CONFIG) + prepared_files: tuple[DiscoveredSqlModelFile, ...] = ( + tuple( + model_file + for model_file in render_files + if prepared_var_substituted_sqls[model_file.file_path] is not None + ) + if native_stage_enabled(NativeStage.MODEL_LOOP) + else () + ) loop: _ModelInputLoop = _ModelInputLoop( discovered_inputs=discovered_inputs, context=context, @@ -536,6 +551,18 @@ def _build_model_inputs( native_header_metadata=( parse_native_header_metadata(model_files=render_files) if native_model_config else {} ), + native_declaration_references=dict( + zip( + (model_file.file_path for model_file in prepared_files), + scan_native_declaration_references( + sqls=tuple( + cast(str, prepared_var_substituted_sqls[model_file.file_path]) + for model_file in prepared_files + ) + ), + strict=True, + ) + ), ) model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile @@ -677,13 +704,26 @@ def _build_model_input( inaccessible_macros=declarations.inaccessible_macros, consumer=model_identity, ) - declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, + declaration_expansion: DeclarationExpansionResult | None = ( + expand_scanned_declaration_references( + sql=var_substituted_sql, + references=loop.native_declaration_references.get(model_file.file_path), + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) + if prepared_var_substituted_sql is not None + else None ) + if declaration_expansion is None: + declaration_expansion = expand_declaration_references_result( + sql=var_substituted_sql, + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) declaration_expanded_sql: str = declaration_expansion.sql macro_expansion: MacroExpansionResult = expand_sql_macros_result( sql=declaration_expanded_sql, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index f075087d2c..dd21579021 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -26,7 +26,7 @@ MODEL_AUDIT_OVERRIDE_KEYS, MODEL_HEADER_METADATA_KEYS, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeModelConfigMismatchError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CachedModelHeaderColumns, CompileModelConfig, @@ -42,6 +42,7 @@ DiscoveredSqlModelFile, ModelSchemaDeclaration, ) +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( native_config_contains_macro_call, ) @@ -359,7 +360,7 @@ def build_model_header_schema_entry( ) ) if native is not None and native.invalid: - raise NativeModelConfigMismatchError( + raise NativeStageMismatchError( f"{file_path}: native model config rejected MODEL columns or audits that the Python " "model config accepts; run with SQLBUILD_COMPILER_ENGINE=python" ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index 5d9f7a896c..90a6663ab3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -8,8 +8,8 @@ from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, PRESERVE_TARGET_VALUE -from sqlbuild.compiler.compile.exceptions import NativeModelConfigMismatchError from sqlbuild.compiler.compile.types import CompileContextKey +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ, INVALID_OUTCOME from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( expand_native_config_templates, @@ -278,7 +278,7 @@ def expand_config_templates( preserve_unknown_context=preserve_unknown_context, ) if outcome == INVALID_OUTCOME: - raise NativeModelConfigMismatchError( + raise NativeStageMismatchError( f"Native model config rejected {context_label} templates that the Python model " "config expands; run with SQLBUILD_COMPILER_ENGINE=python" ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index 8f8c6eca41..444660df65 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -31,7 +31,15 @@ EnumMember, ModelSchemaDeclaration, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.model_loop.constants import ENUM_REFERENCE_KIND_CODE +from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( + build_native_declaration_contexts, +) +from sqlbuild.compiler.model_loop.types import NativeDeclarationReference from sqlbuild.compiler.planner.types import ContractPolicy +from sqlbuild.compiler.scopes.constants import QUALIFIED_IDENTITY_SEPARATOR from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( @@ -152,9 +160,10 @@ def build_declaration_scope_resolver( if loaded_macros is not None: for macro in loaded_macros.values(): declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro + scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup return DeclarationScopeResolver( project_dir=discovered_inputs.project_dir, - lookup=build_scope_lookup(index=scope_index) if lookup is None else lookup, + lookup=scope_lookup, projection=DeclarationRuntimeProjection(declarations=MappingProxyType(declarations)), resource_specific=frozenset( declaration.identity.owner @@ -162,6 +171,11 @@ def build_declaration_scope_resolver( if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None ) | frozenset(grant.resource for grant in scope_index.grants), + native_contexts=( + build_native_declaration_contexts(lookup=scope_lookup, declarations=declarations) + if native_stage_enabled(NativeStage.MODEL_LOOP) + else None + ), ) @@ -201,9 +215,43 @@ def resolve_declaration_context( ) if cached_path_context is not None: return cached_path_context - resolution: DeclarationVisibility = resolve_scope_declaration_visibility( - lookup=resolver.lookup, target=resource or target_path + native_context: DeclarationResolutionContext | None = ( + resolver.native_contexts.context(resources, resource or resources[0].identity) + if resolver.native_contexts is not None + and resources + and not _scope_query_parses_identity(resource=resource, target_path=target_path) + else None + ) + context: DeclarationResolutionContext = ( + native_context + if native_context is not None + else _project_declaration_context( + resolver=resolver, + resolution=resolve_scope_declaration_visibility( + lookup=resolver.lookup, target=resource or target_path + ), + target_path=target_path, + resource=resource, + ) ) + if cache_key is not None: + resolver.cache_context(key=cache_key, context=context) + return context + + +def _scope_query_parses_identity(*, resource: ResourceIdentity | None, target_path: Path) -> bool: + """Whether the scope query reads the path target as a qualified identity, as Python does.""" + + return resource is None and QUALIFIED_IDENTITY_SEPARATOR in str(target_path) + + +def _project_declaration_context( + *, + resolver: DeclarationScopeResolver, + resolution: DeclarationVisibility, + target_path: Path, + resource: ResourceIdentity | None, +) -> DeclarationResolutionContext: enums: dict[str, EnumDeclaration] = {} constants: dict[str, ConstantDeclaration] = {} inaccessible_enums: dict[str, DeclarationRecord] = {} @@ -259,7 +307,7 @@ def resolve_declaration_context( constant_visibility[record.identity.name] = records elif record.identity.kind is DeclarationKind.MACRO: macro_visibility[record.identity.name] = records - context: DeclarationResolutionContext = DeclarationResolutionContext( + return DeclarationResolutionContext( enums=enums, constants=constants, inaccessible_enums=inaccessible_enums, @@ -275,9 +323,6 @@ def resolve_declaration_context( or (resolution.target.matches[0].identity if resolution.target.matches else None) ), ) - if cache_key is not None: - resolver.cache_context(key=cache_key, context=context) - return context def _declaration_path_visibility( @@ -615,19 +660,11 @@ def expand_declaration_references_result( ) visibility = declarations.constant_visibility.get(constant_match.group("name"), ()) member = None - if declarations.consumer is not None: - for visible in usage_visibility( - visibility=visibility, - consumer=declarations.consumer, - ): - usages.append( - UsageRecord( - consumer=declarations.consumer, - declaration=visible.declaration, - through=visible.through, - enum_member=member, - ) - ) + usages.extend( + declaration_reference_usages( + declarations=declarations, visibility=visibility, enum_member=member + ) + ) rendered_parts.append(replacement) spans.append( ExpansionSpan( @@ -646,6 +683,127 @@ def expand_declaration_references_result( ) +def declaration_reference_usages( + *, + declarations: DeclarationResolutionContext, + visibility: tuple[VisibilityRecord, ...], + enum_member: str | None, +) -> tuple[UsageRecord, ...]: + """Usage records one resolved `@enum`/`@const` reference adds for the context's consumer.""" + + consumer: ResourceIdentity | DeclarationIdentity | None = declarations.consumer + if consumer is None: + return () + return tuple( + UsageRecord( + consumer=consumer, + declaration=visible.declaration, + through=visible.through, + enum_member=enum_member, + ) + for visible in usage_visibility(visibility=visibility, consumer=consumer) + ) + + +def expand_scanned_declaration_references( # noqa: PLR0913 + *, + sql: str, + references: tuple[NativeDeclarationReference, ...] | None, + file_path: Path, + declarations: DeclarationResolutionContext, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, +) -> DeclarationExpansionResult | None: + """Splice natively scanned references, or return None when Python must expand and report.""" + + if references is None: + return None + if not references: + return DeclarationExpansionResult(sql=sql, spans=(), usages=()) + parts: list[str] = [] + spans: list[ExpansionSpan] = [] + usages: list[UsageRecord] = [] + cursor: int = 0 + output_length: int = 0 + for kind, name, member, start, end in references: + resolved: tuple[str, tuple[VisibilityRecord, ...]] | None = ( + _scanned_enum_member_text(declarations=declarations, name=name, member=member) + if kind == ENUM_REFERENCE_KIND_CODE + else _scanned_constant_text( + declarations=declarations, + name=name, + file_path=file_path, + value_renderer=value_renderer, + collection_rendering=collection_rendering, + ) + ) + if resolved is None: + return None + replacement, visibility = resolved + parts.append(sql[cursor:start]) + output_length += start - cursor + usages.extend( + declaration_reference_usages( + declarations=declarations, visibility=visibility, enum_member=member + ) + ) + parts.append(replacement) + spans.append( + ExpansionSpan( + source_start=start, + source_end=end, + output_start=output_length, + output_end=output_length + len(replacement), + ) + ) + output_length += len(replacement) + cursor = end + parts.append(sql[cursor:]) + return DeclarationExpansionResult( + sql="".join(parts), spans=tuple(spans), usages=tuple(dict.fromkeys(usages)) + ) + + +def _scanned_enum_member_text( + *, declarations: DeclarationResolutionContext, name: str, member: str | None +) -> tuple[str, tuple[VisibilityRecord, ...]] | None: + declaration: EnumDeclaration | None = declarations.enums.get(name) + if declaration is None: + return None + matched: EnumMember | None = next( + (candidate for candidate in declaration.members if candidate.name == member), None + ) + if matched is None: + return None + return ( + render_enum_member_value(value=matched.value), + declarations.enum_visibility.get(name, ()), + ) + + +def _scanned_constant_text( + *, + declarations: DeclarationResolutionContext, + name: str, + file_path: Path, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, +) -> tuple[str, tuple[VisibilityRecord, ...]] | None: + declaration: ConstantDeclaration | None = declarations.constants.get(name) + if declaration is None: + return None + try: + rendered: str = render_constant_declaration( + declaration=declaration, + value_renderer=value_renderer, + collection_rendering=collection_rendering, + file_path=file_path, + ) + except CompileInputError: + return None + return rendered, declarations.constant_visibility.get(name, ()) + + def usage_visibility( *, visibility: tuple[VisibilityRecord, ...], diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index f681db9f46..5d849a9b26 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -29,7 +29,3 @@ class CompactAnalysisInputError(CompileInputError): class AnalysisCacheEntryError(ValueError): """Raised when a persisted model analysis cache entry is invalid.""" - - -class NativeModelConfigMismatchError(RuntimeError): - """Raised when native model config rejects a value the Python model config accepts.""" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index db17965a78..c924443f2a 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any, overload +import sqlbuild._native as _native from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME @@ -316,6 +317,9 @@ class DeclarationScopeResolver: contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] = field( default_factory=dict, compare=False, repr=False ) + native_contexts: _native.NativeDeclarationContexts | None = field( + default=None, compare=False, repr=False + ) def cache_context(self, *, key: tuple[str, str], context: DeclarationResolutionContext) -> None: """Retain a process-local context for equivalent lexical consumers.""" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index dd19d64fca..83909b5197 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -21,6 +21,7 @@ NativeStage.MODEL_CONFIG: NativeStageTier.PREVIEW, NativeStage.REFERENCE_EXTRACTION: NativeStageTier.PREVIEW, NativeStage.DECLARATION_FILES: NativeStageTier.PREVIEW, + NativeStage.MODEL_LOOP: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), @@ -53,6 +54,7 @@ {"shared_analyses"} ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), + "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( {"sqlbuild.compiler.compile.models:CompactLineageFacts"} diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 3881a1871d..82a422b4d2 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -19,6 +19,7 @@ class NativeStage(StrEnum): MODEL_CONFIG = "model_config" REFERENCE_EXTRACTION = "reference_extraction" DECLARATION_FILES = "declaration_files" + MODEL_LOOP = "model_loop" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/model_loop/__init__.py b/src/sqlbuild/compiler/model_loop/__init__.py new file mode 100644 index 0000000000..46a227a6be --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/__init__.py @@ -0,0 +1 @@ +"""The native model compile loop for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/model_loop/constants.py b/src/sqlbuild/compiler/model_loop/constants.py new file mode 100644 index 0000000000..f88fcb825f --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/constants.py @@ -0,0 +1,11 @@ +"""Codes the native model loop shares with its Rust bindings.""" + +from sqlbuild.compiler.scopes.types import DeclarationKind + +DECLARATION_KIND_CODES: dict[DeclarationKind, int] = { + DeclarationKind.ENUM: 0, + DeclarationKind.CONSTANT: 1, + DeclarationKind.MACRO: 2, +} +NO_DECLARATION_KIND: int = 255 +ENUM_REFERENCE_KIND_CODE: int = DECLARATION_KIND_CODES[DeclarationKind.ENUM] diff --git a/src/sqlbuild/compiler/model_loop/main/__init__.py b/src/sqlbuild/compiler/model_loop/main/__init__.py new file mode 100644 index 0000000000..1956248370 --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/main/__init__.py @@ -0,0 +1 @@ +"""Native model loop entry points.""" diff --git a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py new file mode 100644 index 0000000000..04592edf51 --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py @@ -0,0 +1,77 @@ +"""Hand one scope lookup to the native declaration context classifier.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + LoadedMacro, +) +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.compiler.model_loop.constants import DECLARATION_KIND_CODES, NO_DECLARATION_KIND +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationVisibilityIndex, + ScopeLookup, + VisibilityRecord, +) +from sqlbuild.compiler.scopes.types import DeclarationKind, VisibilityReason + + +def build_native_declaration_contexts( + *, + lookup: ScopeLookup, + declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro], +) -> _native.NativeDeclarationContexts: + """Hand the lookup's declaration positions and runtime values to the native classifier.""" + + index: DeclarationVisibilityIndex = lookup.visibility_index + identity_keys: dict[DeclarationIdentity, int] = {} + positions: list[tuple[object, ...]] = [] + for identity in index.identities: + key: int = identity_keys.setdefault(identity, len(identity_keys)) + value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None = declarations.get( + identity + ) + positions.append( + ( + identity, + lookup.declarations[identity][0], + identity.name, + DECLARATION_KIND_CODES.get(identity.kind, NO_DECLARATION_KIND), + value, + _value_kind(value), + key, + ) + ) + return _native.NativeDeclarationContexts( + positions, + ( + list(index.global_positions), + {owner: list(items) for owner, items in index.local_positions.items()}, + {owner: list(items) for owner, items in index.inherited_positions.items()}, + ), + { + "visibility_record": VisibilityRecord, + "resolution_context": DeclarationResolutionContext, + "allocate": object.__new__, + "reasons": {reason.value: reason for reason in VisibilityReason}, + }, + { + "identity_keys": identity_keys, + "private_positions": index.private_positions, + "grants_by_resource": lookup.grants_by_resource, + }, + ) + + +def _value_kind(value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None) -> int: + if isinstance(value, EnumDeclaration): + return DECLARATION_KIND_CODES[DeclarationKind.ENUM] + if isinstance(value, ConstantDeclaration): + return DECLARATION_KIND_CODES[DeclarationKind.CONSTANT] + if isinstance(value, LoadedMacro): + return DECLARATION_KIND_CODES[DeclarationKind.MACRO] + return NO_DECLARATION_KIND diff --git a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py new file mode 100644 index 0000000000..88b94d4683 --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py @@ -0,0 +1,17 @@ +"""Scan `@enum`/`@const` references of many SQL strings in one native call.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.model_loop.types import NativeDeclarationReference + + +def scan_native_declaration_references( + *, sqls: tuple[str, ...] +) -> tuple[tuple[NativeDeclarationReference, ...] | None, ...]: + """Return each string's `(kind code, name, member, start, end)` references, or None.""" + + return tuple( + None if references is None else tuple(references) + for references in _native.scan_sql_declaration_references(list(sqls)) + ) diff --git a/src/sqlbuild/compiler/model_loop/types.py b/src/sqlbuild/compiler/model_loop/types.py new file mode 100644 index 0000000000..eb88ba5462 --- /dev/null +++ b/src/sqlbuild/compiler/model_loop/types.py @@ -0,0 +1,3 @@ +"""Row shapes the native model loop exchanges with its Rust bindings.""" + +type NativeDeclarationReference = tuple[int, str, str | None, int, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 98d12b46fa..7539197dfc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -873,3 +873,14 @@ class RejectedReferenceCallTestCase: staging_from: str mart_from: str expected_diagnostics: tuple[tuple[str, str, str, int, int, str], ...] + + +@dataclass(frozen=True) +class NativeModelLoopParityTestCase: + """Models whose declarations two engines must resolve, report and write the same way.""" + + description: str + project_files: dict[str, str] + engines: tuple[str, str] + expected_exit_codes: tuple[int, int] + expected_report_text: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_loop.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_loop.py new file mode 100644 index 0000000000..5efb55def9 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_model_loop.py @@ -0,0 +1,106 @@ +"""The preview engine's native model loop compiles exactly as the Python engine does.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + NativeModelLoopParityTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + prepare_compile_reuse_project, + report_without_engine, + run_reuse_compile, + stderr_without_durations, + write_project_file, +) + +_DECLARATIONS: dict[str, str] = { + "enums/order_status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "constants/limits.sql": "CONSTANT (name order_cap, value 3);\n", +} +_MARTS_DECLARATIONS: dict[str, str] = { + "models/marts/_sqlbuild/_enums/order_status.sql": ( + "ENUM (name order_status, members [PLACED, SHIPPED]);\n" + ), + "models/marts/_sqlbuild/_constants/limits.sql": "CONSTANT (name order_cap, value 3);\n", +} +_DECLARATION_MODEL: str = """MODEL (materialized view, description "Order flags."); + +SELECT + order_id, + @enum("order_status").PLACED AS placed_status, + @const("order_cap") AS order_cap, + '@const("order_cap")' AS literal_text +FROM __ref("stg_orders") +""" + + +@pytest.mark.parametrize( + "test_case", + [ + NativeModelLoopParityTestCase( + description="folder_private_declaration_references", + project_files={ + **_MARTS_DECLARATIONS, + "models/marts/order_flags.sql": _DECLARATION_MODEL, + }, + engines=("python", "native-preview"), + expected_exit_codes=(0, 0), + expected_report_text="order_flags", + ), + NativeModelLoopParityTestCase( + description="python_identity_error_for_a_folder_containing_a_colon", + project_files={**_DECLARATIONS, "models/a:b/order_flags.sql": _DECLARATION_MODEL}, + engines=("python", "native-preview"), + expected_exit_codes=(1, 1), + expected_report_text="Unknown qualified identity kind", + ), + NativeModelLoopParityTestCase( + description="rejected_reference_call_beside_declaration_references", + project_files={ + **_MARTS_DECLARATIONS, + "models/marts/order_flags.sql": _DECLARATION_MODEL.replace( + '__ref("stg_orders")', "__ref('stg_orders')" + ), + }, + engines=("python", "native-preview"), + expected_exit_codes=(1, 1), + expected_report_text="P012", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_declarations_when_compiling_with_preview_then_output_matches_python( + test_case: NativeModelLoopParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + for relative_path, contents in test_case.project_files.items(): + write_project_file(prepared_project, relative_path, contents) + runs: list[CompileReuseRun] = [ + run_reuse_compile( + project_dir=copy_compile_project( + source=prepared_project, destination=tmp_path / engine + ), + env={}, + global_args=("--compiler-engine", engine), + ) + for engine in test_case.engines + ] + + assert tuple(run.returncode for run in runs) == test_case.expected_exit_codes + assert report_without_engine(runs[1]) == report_without_engine(runs[0]) + assert stderr_without_durations(stderr=runs[1].stderr) == stderr_without_durations( + stderr=runs[0].stderr + ) + assert runs[1].compiled == runs[0].compiled + assert test_case.expected_report_text in runs[0].report + runs[0].stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 1e6153f2e4..ab41cad958 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -21,13 +21,14 @@ expand_template_data, ) from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, MACRO_CALL_PATTERN -from sqlbuild.compiler.compile.exceptions import CompileInputError, NativeModelConfigMismatchError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.model_config.constants import INVALID_OUTCOME, UNSUPPORTED_OUTCOME from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( native_config_contains_macro_call, @@ -227,7 +228,7 @@ def native_rejection_error( _ = model_config_engine_outcome( project_dir=project_dir, engine="native-preview", monkeypatch=monkeypatch ) - except NativeModelConfigMismatchError as error: + except NativeStageMismatchError as error: return str(error) return "no error" @@ -332,7 +333,7 @@ def _native_expansion(value: object, flags: TemplateFlags) -> object: ) except CompileInputError: return _RAISES - except NativeModelConfigMismatchError as error: + except NativeStageMismatchError as error: return str(error) return (_expansion_shape(result), reads.environment_names, reads.read_run_id) diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/__init__.py b/tests/integration/src/sqlbuild/compiler/model_loop/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py new file mode 100644 index 0000000000..65a567c33b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DeclarationReferenceParityTestCase: + """Seeded SQL with `@enum`/`@const` references expanded natively and in Python.""" + + description: str + seed: int + count: int + expected_minimum_native_references: int + expected_minimum_deferred: int + expected_minimum_python_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py b/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py new file mode 100644 index 0000000000..bcfb384ed1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py @@ -0,0 +1,190 @@ +"""Generated SQL and declaration contexts for the native model loop parity tests.""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from pathlib import Path + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.render.declarations import ( + expand_declaration_references_result, + expand_scanned_declaration_references, +) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ( + DeclarationExpansionResult, + DeclarationResolutionContext, +) +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration, EnumMember +from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( + scan_native_declaration_references, +) +from sqlbuild.compiler.model_loop.types import NativeDeclarationReference +from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity, VisibilityRecord +from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, VisibilityReason +from sqlbuild.sql_values.main.normalize import normalize_sql_value +from sqlbuild.sql_values.types import CollectionRendering + +_CONSUMER: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, "orders") +_FILE_PATH: Path = Path("models/orders.sql") +_ENUMS: dict[str, EnumDeclaration] = { + "order_status": EnumDeclaration( + "order_status", + (EnumMember("PLACED", "placed"), EnumMember("SHIPPED", 2), EnumMember("NOTED", "it's")), + "VARCHAR", + Path("enums/status.sql"), + ), +} +_CONSTANTS: dict[str, ConstantDeclaration] = { + "sales_cap": ConstantDeclaration( + "sales_cap", normalize_sql_value(raw_value=3, context="test"), Path("constants/cap.sql") + ), + "label": ConstantDeclaration( + "label", normalize_sql_value(raw_value="o'k \u00e9", context="test"), Path("c/label.sql") + ), + "regions": ConstantDeclaration( + "regions", + normalize_sql_value(raw_value=["north", "south"], context="test"), + Path("constants/regions.sql"), + ), +} +DECLARATIONS: DeclarationResolutionContext = DeclarationResolutionContext( + enums=_ENUMS, + constants=_CONSTANTS, + enum_visibility={ + "order_status": ( + VisibilityRecord( + _CONSUMER, + DeclarationIdentity(DeclarationKind.ENUM, "order_status"), + VisibilityReason.GLOBAL, + ), + ) + }, + constant_visibility={ + name: ( + VisibilityRecord( + _CONSUMER, + DeclarationIdentity(DeclarationKind.CONSTANT, name), + VisibilityReason.GLOBAL, + ), + ) + for name in _CONSTANTS + }, + consumer=_CONSUMER, +) +_WHITESPACE: tuple[str, ...] = ("", " ", "\t", "\n ", "\x1c", "\u00a0") +_REFERENCES: tuple[str, ...] = ( + '@enum({s}"order_status"{s}){s}.{s}PLACED', + "@enum('order_status').SHIPPED", + '@enum("order_status").NOTED', + '@const("sales_cap")', + "@const({s}'label'{s})", + '@const("regions")', +) +_INVALID_REFERENCES: tuple[str, ...] = ( + '@enum("order_status").MISSING', + '@const("unknown_cap")', + '@enum("order_status")', + "@const(sales_cap)", +) +_TEXT: tuple[str, ...] = ( + "SELECT ", + " FROM orders", + ", ", + "caf\u00e9 ", + "\U0001f600 ", + "@constant ", + "@enumerate ", + "price$1 ", + '$$ @const("sales_cap") $$', + '$tag$ @enum("x").Y $tag$', + "'@const(\"sales_cap\")'", + "'it''s'", + '"quoted @const(\\"label\\")"', + "`tick`", + '-- @const("sales_cap")\n', + '/* @enum("order_status").PLACED */', + "@\u00e9 ", +) +_RARE: tuple[str, ...] = ( + "'unterminated ", + "/* open ", + "`open ", + "$q$ open ", + "@const\u00e9 ", + *_INVALID_REFERENCES, +) + + +@dataclass(frozen=True) +class ReferenceParity: + """Python's expansion (or error text) and the native one (or None) of one SQL string.""" + + sql: str + python: DeclarationExpansionResult | str + native: DeclarationExpansionResult | None + native_references: int + + +def generated_reference_sql(*, rng: random.Random) -> str: + """Return SQL mixing references, quoted text, comments and Unicode in random order.""" + + parts: list[str] = [ + rng.choices((_reference_part, _text_part), weights=(4, 6))[0](rng) + for _ in range(rng.randint(1, 12)) + ] + parts.extend(rng.choices(_RARE, k=int(rng.random() < 0.15))) + rng.shuffle(parts) + return "".join(parts) + + +def reference_parities(*, sqls: list[str]) -> list[ReferenceParity]: + """Expand every string in Python and through the native scan with Python rendering.""" + + scanned: tuple[tuple[NativeDeclarationReference, ...] | None, ...] = ( + scan_native_declaration_references(sqls=tuple(sqls)) + ) + return [ + ReferenceParity( + sql=sql, + python=_python_outcome(sql=sql), + native=expand_scanned_declaration_references( + sql=sql, + references=references, + file_path=_FILE_PATH, + declarations=DECLARATIONS, + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + ), + native_references=len(references or ()), + ) + for sql, references in zip(sqls, scanned, strict=True) + ] + + +def is_native(parity: ReferenceParity) -> bool: + """Whether the native path expanded the string instead of deferring to Python.""" + + return parity.native is not None + + +def _reference_part(rng: random.Random) -> str: + return rng.choice(_REFERENCES).format(s=rng.choice(_WHITESPACE)) + + +def _text_part(rng: random.Random) -> str: + return rng.choice(_TEXT) + + +def _python_outcome(*, sql: str) -> DeclarationExpansionResult | str: + try: + return expand_declaration_references_result( + sql=sql, + file_path=_FILE_PATH, + declarations=DECLARATIONS, + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + ) + except CompileInputError as error: + return str(error) diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py b/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py new file mode 100644 index 0000000000..bcdb646bfb --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py @@ -0,0 +1,60 @@ +"""Natively scanned `@enum`/`@const` references expand exactly as Python expands them.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.model_loop._test_types import ( + DeclarationReferenceParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.model_loop.helpers import ( + ReferenceParity, + generated_reference_sql, + is_native, + reference_parities, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationReferenceParityTestCase( + description="seeded references among quotes, comments, dollar literals and Unicode", + seed=20261007, + count=3000, + expected_minimum_native_references=2000, + expected_minimum_deferred=600, + expected_minimum_python_errors=400, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_sql_when_expanding_references_natively_then_python_output_matches( + test_case: DeclarationReferenceParityTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_reference_sql(rng=rng) for _ in range(test_case.count)] + + parities: list[ReferenceParity] = reference_parities(sqls=sqls) + + expanded: list[ReferenceParity] = list(filter(is_native, parities)) + + assert ( + mismatches( + inputs=[parity.sql for parity in expanded], + expected=[parity.python for parity in expanded], + actual=[parity.native for parity in expanded], + ), + sum(parity.native_references for parity in expanded) + >= test_case.expected_minimum_native_references, + len(parities) - len(expanded) >= test_case.expected_minimum_deferred, + sum(isinstance(parity.python, str) for parity in parities) + >= test_case.expected_minimum_python_errors, + ) == ([], True, True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index 846f45dbfe..39506a0b8f 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -55,3 +55,16 @@ class ExpectedNameScanTestCase: expected_minimum_scanned: int expected_minimum_deferred: int expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class DeclarationContextParityTestCase: + """Seeded scoped projects whose consumers resolve declaration contexts under both paths.""" + + description: str + seed: int + count: int + expected_minimum_native: int + expected_minimum_granted: int + expected_minimum_private: int + expected_minimum_python_only: int diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 8a18d8df9c..d672aef9f4 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -4,8 +4,8 @@ import random from collections.abc import Callable, Mapping -from dataclasses import dataclass -from itertools import compress +from dataclasses import dataclass, fields, replace +from itertools import chain, compress from pathlib import Path import pytest @@ -13,10 +13,16 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope +from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_expected_model_names from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import ( + DeclarationResolutionContext, + DeclarationScopeBuild, + DeclarationScopeResolver, + LoadedMacro, +) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs @@ -29,7 +35,13 @@ ) from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main.load_or_build_scope_index import load_or_build_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup +from sqlbuild.compiler.scopes.models import ( + ResourceIdentity, + ScopeIndex, + ScopeLookup, + VisibilityRecord, +) +from sqlbuild.compiler.scopes.types import VisibilityReason from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.helpers import mismatches @@ -483,3 +495,121 @@ def _python_expected_names(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[str, ) except CompileInputError as error: return str(error) + + +@dataclass(frozen=True) +class ContextParity: + """Native and Python declaration contexts for every consumer of some projects.""" + + targets: tuple[str, ...] = () + native: tuple[object, ...] = () + python: tuple[object, ...] = () + native_contexts: int = 0 + granted_contexts: int = 0 + private_contexts: int = 0 + python_only_paths: int = 0 + + +_CONTEXT_DICT_FIELDS: tuple[str, ...] = tuple( + item.name for item in fields(DeclarationResolutionContext) if item.name != "consumer" +) + + +def declaration_context_parity( + *, project_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> ContextParity: + """Resolve every resource, resource path and declaration file natively and in Python.""" + + try: + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) + scope: DeclarationScopeBuild = build_declaration_scope( + discovered_inputs=discovered, + loaded_macros=macros, + sql_lexical_syntax=DuckDbAdapter().sql_lexical_syntax, + ) + except (DiscoveryError, CompileInputError): + return ContextParity() + resolver: DeclarationScopeResolver = scope.resolver + targets: list[tuple[Path, ResourceIdentity | None]] = [ + *( + (Path(records[0].path), identity) + for identity, records in resolver.lookup.resources.items() + ), + *((Path(path), None) for path in resolver.lookup.resources_by_path), + *((Path(record.path), None) for record in scope.index.declarations), + ] + native: list[DeclarationResolutionContext] = [ + resolve_declaration_context( + resolver=replace(resolver, contexts_by_directory={}), + file_path=file_path, + resource=resource, + ) + for file_path, resource in targets + ] + python: list[DeclarationResolutionContext] = [ + resolve_declaration_context( + resolver=replace(resolver, contexts_by_directory={}, native_contexts=None), + file_path=file_path, + resource=resource, + ) + for file_path, resource in targets + ] + indexed: list[bool] = [ + bool(resource) or file_path.as_posix() in resolver.lookup.resources_by_path + for file_path, resource in targets + ] + records: list[list[VisibilityRecord]] = list(map(_visibility_records, native)) + return ContextParity( + targets=tuple( + f"{project_dir.name}:{file_path}:{resource}" for file_path, resource in targets + ), + native=tuple(map(_context_shape, native)), + python=tuple(map(_context_shape, python)), + native_contexts=sum(indexed), + granted_contexts=sum(map(_has_grant, records)), + private_contexts=sum(map(_has_private_value, records)), + python_only_paths=len(indexed) - sum(indexed), + ) + + +def combined_context_parity(parities: list[ContextParity]) -> ContextParity: + """Concatenate the compared contexts and add up the coverage counts of several projects.""" + + return ContextParity( + targets=tuple(chain.from_iterable(parity.targets for parity in parities)), + native=tuple(chain.from_iterable(parity.native for parity in parities)), + python=tuple(chain.from_iterable(parity.python for parity in parities)), + native_contexts=sum(parity.native_contexts for parity in parities), + granted_contexts=sum(parity.granted_contexts for parity in parities), + private_contexts=sum(parity.private_contexts for parity in parities), + python_only_paths=sum(parity.python_only_paths for parity in parities), + ) + + +def _has_grant(records: list[VisibilityRecord]) -> bool: + return any(record.through is not None for record in records) + + +def _has_private_value(records: list[VisibilityRecord]) -> bool: + return any(record.reason is VisibilityReason.PRIVATE_OWNER for record in records) + + +def _visibility_records(context: DeclarationResolutionContext) -> list[VisibilityRecord]: + return list( + chain.from_iterable( + chain( + context.enum_visibility.values(), + context.constant_visibility.values(), + context.macro_visibility.values(), + ) + ) + ) + + +def _context_shape(context: DeclarationResolutionContext) -> tuple[object, ...]: + return ( + *(tuple(getattr(context, name).items()) for name in _CONTEXT_DICT_FIELDS), + context.consumer, + ) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_declaration_contexts.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_declaration_contexts.py new file mode 100644 index 0000000000..84b88ea5de --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_declaration_contexts.py @@ -0,0 +1,67 @@ +"""Native declaration resolution contexts match Python's for every consumer of a project.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.scopes._test_types import ( + DeclarationContextParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( + SCOPED_PROJECT, + ContextParity, + combined_context_parity, + declaration_context_parity, + generated_scope_files, + write_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationContextParityTestCase( + description="seeded scoped projects with grants, private and folder-scoped values", + seed=20261007, + count=60, + expected_minimum_native=250, + expected_minimum_granted=40, + expected_minimum_private=50, + expected_minimum_python_only=60, + ), + ], + ids=lambda case: case.description, +) +def test_given_scoped_projects_when_resolving_contexts_natively_then_python_contexts_match( + test_case: DeclarationContextParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + project_dirs: list[Path] = [tmp_path / "scoped"] + [ + tmp_path / f"project_{index}" for index in range(test_case.count) + ] + write_project(project_dir=project_dirs[0], files=SCOPED_PROJECT) + for project_dir in project_dirs[1:]: + write_project(project_dir=project_dir, files=generated_scope_files(rng=rng)) + + parity: ContextParity = combined_context_parity( + [ + declaration_context_parity(project_dir=project_dir, monkeypatch=monkeypatch) + for project_dir in project_dirs + ] + ) + + assert ( + mismatches(inputs=[*parity.targets], expected=[*parity.python], actual=[*parity.native]), + parity.native_contexts >= test_case.expected_minimum_native, + parity.granted_contexts >= test_case.expected_minimum_granted, + parity.private_contexts >= test_case.expected_minimum_private, + parity.python_only_paths >= test_case.expected_minimum_python_only, + ) == ([], True, True, True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py index 10a3546659..8ce2f09dc2 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -87,6 +87,15 @@ class UnorderedAttributeTestCase: expected_identical: bool +@dataclass(frozen=True) +class OmittedFieldTestCase: + """Dataclass fields left out of a capture, and the capture that remains.""" + + description: str + omitted: frozenset[str] + expected_capture: object + + @dataclass(frozen=True) class FrontierCaptureTestCase: """Frontier stages compiled in order on one engine and the capture files they write.""" diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 4cc4412381..3f004a9f97 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -85,6 +85,24 @@ stage=NativeStage.DECLARATION_FILES, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_model_loop", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MODEL_LOOP, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_model_loop", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MODEL_LOOP, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_model_loop", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MODEL_LOOP, + expected_enabled=True, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index e589d0e884..cff5e5fb02 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -14,6 +14,7 @@ from sqlbuild.compiler.frontier.constants import ( COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR, + STAGE_CAPTURE_OMITTED_ATTRIBUTES, STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_UNORDERED_ATTRIBUTES, ) @@ -21,6 +22,7 @@ from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, + OmittedFieldTestCase, SharedCaptureTestCase, StageCaptureOrderTestCase, StageCaptureTestCase, @@ -250,6 +252,43 @@ def test_given_declared_memo_attribute_when_rendering_capture_then_only_it_is_so assert (first == second) is test_case.expected_identical +@pytest.mark.parametrize( + "test_case", + [ + OmittedFieldTestCase( + description="declared_process_handle_field_is_left_out", + omitted=frozenset({"status"}), + expected_capture={ + "__type__": f"{HELPERS_MODULE}:OrderLine", + "order_id": 7, + "path": {"__path__": "models/a.sql"}, + }, + ), + OmittedFieldTestCase( + description="undeclared_dataclass_keeps_every_field", + omitted=frozenset(), + expected_capture={ + "__type__": f"{HELPERS_MODULE}:OrderLine", + "order_id": 7, + "status": {"__enum__": f"{HELPERS_MODULE}:OrderStatus.PLACED"}, + "path": {"__path__": "models/a.sql"}, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_declared_omitted_field_when_rendering_dataclass_capture_then_it_is_left_out( + test_case: OmittedFieldTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + STAGE_CAPTURE_OMITTED_ATTRIBUTES, f"{HELPERS_MODULE}:OrderLine", test_case.omitted + ) + + capture: object = json.loads(render_stage_capture(order_line())) + + assert capture == test_case.expected_capture + + @pytest.mark.parametrize( "test_case", [