diff --git a/Cargo.lock b/Cargo.lock index 4e91536041..af77bdac45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1176,6 +1176,7 @@ dependencies = [ "blake3", "rayon", "sha2", + "sqlbuild-core", "tempfile", "walkdir", ] diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs index e3c29d7550..1b4fb3a6af 100644 --- a/crates/sqlbuild-analysis/src/assembly/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -2,3 +2,4 @@ pub mod analysis_session; pub mod project; +pub mod target_checks; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs new file mode 100644 index 0000000000..947493d5bb --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs @@ -0,0 +1,14 @@ +//! Error codes and values of the target namespace checks. + +/// A target, resource or loader schema that keeps the logical schema. +pub const PRESERVE_TARGET_VALUE: &str = "preserve"; +/// Execution needs an explicit database and schema for every model and seed. +pub const MISSING_TARGET_PARTS_CODE: &str = "S101"; +/// Two targets write managed loader data to one namespace. +pub const LOADER_COLLISION_CODE: &str = "S102"; +/// A managed write has no explicit physical schema. +pub const MANAGED_WRITE_SCHEMA_CODE: &str = "S103"; +/// A named target chooses no schema strategy. +pub const NAMED_TARGET_SCHEMA_CODE: &str = "S104"; +pub(crate) const DATABASE_PART: &str = "database"; +pub(crate) const SCHEMA_PART: &str = "schema"; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs new file mode 100644 index 0000000000..9f066da682 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs @@ -0,0 +1,16 @@ +//! The target namespace check failures. + +use crate::assembly::target_checks::models::{LoaderCollision, MissingParts}; + +/// One target namespace check failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetError { + /// A resource of an adapter that needs both parts resolved without some of them. + MissingParts(MissingParts), + /// Two targets on one connection write managed loader data to one namespace. + LoaderCollision(LoaderCollision), + /// A managed write, named by its label, without an explicit physical schema. + ManagedWriteSchema(String), + /// A named target without an explicit schema. + NamedTargetSchema(String), +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/explicit_write_schema.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/explicit_write_schema.rs new file mode 100644 index 0000000000..df7a977b0c --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/explicit_write_schema.rs @@ -0,0 +1,11 @@ +//! `_optional_explicit_schema`: a schema that names a physical write schema. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; + +/// The stripped schema, unless it is blank or `preserve`. +pub fn explicit_write_schema(value: Option<&str>) -> Option { + let schema: &str = python_strip(value?); + (!schema.is_empty() && schema != PRESERVE_TARGET_VALUE).then(|| schema.to_owned()) +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/first_shared_namespace.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/first_shared_namespace.rs new file mode 100644 index 0000000000..20c84db70d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/first_shared_namespace.rs @@ -0,0 +1,24 @@ +//! The namespace a loader collision reports: the first shared one by its Python `repr`. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +/// Python's `repr` of an optional string. +fn optional_repr(value: Option<&String>) -> String { + value.map_or_else(|| "None".to_owned(), |text| python_str_repr(text)) +} + +/// `sorted(namespaces, key=repr)[0]` of `(database, schema)` pairs, or `None` when empty. +pub fn first_shared_namespace( + namespaces: &[(Option, Option)], +) -> Option<(Option, Option)> { + namespaces + .iter() + .min_by_key(|(database, schema)| { + format!( + "({}, {})", + optional_repr(database.as_ref()), + optional_repr(schema.as_ref()) + ) + }) + .cloned() +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/missing_target_parts.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/missing_target_parts.rs new file mode 100644 index 0000000000..83cee6f009 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/missing_target_parts.rs @@ -0,0 +1,15 @@ +//! The parts of a relation location execution needs but the location lacks. + +use crate::assembly::target_checks::constants::{DATABASE_PART, SCHEMA_PART}; + +/// `database` and then `schema`, each when it is missing. +pub fn missing_target_parts(database: Option<&str>, schema: Option<&str>) -> Vec<&'static str> { + let mut missing: Vec<&'static str> = Vec::new(); + if database.is_none() { + missing.push(DATABASE_PART); + } + if schema.is_none() { + missing.push(SCHEMA_PART); + } + missing +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/mod.rs new file mode 100644 index 0000000000..f506d3646f --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/mod.rs @@ -0,0 +1,4 @@ +pub mod explicit_write_schema; +pub mod first_shared_namespace; +pub mod missing_target_parts; +pub mod target_error_message; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs new file mode 100644 index 0000000000..1adc077120 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -0,0 +1,46 @@ +//! The message Python raises for a target namespace check failure. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use crate::assembly::target_checks::errors::TargetError; +use crate::assembly::target_checks::main::missing_target_parts::missing_target_parts; + +/// Python's `repr` of an optional string. +fn optional_repr(value: Option<&String>) -> String { + value.map_or_else(|| "None".to_owned(), |text| python_str_repr(text)) +} + +/// The `PlannerInputError` message of `error`. +pub fn target_error_message(error: &TargetError) -> String { + match error { + TargetError::MissingParts(parts) => format!( + "{} execution requires explicit target {}. {} '{}' resolved to database={} schema={}. \ + Set them in the model config, the environment config, or the project defaults.", + parts.adapter_name, + missing_target_parts(parts.database.as_deref(), parts.schema.as_deref()).join(", "), + parts.resource_kind, + parts.resource_name, + optional_repr(parts.database.as_ref()), + optional_repr(parts.schema.as_ref()), + ), + TargetError::LoaderCollision(collision) => format!( + "Managed loader write collision: targets '{left}' and '{right}' both resolve to \ + database={} schema={}. Set distinct targets.{left}.loader_schema and \ + targets.{right}.loader_schema values.", + optional_repr(collision.database.as_ref()), + optional_repr(collision.schema.as_ref()), + left = collision.left_target, + right = collision.right_target, + ), + TargetError::ManagedWriteSchema(label) => format!( + "{label} has no explicitly resolved physical write schema. Set schema on the \ + resource or its defaults, selected target, loader_schema, or connection." + ), + TargetError::NamedTargetSchema(target) => format!( + "Named target '{target}' must explicitly set schema to a nonblank literal or \ + '{PRESERVE_TARGET_VALUE}'. Resource schemas, defaults, and connection schemas do \ + not select a named target schema strategy." + ), + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/mod.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/mod.rs new file mode 100644 index 0000000000..d11b079ee9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/mod.rs @@ -0,0 +1,7 @@ +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/models.rs new file mode 100644 index 0000000000..331ef0bbee --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/models.rs @@ -0,0 +1,20 @@ +//! The target namespace check failures, with the values their messages name. + +/// A resource whose relation location misses a database or schema. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MissingParts { + pub adapter_name: String, + pub resource_kind: String, + pub resource_name: String, + pub database: Option, + pub schema: Option, +} + +/// Two targets and the first namespace they share. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoaderCollision { + pub left_target: String, + pub right_target: String, + pub database: Option, + pub schema: Option, +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests.rs new file mode 100644 index 0000000000..40dbc81550 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_target_checks.rs"] +mod target_checks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/helpers.rs new file mode 100644 index 0000000000..8c267e2fd6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/helpers.rs @@ -0,0 +1,3 @@ +pub(crate) fn owned(value: Option<&str>) -> Option { + value.map(str::to_owned) +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs new file mode 100644 index 0000000000..e7a26a098f --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs @@ -0,0 +1,139 @@ +use crate::assembly::target_checks::errors::TargetError; +use crate::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; +use crate::assembly::target_checks::main::first_shared_namespace::first_shared_namespace; +use crate::assembly::target_checks::main::target_error_message::target_error_message; +use crate::assembly::target_checks::models::{LoaderCollision, MissingParts}; +use crate::assembly::target_checks::tests::helpers::owned; +use crate::assembly::target_checks::tests::test_types::{ + MessageTestCase, NamespaceTestCase, SchemaTestCase, +}; + +#[test] +fn given_target_error_when_formatting_then_matches_python_message() { + let test_cases = [ + MessageTestCase { + description: "a seed missing both parts", + error: TargetError::MissingParts(MissingParts { + adapter_name: "snowflake".to_owned(), + resource_kind: "seed".to_owned(), + resource_name: "products".to_owned(), + database: None, + schema: None, + }), + expected_message: "snowflake execution requires explicit target database, schema. seed 'products' resolved to database=None schema=None. Set them in the model config, the environment config, or the project defaults.", + }, + MessageTestCase { + description: "a model missing its schema, with a quoted database", + error: TargetError::MissingParts(MissingParts { + adapter_name: "bigquery".to_owned(), + resource_kind: "model".to_owned(), + resource_name: "orders".to_owned(), + database: owned(Some("it's")), + schema: None, + }), + expected_message: "bigquery execution requires explicit target schema. model 'orders' resolved to database=\"it's\" schema=None. Set them in the model config, the environment config, or the project defaults.", + }, + MessageTestCase { + description: "a loader collision", + error: TargetError::LoaderCollision(LoaderCollision { + left_target: "dev".to_owned(), + right_target: "prod".to_owned(), + database: None, + schema: owned(Some("raw")), + }), + expected_message: "Managed loader write collision: targets 'dev' and 'prod' both resolve to database=None schema='raw'. Set distinct targets.dev.loader_schema and targets.prod.loader_schema values.", + }, + MessageTestCase { + description: "a managed write without a schema", + error: TargetError::ManagedWriteSchema("Model 'orders'".to_owned()), + expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection.", + }, + MessageTestCase { + description: "a named target without a schema", + error: TargetError::NamedTargetSchema("ci".to_owned()), + expected_message: "Named target 'ci' must explicitly set schema to a nonblank literal or 'preserve'. Resource schemas, defaults, and connection schemas do not select a named target schema strategy.", + }, + ]; + + for test_case in test_cases { + let message = target_error_message(&test_case.error); + + assert_eq!( + message, test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_schema_when_reading_explicit_write_schema_then_matches_python() { + let test_cases = [ + SchemaTestCase { + description: "a padded schema is stripped", + schema: Some(" analytics\t"), + expected_schema: Some("analytics"), + }, + SchemaTestCase { + description: "preserve is not a physical schema", + schema: Some(" preserve "), + expected_schema: None, + }, + SchemaTestCase { + description: "a blank schema is none", + schema: Some("\u{3000}"), + expected_schema: None, + }, + SchemaTestCase { + description: "no schema is none", + schema: None, + expected_schema: None, + }, + ]; + + for test_case in test_cases { + let schema = explicit_write_schema(test_case.schema); + + assert_eq!( + schema.as_deref(), + test_case.expected_schema, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_shared_namespaces_when_choosing_first_then_sorts_by_python_repr() { + let test_cases = [ + NamespaceTestCase { + description: "a quoted string sorts before None", + namespaces: &[(None, Some("raw")), (Some("db"), Some("raw"))], + expected_namespace: Some((Some("db"), Some("raw"))), + }, + NamespaceTestCase { + description: "no namespace is shared", + namespaces: &[], + expected_namespace: None, + }, + ]; + + for test_case in test_cases { + let namespaces: Vec<(Option, Option)> = test_case + .namespaces + .iter() + .map(|(database, schema)| (owned(*database), owned(*schema))) + .collect(); + + let first = first_shared_namespace(&namespaces); + + assert_eq!( + first, + test_case + .expected_namespace + .map(|(database, schema)| (owned(database), owned(schema))), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_types.rs new file mode 100644 index 0000000000..a6bab15fbe --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_types.rs @@ -0,0 +1,19 @@ +use crate::assembly::target_checks::errors::TargetError; + +pub(crate) struct MessageTestCase { + pub(crate) description: &'static str, + pub(crate) error: TargetError, + pub(crate) expected_message: &'static str, +} + +pub(crate) struct SchemaTestCase { + pub(crate) description: &'static str, + pub(crate) schema: Option<&'static str>, + pub(crate) expected_schema: Option<&'static str>, +} + +pub(crate) struct NamespaceTestCase { + pub(crate) description: &'static str, + pub(crate) namespaces: &'static [(Option<&'static str>, Option<&'static str>)], + pub(crate) expected_namespace: Option<(Option<&'static str>, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/_helpers/mod.rs b/crates/sqlbuild-analysis/src/diff_selection/_helpers/mod.rs new file mode 100644 index 0000000000..ce34251a4c --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod tokens; diff --git a/crates/sqlbuild-analysis/src/diff_selection/_helpers/tokens.rs b/crates/sqlbuild-analysis/src/diff_selection/_helpers/tokens.rs new file mode 100644 index 0000000000..6c810daa23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/_helpers/tokens.rs @@ -0,0 +1,128 @@ +//! One diff selector token resolved to the resources it names. + +use std::collections::{HashSet, VecDeque}; + +use crate::diff_selection::constants::{ + EMPTY_SELECTOR_CODE, GRAPH_OPERATOR, INTERSECTION_SEPARATOR, MODEL_PATH_PREFIX, + MODEL_PATH_ROOT, PATH_PREFIX, PATH_SEPARATORS, TAG_PREFIX, UNKNOWN_NAME_CODE, + UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, +}; +use crate::diff_selection::errors::SelectorError; +use crate::diff_selection::models::DiffGraph; + +/// Resources reachable from `starts` along `edges`, without the starts unless reached. +fn reachable(starts: &HashSet, edges: &[Vec]) -> HashSet { + let mut visited: HashSet = HashSet::new(); + let mut frontier: VecDeque = starts.iter().copied().collect(); + while let Some(current) = frontier.pop_front() { + for neighbor in edges.get(current).into_iter().flatten() { + if visited.insert(*neighbor) { + frontier.push_back(*neighbor); + } + } + } + visited +} + +/// `keys` with their upstream and downstream closures when requested. +fn expanded( + keys: HashSet, + (upstream, downstream): (bool, bool), + graph: &DiffGraph, +) -> HashSet { + let mut result: HashSet = keys.clone(); + if upstream { + result.extend(reachable(&keys, &graph.upstream)); + } + if downstream { + result.extend(reachable(&keys, &graph.downstream)); + } + result +} + +/// `_resolve_selector_token`. +pub(crate) fn resolve_token( + token: &str, + graph: &DiffGraph, +) -> Result, SelectorError> { + let parts: Vec<&str> = token.split(INTERSECTION_SEPARATOR).collect(); + if parts.len() > 1 { + let mut resolved: Vec> = Vec::with_capacity(parts.len()); + for part in parts { + resolved.push(resolve_token(part, graph)?); + } + let mut parts = resolved.into_iter(); + let mut result: HashSet = parts.next().unwrap_or_default(); + for part in parts { + result.retain(|key| part.contains(key)); + } + return Ok(result); + } + let operators: (bool, bool) = ( + token.starts_with(GRAPH_OPERATOR), + token.ends_with(GRAPH_OPERATOR), + ); + let core: &str = token + .trim_start_matches(GRAPH_OPERATOR) + .trim_end_matches(GRAPH_OPERATOR); + if core.is_empty() { + return Err(SelectorError { + message: "empty selector".to_owned(), + code: EMPTY_SELECTOR_CODE, + }); + } + if let Some(tag) = core.strip_prefix(TAG_PREFIX) { + let keys: HashSet = graph + .tags + .iter() + .find(|(name, _)| name == tag) + .map(|(_, keys)| keys.iter().copied().collect()) + .unwrap_or_default(); + if keys.is_empty() { + return Err(SelectorError { + message: format!("no models found with tag '{tag}'"), + code: UNKNOWN_TAG_CODE, + }); + } + return Ok(expanded(keys, operators, graph)); + } + if core.starts_with(PATH_PREFIX) || core.contains(PATH_SEPARATORS) { + let normalized: String = core + .strip_prefix(PATH_PREFIX) + .unwrap_or(core) + .replace('\\', "/"); + let mut path: &str = normalized.trim_matches('/'); + if path == MODEL_PATH_ROOT { + path = ""; + } else if let Some(rest) = path.strip_prefix(MODEL_PATH_PREFIX) { + path = rest; + } + let nested: String = format!("{path}/"); + let keys: HashSet = graph + .paths + .iter() + .filter(|(_, folder)| folder == path || folder.starts_with(&nested)) + .map(|(key, _)| *key) + .collect(); + if keys.is_empty() { + return Err(SelectorError { + message: format!("no models found under path '{path}'"), + code: UNKNOWN_PATH_CODE, + }); + } + return Ok(expanded(keys, operators, graph)); + } + let key: Option = graph + .names + .iter() + .rev() + .find(|(name, _)| name == core) + .map(|(_, key)| *key); + let Some(key) = key else { + return Err(SelectorError { + message: format!("unknown selector name '{core}'"), + code: UNKNOWN_NAME_CODE, + }); + }; + Ok(expanded(HashSet::from([key]), operators, graph)) +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/constants.rs b/crates/sqlbuild-analysis/src/diff_selection/constants.rs new file mode 100644 index 0000000000..e85095e908 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/constants.rs @@ -0,0 +1,13 @@ +//! Diff selector syntax and the codes of its errors. + +pub(crate) const TAG_PREFIX: &str = "tag:"; +pub(crate) const PATH_PREFIX: &str = "path:"; +pub(crate) const PATH_SEPARATORS: [char; 2] = ['/', '\\']; +pub(crate) const MODEL_PATH_ROOT: &str = "models"; +pub(crate) const MODEL_PATH_PREFIX: &str = "models/"; +pub(crate) const GRAPH_OPERATOR: char = '+'; +pub(crate) const INTERSECTION_SEPARATOR: char = ','; +pub(crate) const EMPTY_SELECTOR_CODE: &str = "S001"; +pub(crate) const UNKNOWN_NAME_CODE: &str = "S007"; +pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; +pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; diff --git a/crates/sqlbuild-analysis/src/diff_selection/errors.rs b/crates/sqlbuild-analysis/src/diff_selection/errors.rs new file mode 100644 index 0000000000..5f219257c6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/errors.rs @@ -0,0 +1,8 @@ +//! A diff selector that resolves to nothing, as Python's `PlannerInputError` reports it. + +/// The message and code of a rejected selector. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SelectorError { + pub message: String, + pub code: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs new file mode 100644 index 0000000000..5f8a0fa05f --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs @@ -0,0 +1 @@ +pub mod resolve_diff_selection; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/resolve_diff_selection.rs b/crates/sqlbuild-analysis/src/diff_selection/main/resolve_diff_selection.rs new file mode 100644 index 0000000000..ab1b30c843 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/main/resolve_diff_selection.rs @@ -0,0 +1,33 @@ +//! `_resolve_model_selectors`: the resources diff `--select` names and `--exclude` does not. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::diff_selection::_helpers::tokens::resolve_token; +use crate::diff_selection::errors::SelectorError; +use crate::diff_selection::models::DiffGraph; + +/// Every resource each whitespace-separated token of `selectors` names. +fn resolved(selectors: &[String], graph: &DiffGraph) -> Result, SelectorError> { + let mut keys: HashSet = HashSet::new(); + for selector in selectors { + for token in selector + .split(python_isspace) + .filter(|token| !token.is_empty()) + { + keys.extend(resolve_token(token, graph)?); + } + } + Ok(keys) +} + +/// The selected resources without the excluded ones, or the first selector error. +pub fn resolve_diff_selection( + graph: &DiffGraph, + (select, exclude): (&[String], &[String]), +) -> Result, SelectorError> { + let selected: HashSet = resolved(select, graph)?; + let excluded: HashSet = resolved(exclude, graph)?; + Ok(selected.difference(&excluded).copied().collect()) +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/mod.rs b/crates/sqlbuild-analysis/src/diff_selection/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/diff_selection/models.rs b/crates/sqlbuild-analysis/src/diff_selection/models.rs new file mode 100644 index 0000000000..d95cbdf0dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/models.rs @@ -0,0 +1,16 @@ +//! The project graph a diff selection resolves over, with resources as indexes. + +/// Selectable names, lineage edges and selector indexes over resource indexes. +#[derive(Debug, Clone, Default)] +pub struct DiffGraph { + /// `(name, resource)` of models, then sources, then seeds; a later name wins. + pub names: Vec<(String, usize)>, + /// Each resource's upstream resources. + pub upstream: Vec>, + /// Each resource's downstream resources. + pub downstream: Vec>, + /// `(tag, resources)`. + pub tags: Vec<(String, Vec)>, + /// `(resource, model folder)`. + pub paths: Vec<(usize, String)>, +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/tests.rs b/crates/sqlbuild-analysis/src/diff_selection/tests.rs new file mode 100644 index 0000000000..6e7eda66c5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_resolve_diff_selection.rs"] +mod resolve_diff_selection; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/diff_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/diff_selection/tests/helpers.rs new file mode 100644 index 0000000000..3bffe9bb13 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/tests/helpers.rs @@ -0,0 +1,25 @@ +use crate::diff_selection::models::DiffGraph; + +/// `orders` (0) reads source `raw_orders` (1) and feeds `revenue` (2); `customers` (3) stands apart. +pub(crate) fn orders_graph() -> DiffGraph { + DiffGraph { + names: vec![ + ("orders".to_owned(), 0), + ("revenue".to_owned(), 2), + ("customers".to_owned(), 3), + ("raw_orders".to_owned(), 1), + ], + upstream: vec![vec![1], vec![], vec![0], vec![]], + downstream: vec![vec![2], vec![0], vec![], vec![]], + tags: vec![("finance".to_owned(), vec![2, 3])], + paths: vec![ + (0, "staging".to_owned()), + (2, "marts/finance".to_owned()), + (3, "marts".to_owned()), + ], + } +} + +pub(crate) fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/tests/test_resolve_diff_selection.rs b/crates/sqlbuild-analysis/src/diff_selection/tests/test_resolve_diff_selection.rs new file mode 100644 index 0000000000..d3485cfb06 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/tests/test_resolve_diff_selection.rs @@ -0,0 +1,76 @@ +use crate::diff_selection::main::resolve_diff_selection::resolve_diff_selection; +use crate::diff_selection::tests::helpers::{orders_graph, owned}; +use crate::diff_selection::tests::test_types::DiffSelectionTestCase; + +#[test] +fn given_selectors_when_resolving_diff_selection_then_matches_python_semantics() { + let test_cases = [ + DiffSelectionTestCase { + description: "graph operators expand both ways", + select: &["+orders+"], + exclude: &[], + expected_selection: Ok(&[0, 1, 2]), + }, + DiffSelectionTestCase { + description: "whitespace separates tokens and exclude removes", + select: &["orders\u{3000}customers"], + exclude: &["customers"], + expected_selection: Ok(&[0]), + }, + DiffSelectionTestCase { + description: "a comma intersects tag and path", + select: &["tag:finance,path:models\\marts\\"], + exclude: &[], + expected_selection: Ok(&[2, 3]), + }, + DiffSelectionTestCase { + description: "a nested path selects only below it", + select: &["marts/finance+"], + exclude: &[], + expected_selection: Ok(&[2]), + }, + DiffSelectionTestCase { + description: "an operator-only token is empty", + select: &["++"], + exclude: &[], + expected_selection: Err("S001"), + }, + DiffSelectionTestCase { + description: "an unknown name", + select: &["orders,payments"], + exclude: &[], + expected_selection: Err("S007"), + }, + DiffSelectionTestCase { + description: "an unknown tag", + select: &["tag:ops"], + exclude: &[], + expected_selection: Err("S008"), + }, + DiffSelectionTestCase { + description: "an unknown path", + select: &["path:models/intermediate"], + exclude: &[], + expected_selection: Err("S009"), + }, + ]; + + for test_case in test_cases { + let resolved = resolve_diff_selection( + &orders_graph(), + (&owned(test_case.select), &owned(test_case.exclude)), + ); + let mut selection: Vec = resolved.clone().unwrap_or_default().into_iter().collect(); + selection.sort_unstable(); + + assert_eq!( + resolved + .as_ref() + .map(|_| selection.as_slice()) + .map_err(|error| error.code), + test_case.expected_selection, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/tests/test_types.rs b/crates/sqlbuild-analysis/src/diff_selection/tests/test_types.rs new file mode 100644 index 0000000000..cfcabca4e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct DiffSelectionTestCase { + pub(crate) description: &'static str, + pub(crate) select: &'static [&'static str], + pub(crate) exclude: &'static [&'static str], + /// The selected resources, sorted, or the error code. + pub(crate) expected_selection: Result<&'static [usize], &'static str>, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 36186f5894..e93b39e552 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -10,12 +10,14 @@ pub mod compiler; mod constants; pub mod contracts; pub mod dag_artifact; +pub mod diff_selection; pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; pub mod lineage; pub mod lineage_graph; pub mod manifest; +pub mod python_plan; pub mod python_values; pub mod query_analysis; pub mod relation_names; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/mod.rs b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs new file mode 100644 index 0000000000..8cb536efba --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs @@ -0,0 +1,2 @@ +pub mod ordered_python_names; +pub mod upstream_closure; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/ordered_python_names.rs b/crates/sqlbuild-analysis/src/python_plan/main/ordered_python_names.rs new file mode 100644 index 0000000000..64f96da583 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/main/ordered_python_names.rs @@ -0,0 +1,44 @@ +//! `_ordered_python_names`: selected Python nodes in dependency order, ties by name. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +/// `selected` ordered so each node follows its selected upstreams; nodes on a cycle are left out. +pub fn ordered_python_names( + selected: &[String], + upstream: &HashMap>, +) -> Vec { + let chosen: HashSet<&str> = selected.iter().map(String::as_str).collect(); + let mut in_degree: HashMap<&str, usize> = HashMap::new(); + let mut downstream: HashMap<&str, Vec<&str>> = HashMap::new(); + for name in &chosen { + let upstreams: Vec<&str> = upstream + .get(*name) + .into_iter() + .flatten() + .map(String::as_str) + .filter(|upstream_name| chosen.contains(upstream_name)) + .collect(); + in_degree.insert(name, upstreams.len()); + for upstream_name in upstreams { + downstream.entry(upstream_name).or_default().push(name); + } + } + let mut ready: BTreeSet<&str> = in_degree + .iter() + .filter(|(_, degree)| **degree == 0) + .map(|(name, _)| *name) + .collect(); + let mut ordered: Vec = Vec::new(); + while let Some(name) = ready.pop_first() { + ordered.push(name.to_owned()); + for downstream_name in downstream.get(name).into_iter().flatten() { + if let Some(degree) = in_degree.get_mut(downstream_name) { + *degree -= 1; + if *degree == 0 { + ready.insert(downstream_name); + } + } + } + } + ordered +} diff --git a/crates/sqlbuild-analysis/src/python_plan/main/upstream_closure.rs b/crates/sqlbuild-analysis/src/python_plan/main/upstream_closure.rs new file mode 100644 index 0000000000..e216ba75c9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/main/upstream_closure.rs @@ -0,0 +1,19 @@ +//! Every node upstream of one Python node, sorted. + +use std::collections::{HashMap, HashSet, VecDeque}; + +/// `sorted(transitive_closure(start=start, edges=upstream))`: `start` only when on a cycle. +pub fn upstream_closure(start: &str, upstream: &HashMap>) -> Vec { + let mut visited: HashSet<&str> = HashSet::new(); + let mut frontier: VecDeque<&str> = VecDeque::from([start]); + while let Some(current) = frontier.pop_front() { + for neighbor in upstream.get(current).into_iter().flatten() { + if visited.insert(neighbor) { + frontier.push_back(neighbor); + } + } + } + let mut names: Vec = visited.into_iter().map(str::to_owned).collect(); + names.sort(); + names +} diff --git a/crates/sqlbuild-analysis/src/python_plan/mod.rs b/crates/sqlbuild-analysis/src/python_plan/mod.rs new file mode 100644 index 0000000000..e29b572755 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/mod.rs @@ -0,0 +1,4 @@ +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/python_plan/tests.rs b/crates/sqlbuild-analysis/src/python_plan/tests.rs new file mode 100644 index 0000000000..4f8ba4548a --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_python_plan.rs"] +mod python_plan; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs new file mode 100644 index 0000000000..55a49940b2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -0,0 +1,12 @@ +use std::collections::HashMap; + +pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { + rows.iter() + .map(|(name, upstream)| { + ( + (*name).to_owned(), + upstream.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_python_plan.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_python_plan.rs new file mode 100644 index 0000000000..238d47b04b --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_python_plan.rs @@ -0,0 +1,65 @@ +use crate::python_plan::main::ordered_python_names::ordered_python_names; +use crate::python_plan::main::upstream_closure::upstream_closure; +use crate::python_plan::tests::helpers::edges; +use crate::python_plan::tests::test_types::{ClosureTestCase, OrderTestCase}; + +#[test] +fn given_selected_nodes_when_ordering_then_matches_python_ready_queue() { + let test_cases = [ + OrderTestCase { + description: "ready nodes go by name, dependents after their upstreams", + selected: &["load_orders", "clean_orders", "audit_orders", "fetch_rates"], + upstream: &[ + ("clean_orders", &["load_orders", "fetch_rates"]), + ("audit_orders", &["clean_orders", "unselected"]), + ], + expected_order: &["fetch_rates", "load_orders", "clean_orders", "audit_orders"], + }, + OrderTestCase { + description: "nodes on a cycle are left out", + selected: &["a", "b", "c"], + upstream: &[("a", &["b"]), ("b", &["a"])], + expected_order: &["c"], + }, + ]; + + for test_case in test_cases { + let selected: Vec = test_case + .selected + .iter() + .map(|name| (*name).to_owned()) + .collect(); + + let order = ordered_python_names(&selected, &edges(test_case.upstream)); + + assert_eq!(order, test_case.expected_order, "{}", test_case.description); + } +} + +#[test] +fn given_start_when_closing_upstream_then_matches_python_transitive_closure() { + let test_cases = [ + ClosureTestCase { + description: "every upstream, sorted, without the start", + start: "load", + upstream: &[("load", &["fetch", "auth"]), ("fetch", &["auth"])], + expected_closure: &["auth", "fetch"], + }, + ClosureTestCase { + description: "the start is included when it is on a cycle", + start: "a", + upstream: &[("a", &["b"]), ("b", &["a"])], + expected_closure: &["a", "b"], + }, + ]; + + for test_case in test_cases { + let closure = upstream_closure(test_case.start, &edges(test_case.upstream)); + + assert_eq!( + closure, test_case.expected_closure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs new file mode 100644 index 0000000000..7cab9c803f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(crate) struct OrderTestCase { + pub(crate) description: &'static str, + pub(crate) selected: &'static [&'static str], + pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], + pub(crate) expected_order: &'static [&'static str], +} + +pub(crate) struct ClosureTestCase { + pub(crate) description: &'static str, + pub(crate) start: &'static str, + pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], + pub(crate) expected_closure: &'static [&'static str], +} diff --git a/crates/sqlbuild-cache/Cargo.toml b/crates/sqlbuild-cache/Cargo.toml index c24885ac4b..0116e0d5e3 100644 --- a/crates/sqlbuild-cache/Cargo.toml +++ b/crates/sqlbuild-cache/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true blake3.workspace = true rayon.workspace = true sha2.workspace = true +sqlbuild-core.workspace = true tempfile.workspace = true walkdir.workspace = true diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs index d329f837c8..679ef09814 100644 --- a/crates/sqlbuild-cache/src/lib.rs +++ b/crates/sqlbuild-cache/src/lib.rs @@ -4,6 +4,9 @@ pub mod artifacts; pub mod digest; +pub mod macro_store_keys; +pub mod module_digests; pub mod project_reuse; pub mod project_snapshot; +pub mod stage_capture; pub mod store; diff --git a/crates/sqlbuild-cache/src/macro_store_keys/_helpers/key_text.rs b/crates/sqlbuild-cache/src/macro_store_keys/_helpers/key_text.rs new file mode 100644 index 0000000000..b27440aff8 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/_helpers/key_text.rs @@ -0,0 +1,18 @@ +//! The JSON text a stored macro call key is built from. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// Compact unescaped JSON of `value`; `None` when a lone surrogate leaves it unstorable. +pub(crate) fn key_text(value: &JsonValue) -> Result, JsonEmitError> { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + ensure_ascii: false, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }); + match dumps(value, &dialect) { + Ok(text) => Ok(Some(text)), + Err(JsonEmitError::LoneSurrogate) => Ok(None), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/_helpers/mod.rs b/crates/sqlbuild-cache/src/macro_store_keys/_helpers/mod.rs new file mode 100644 index 0000000000..46dacf77f8 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod key_text; diff --git a/crates/sqlbuild-cache/src/macro_store_keys/main/canonical_sort_text.rs b/crates/sqlbuild-cache/src/macro_store_keys/main/canonical_sort_text.rs new file mode 100644 index 0000000000..4846de26d9 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/main/canonical_sort_text.rs @@ -0,0 +1,16 @@ +//! The text a canonical set member is ordered by. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(item, sort_keys=True)`. +pub fn canonical_sort_text(item: &JsonValue) -> Result { + dumps( + item, + &JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }), + ) +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/main/context_store_token.rs b/crates/sqlbuild-cache/src/macro_store_keys/main/context_store_token.rs new file mode 100644 index 0000000000..bd19b0b1ac --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/main/context_store_token.rs @@ -0,0 +1,13 @@ +//! `context_store_token`: the digest of the whole context a macro may read. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::digest::main::content_digest::content_digest; +use crate::digest::main::hex_digest::hex_digest; +use crate::macro_store_keys::_helpers::key_text::key_text; + +/// The digest of the canonical context's key text, or `None` when it cannot be stored. +pub fn context_store_token(canonical: &JsonValue) -> Result, JsonEmitError> { + Ok(key_text(canonical)?.map(|text| hex_digest(&content_digest(&[text])))) +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/main/macro_store_token.rs b/crates/sqlbuild-cache/src/macro_store_keys/main/macro_store_token.rs new file mode 100644 index 0000000000..ff21556eba --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/main/macro_store_token.rs @@ -0,0 +1,23 @@ +//! `macro_store_token`: a macro named by its name, file, identity and exact source. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::digest::main::content_digest::content_digest; +use crate::digest::main::hex_digest::hex_digest; +use crate::macro_store_keys::_helpers::key_text::key_text; + +/// The key text of `[name, relative path, canonical identity, source digest]`, if storable. +pub fn macro_store_token( + (name, relative_path): (JsonValue, JsonValue), + identity: JsonValue, + raw_source: &str, +) -> Result, JsonEmitError> { + let source_digest: String = hex_digest(&content_digest(&[raw_source])); + key_text(&JsonValue::Array(vec![ + name, + relative_path, + identity, + JsonValue::String(source_digest), + ])) +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/main/mod.rs b/crates/sqlbuild-cache/src/macro_store_keys/main/mod.rs new file mode 100644 index 0000000000..c1df29bfa1 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/main/mod.rs @@ -0,0 +1,3 @@ +pub mod canonical_sort_text; +pub mod context_store_token; +pub mod macro_store_token; diff --git a/crates/sqlbuild-cache/src/macro_store_keys/mod.rs b/crates/sqlbuild-cache/src/macro_store_keys/mod.rs new file mode 100644 index 0000000000..668a17d0aa --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/macro_store_keys/tests.rs b/crates/sqlbuild-cache/src/macro_store_keys/tests.rs new file mode 100644 index 0000000000..f5ec18cce2 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_store_keys.rs"] +mod store_keys; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/macro_store_keys/tests/helpers.rs b/crates/sqlbuild-cache/src/macro_store_keys/tests/helpers.rs new file mode 100644 index 0000000000..c544d873b2 --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/tests/helpers.rs @@ -0,0 +1,9 @@ +use sqlbuild_core::json::models::JsonValue; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn array(items: Vec) -> JsonValue { + JsonValue::Array(items) +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/tests/test_store_keys.rs b/crates/sqlbuild-cache/src/macro_store_keys/tests/test_store_keys.rs new file mode 100644 index 0000000000..202c48451e --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/tests/test_store_keys.rs @@ -0,0 +1,87 @@ +use sqlbuild_core::json::models::JsonValue; + +use crate::macro_store_keys::main::canonical_sort_text::canonical_sort_text; +use crate::macro_store_keys::main::context_store_token::context_store_token; +use crate::macro_store_keys::main::macro_store_token::macro_store_token; +use crate::macro_store_keys::tests::helpers::{array, text}; +use crate::macro_store_keys::tests::test_types::StoreKeyTestCase; + +#[test] +fn given_macro_identity_when_keying_then_matches_python_key_text() { + let test_cases = [ + StoreKeyTestCase { + description: "name, path, identity and source digest as compact JSON", + value: array(vec![text("tuple"), array(vec![text("a")])]), + expected_key: Some( + "[\"cents\",\"macros/money.py\",[\"tuple\",[\"a\"]],\"09b7bf2b19ceac419ff40c8cbe965d0c135424c443d286890f788ec6efb5f92d\"]", + ), + }, + StoreKeyTestCase { + description: "a lone surrogate has no key", + value: JsonValue::Utf16(vec![0xd800]), + expected_key: None, + }, + ]; + + for test_case in test_cases { + let key = macro_store_token( + (text("cents"), text("macros/money.py")), + test_case.value, + "SELECT 1", + ) + .unwrap_or_default(); + + assert_eq!( + key.as_deref(), + test_case.expected_key, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_canonical_context_when_keying_then_matches_python_digest() { + let test_cases = [StoreKeyTestCase { + description: "the digest of the compact context text", + value: array(vec![ + array(vec![ + text("x.MacroContext"), + array(vec![array(vec![text("adapter_name"), text("duckdb")])]), + ]), + JsonValue::Null, + ]), + expected_key: Some("03272a69b9c5e3b3da0067db1a2deab59532aafa30306f624a7c3240eba4740b"), + }]; + + for test_case in test_cases { + let key = context_store_token(&test_case.value).unwrap_or_default(); + + assert_eq!( + key.as_deref(), + test_case.expected_key, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_set_member_when_sorting_then_matches_python_json_dumps() { + let test_cases = [StoreKeyTestCase { + description: "ASCII-escaped with spaced separators", + value: array(vec![text("int"), text("\u{e9}")]), + expected_key: Some("[\"int\", \"\\u00e9\"]"), + }]; + + for test_case in test_cases { + let key = canonical_sort_text(&test_case.value).ok(); + + assert_eq!( + key.as_deref(), + test_case.expected_key, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/macro_store_keys/tests/test_types.rs b/crates/sqlbuild-cache/src/macro_store_keys/tests/test_types.rs new file mode 100644 index 0000000000..00c90d745f --- /dev/null +++ b/crates/sqlbuild-cache/src/macro_store_keys/tests/test_types.rs @@ -0,0 +1,7 @@ +use sqlbuild_core::json::models::JsonValue; + +pub(crate) struct StoreKeyTestCase { + pub(crate) description: &'static str, + pub(crate) value: JsonValue, + pub(crate) expected_key: Option<&'static str>, +} diff --git a/crates/sqlbuild-cache/src/module_digests/_helpers/mod.rs b/crates/sqlbuild-cache/src/module_digests/_helpers/mod.rs new file mode 100644 index 0000000000..995a5f854c --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod python_code_points; diff --git a/crates/sqlbuild-cache/src/module_digests/_helpers/python_code_points.rs b/crates/sqlbuild-cache/src/module_digests/_helpers/python_code_points.rs new file mode 100644 index 0000000000..c7a2d7d85d --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/_helpers/python_code_points.rs @@ -0,0 +1,14 @@ +//! The code points of a Python `str`, the order Python compares strings in. + +use sqlbuild_core::json::models::JsonValue; + +/// The code points of a JSON string, a lone surrogate as its own value; empty for other values. +pub(crate) fn python_code_points(value: &JsonValue) -> Vec { + match value { + JsonValue::String(text) => text.chars().map(u32::from).collect(), + JsonValue::Utf16(units) => char::decode_utf16(units.iter().copied()) + .map(|unit| unit.map_or_else(|lone| u32::from(lone.unpaired_surrogate()), u32::from)) + .collect(), + _ => Vec::new(), + } +} diff --git a/crates/sqlbuild-cache/src/module_digests/main/mod.rs b/crates/sqlbuild-cache/src/module_digests/main/mod.rs new file mode 100644 index 0000000000..f15d3eb0b5 --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/main/mod.rs @@ -0,0 +1 @@ +pub mod module_digests_metadata; diff --git a/crates/sqlbuild-cache/src/module_digests/main/module_digests_metadata.rs b/crates/sqlbuild-cache/src/module_digests/main/module_digests_metadata.rs new file mode 100644 index 0000000000..7f5c37993b --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/main/module_digests_metadata.rs @@ -0,0 +1,26 @@ +//! The store metadata that records each module file's content digest. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::module_digests::_helpers::python_code_points::python_code_points; +use crate::module_digests::types::RowKey; + +/// `json.dumps(sorted([path, digest] for path, digest in rows))`. +pub fn module_digests_metadata(rows: Vec<(JsonValue, JsonValue)>) -> Result { + let mut keyed: Vec<(RowKey, JsonValue)> = rows + .into_iter() + .map(|(path, digest)| { + ( + (python_code_points(&path), python_code_points(&digest)), + JsonValue::Array(vec![path, digest]), + ) + }) + .collect(); + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + dumps( + &JsonValue::Array(keyed.into_iter().map(|(_, row)| row).collect()), + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + ) +} diff --git a/crates/sqlbuild-cache/src/module_digests/mod.rs b/crates/sqlbuild-cache/src/module_digests/mod.rs new file mode 100644 index 0000000000..b4686f3a41 --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/module_digests/tests.rs b/crates/sqlbuild-cache/src/module_digests/tests.rs new file mode 100644 index 0000000000..6820d96d0f --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_module_digests_metadata.rs"] +mod module_digests_metadata; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/module_digests/tests/helpers.rs b/crates/sqlbuild-cache/src/module_digests/tests/helpers.rs new file mode 100644 index 0000000000..7b01c03dd0 --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/tests/helpers.rs @@ -0,0 +1,5 @@ +use sqlbuild_core::json::models::JsonValue; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} diff --git a/crates/sqlbuild-cache/src/module_digests/tests/test_module_digests_metadata.rs b/crates/sqlbuild-cache/src/module_digests/tests/test_module_digests_metadata.rs new file mode 100644 index 0000000000..74ff00b147 --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/tests/test_module_digests_metadata.rs @@ -0,0 +1,37 @@ +use sqlbuild_core::json::models::JsonValue; + +use crate::module_digests::main::module_digests_metadata::module_digests_metadata; +use crate::module_digests::tests::helpers::text; +use crate::module_digests::tests::test_types::MetadataTestCase; + +#[test] +fn given_module_digests_when_encoding_then_matches_python_sorted_json() { + let test_cases = [ + MetadataTestCase { + description: "rows sorted by path, non-ASCII escaped", + rows: vec![ + (text("z/orders.py"), text("b")), + (text("caf\u{e9}.py"), text("a")), + ], + expected_metadata: "[[\"caf\\u00e9.py\", \"a\"], [\"z/orders.py\", \"b\"]]", + }, + MetadataTestCase { + description: "a lone surrogate sorts by code point, before astral characters", + rows: vec![ + (text("\u{1f600}.py"), text("a")), + (JsonValue::Utf16(vec![0xdcff]), text("b")), + ], + expected_metadata: "[[\"\\udcff\", \"b\"], [\"\\ud83d\\ude00.py\", \"a\"]]", + }, + ]; + + for test_case in test_cases { + let metadata = module_digests_metadata(test_case.rows).unwrap_or_default(); + + assert_eq!( + metadata, test_case.expected_metadata, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/module_digests/tests/test_types.rs b/crates/sqlbuild-cache/src/module_digests/tests/test_types.rs new file mode 100644 index 0000000000..dc2e119cdf --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/tests/test_types.rs @@ -0,0 +1,7 @@ +use sqlbuild_core::json::models::JsonValue; + +pub(crate) struct MetadataTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec<(JsonValue, JsonValue)>, + pub(crate) expected_metadata: &'static str, +} diff --git a/crates/sqlbuild-cache/src/module_digests/types.rs b/crates/sqlbuild-cache/src/module_digests/types.rs new file mode 100644 index 0000000000..ffe73b66c8 --- /dev/null +++ b/crates/sqlbuild-cache/src/module_digests/types.rs @@ -0,0 +1,4 @@ +//! The key metadata rows are sorted by. + +/// A row's path and digest code points, the key Python sorts rows by. +pub type RowKey = (Vec, Vec); diff --git a/crates/sqlbuild-cache/src/stage_capture/_helpers/dialect.rs b/crates/sqlbuild-cache/src/stage_capture/_helpers/dialect.rs new file mode 100644 index 0000000000..3e85a0676d --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/_helpers/dialect.rs @@ -0,0 +1,12 @@ +//! The serializer options capture text is written with. + +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; + +/// `json.dumps(value, ensure_ascii=False, allow_nan=False, ...)` with `indent` and separators. +pub(crate) fn capture_dialect(indent: Option, separators: (&str, &str)) -> JsonDialect { + JsonDialect::Stdlib(StdlibJsonOptions { + ensure_ascii: false, + allow_nan: false, + ..StdlibJsonOptions::new(indent).with_separators(separators.0, separators.1) + }) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/_helpers/mod.rs b/crates/sqlbuild-cache/src/stage_capture/_helpers/mod.rs new file mode 100644 index 0000000000..20ec29723a --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod dialect; +pub(crate) mod noise; diff --git a/crates/sqlbuild-cache/src/stage_capture/_helpers/noise.rs b/crates/sqlbuild-cache/src/stage_capture/_helpers/noise.rs new file mode 100644 index 0000000000..2c90868d76 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/_helpers/noise.rs @@ -0,0 +1,130 @@ +//! The three noise patterns of a capture, matched as Python's `re` matches them. + +use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; +use sqlbuild_core::text::main::python_isalnum::python_isalnum; + +use crate::stage_capture::constants::{ + ENGINE_NAMESPACE_PREFIX, INVOCATION_ID_MASK, INVOCATION_ID_PARTS, REFACTOR_STAGING_MASK, + REFACTOR_STAGING_PREFIX, +}; + +/// Python's `\w` for a `str` pattern. +fn is_word(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// Python's `\b` between `characters[at - 1]` and `characters[at]`. +fn at_boundary(characters: &[char], at: usize) -> bool { + let before: bool = at > 0 && is_word(characters[at - 1]); + let after: bool = characters + .get(at) + .is_some_and(|character| is_word(*character)); + before != after +} + +/// Whether `characters[at..]` starts with `literal`. +fn starts_with(characters: &[char], at: usize, literal: &str) -> bool { + (at..) + .zip(literal.chars()) + .all(|(position, expected)| characters.get(position) == Some(&expected)) +} + +/// The number of characters from `at` that satisfy `accepts`, at most `limit`. +fn run_length(characters: &[char], at: usize, limit: usize, accepts: fn(char) -> bool) -> usize { + characters[at.min(characters.len())..] + .iter() + .take(limit) + .take_while(|character| accepts(**character)) + .count() +} + +/// Whether `character` is a lowercase hexadecimal digit, `[0-9a-f]`. +fn is_lower_hex(character: char) -> bool { + character.is_ascii_digit() || ('a'..='f').contains(&character) +} + +/// Whether `character` is in `[0-9a-z_]`. +fn is_staging_char(character: char) -> bool { + character.is_ascii_digit() || character.is_ascii_lowercase() || character == '_' +} + +/// The end of `\b\d{8}T\d{6}Z_[0-9a-f]{12}\b` matched at `at`. +fn invocation_id_end(characters: &[char], at: usize) -> Option { + let (date, time, hex) = INVOCATION_ID_PARTS; + if !at_boundary(characters, at) || run_length(characters, at, date, is_decimal_digit) != date { + return None; + } + let mut position: usize = at + date; + if characters.get(position) != Some(&'T') { + return None; + } + position += 1; + if run_length(characters, position, time, is_decimal_digit) != time { + return None; + } + position += time; + if !starts_with(characters, position, "Z_") { + return None; + } + position += 2; + if run_length(characters, position, hex, is_lower_hex) != hex { + return None; + } + position += hex; + at_boundary(characters, position).then_some(position) +} + +/// The end of `-native-v\d+\b` matched at `at`. +fn engine_namespace_end(characters: &[char], at: usize) -> Option { + if !starts_with(characters, at, ENGINE_NAMESPACE_PREFIX) { + return None; + } + let start: usize = at + ENGINE_NAMESPACE_PREFIX.chars().count(); + let digits: usize = run_length(characters, start, usize::MAX, is_decimal_digit); + (digits > 0 && at_boundary(characters, start + digits)).then_some(start + digits) +} + +/// The end of `\bsqb-refactor-[0-9a-z_]+` matched at `at`. +fn refactor_staging_end(characters: &[char], at: usize) -> Option { + if !at_boundary(characters, at) || !starts_with(characters, at, REFACTOR_STAGING_PREFIX) { + return None; + } + let start: usize = at + REFACTOR_STAGING_PREFIX.chars().count(); + let length: usize = run_length(characters, start, usize::MAX, is_staging_char); + (length > 0).then_some(start + length) +} + +/// `pattern.sub(replacement, text)` for a pattern that never matches the empty string. +fn substituted( + text: &str, + matched_end: fn(&[char], usize) -> Option, + replacement: &str, +) -> String { + let characters: Vec = text.chars().collect(); + let mut result: String = String::with_capacity(text.len()); + let mut position: usize = 0; + while position < characters.len() { + match matched_end(&characters, position) { + Some(end) => { + result.push_str(replacement); + position = end; + } + None => { + result.push(characters[position]); + position += 1; + } + } + } + result +} + +/// Mask invocation ids, then native store suffixes, then refactor staging folders. +pub(crate) fn masked(text: &str) -> String { + let without_ids: String = substituted(text, invocation_id_end, INVOCATION_ID_MASK); + let without_namespaces: String = substituted(&without_ids, engine_namespace_end, ""); + substituted( + &without_namespaces, + refactor_staging_end, + REFACTOR_STAGING_MASK, + ) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs new file mode 100644 index 0000000000..575e072600 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -0,0 +1,365 @@ +//! Stage capture document keys, noise masks and the per-type encoding tables. + +/// The key that replaces a shared subtree with its digest. +pub const SHARED_MARKER: &str = "__shared__"; +/// The key of the capture root in a document with shared nodes. +pub const ROOT_KEY: &str = "__capture__"; +/// The key of the shared nodes of a document. +pub const SHARED_NODES_KEY: &str = "__shared_nodes__"; +/// Characters of compact JSON from which a subtree is shared. +pub const SHARED_MIN_CHARS: usize = 1024; +/// Keys a mapping must avoid to be written as a JSON object. +pub const RESERVED_KEYS: [&str; 3] = ["__shared__", "__capture__", "__shared_nodes__"]; +/// The text that replaces a compile invocation id. +pub const INVOCATION_ID_MASK: &str = ""; +/// The prefix of a native store namespace suffix such as `-native-v1`. +pub const ENGINE_NAMESPACE_PREFIX: &str = "-native-v"; +/// The prefix of a refactor staging folder name. +pub const REFACTOR_STAGING_PREFIX: &str = "sqb-refactor-"; +/// The text that replaces a refactor staging folder name. +pub const REFACTOR_STAGING_MASK: &str = "sqb-refactor-"; +/// Invocation id digits before the `T`, digits before the `Z`, and hexadecimal digits after `_`. +pub const INVOCATION_ID_PARTS: (usize, usize, usize) = (8, 6, 12); +/// `__slots__` entries that are not instance attributes. +pub const SKIPPED_SLOTS: [&str; 2] = ["__dict__", "__weakref__"]; +/// Attributes whose mappings are captured without their order, by type. +pub const UNORDERED_ATTRIBUTES: [(&str, &[&str]); 1] = [( + "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog", + &["analysis_shapes", "expression_shapes", "schemas"], +)]; +/// The attributes captured, in order, for each native type. +pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 13] = [ + ( + "sqlbuild._native:DiscoveredSqlModelFile", + &[ + "file_path", + "relative_path", + "contents", + "header_values", + "header_column_locations", + "output_column_locations", + "query_sql", + "enum_declarations", + "constant_declarations", + "extract_implicit_alias_columns", + ], + ), + ( + "sqlbuild._native:NativeAuditInput", + &[ + "audit_file", + "audit_block", + "sql_body", + "evaluation_mode", + "measurement_contract", + "thresholds", + "minimum_samples", + "measure_sql", + "evidence_sql", + "evidence_limit", + "references", + "attached_target_kind", + "attached_target_name", + "attached_column_name", + "severity", + "run_scope", + "always_run", + "description", + "declaration_usages", + "name", + ], + ), + ( + "sqlbuild._native:NativeCompiledAudit", + &[ + "key", + "scope_deps", + "name", + "definition_name", + "audit_file", + "audit_block", + "sql_body", + "evaluation_mode", + "measurement_contract", + "thresholds", + "minimum_samples", + "measure_sql", + "evidence_sql", + "evidence_limit", + "references", + "attached_target_kind", + "attached_target_name", + "attached_column_name", + "severity", + "run_scope", + "always_run", + "description", + ], + ), + ( + "sqlbuild._native:NativeFunctionInput", + &[ + "function_file", + "name", + "arguments", + "returns", + "body_sql", + "return_columns", + "references", + "database", + "schema", + "logical_database", + "logical_schema", + "fingerprint_database", + "fingerprint_schema", + "fingerprint_logical_database", + "fingerprint_logical_schema", + "language", + "runtime_version", + "entry_point", + "packages", + "declaration_usages", + "tags", + "description", + ], + ), + ( + "sqlbuild._native:NativeCompiledFunction", + &[ + "key", + "deps", + "name", + "relative_path", + "arguments", + "returns", + "body_sql", + "destination", + "fingerprint_destination", + "return_columns", + "references", + "language", + "source_file_path", + "runtime_version", + "entry_point", + "packages", + "tags", + "description", + ], + ), + ( + "sqlbuild._native:NativeSourceInput", + &["source_entry", "source_file", "declaration_usages"], + ), + ( + "sqlbuild._native:NativeCompiledSource", + &["key", "deps", "name", "source_entry", "source_file"], + ), + ( + "sqlbuild._native:NativeSeedInput", + &["seed_file", "schema_entry", "schema_file"], + ), + ( + "sqlbuild._native:NativeCompiledSeed", + &[ + "key", + "deps", + "name", + "seed_file", + "schema_entry", + "schema_file", + "destination", + "external", + ], + ), + ( + "sqlbuild._native:NativeInputsView", + &[ + "project_config", + "local_config", + "discovered_inputs", + "sql_lexical_syntax", + "run_id", + "effective_target_name", + "effective_target", + "compile_cache_dir", + "effective_connection", + "effective_settings", + "no_sql_validation", + "effective_vars", + "macro_context", + "loaded_macros", + "public_enums", + "public_constants", + "model_inputs", + "seed_inputs", + "source_inputs", + "sql_function_inputs", + "test_inputs", + "scenario_inputs", + "audit_inputs", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "declaration_scope", + ], + ), + ( + "sqlbuild._native:NativeProjectView", + &[ + "run_id", + "effective_target_name", + "effective_connection", + "effective_vars", + "binding_catalog", + "effective_target_database", + "effective_target_schema", + "sql_analysis_dialect", + "sql_lexical_syntax", + "compile_cache_dir", + "settings", + "scenario", + "models", + "sources", + "seeds", + "functions", + "audits", + "sql_tests", + "sql_scenarios", + "loader_functions", + "hook_functions", + "enforce_explicit_references", + "sql_hook_files", + "materialization_files", + "public_enums", + "public_constants", + "loaded_macros", + "diagnostics", + "unmatched_literal_sql_relations", + "external_sql_reference_resolver", + "scope_index", + "sql_expansions", + ], + ), + ( + "sqlbuild._native:NativeCompiledModel", + &[ + "key", + "deps", + "name", + "relative_path", + "query_sql", + "config", + "destination", + "references", + "schema_entry", + "inferred_columns", + "fast_lineage_columns", + "fast_lineage_has_star", + "fast_lineage_star_resolved", + "authored_sql", + "authored_query_sql", + "output_column_locations", + "extract_implicit_alias_columns", + "macro_deps", + "enum_declarations", + "constant_declarations", + "enum_columns", + "binding_diagnostics", + "binding_validated", + "dynamic_column_contract", + "unchecked_output_columns", + "rejected_sql_analysis_opt_out", + ], + ), + ( + "sqlbuild._native:NativeModelInput", + &[ + "model_file", + "config", + "query_sql", + "macro_source_sql", + "references", + "schema_entry", + "schema_file", + "sql_validation_enabled", + "rejected_sql_analysis_opt_out", + "enum_declarations", + "constant_declarations", + "enum_columns", + "macro_deps", + "macro_usages", + "declaration_usages", + "sql_expansion", + "lint_expansion", + ], + ), +]; +/// Attributes left out of the capture, by type. +pub const OMITTED_ATTRIBUTES: [(&str, &[&str]); 7] = [ + ("sqlbuild.compiler.scopes.models:ScopeLookup", &["native"]), + ( + "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog", + &["shared_analyses"], + ), + ( + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", + &["native_session"], + ), + ( + "sqlbuild.compiler.compile.models:DeclarationScopeResolver", + &["native_contexts"], + ), + ( + "sqlbuild.compiler.compile.models:DeclarationScopeBuild", + &["sql_test_scans"], + ), + ( + "sqlbuild.compiler.compile.models:CompileProjectInputs", + &["native_project"], + ), + ( + "sqlbuild.compiler.compile.models:CompiledProject", + &["lineage_graph", "native_project", "native_session"], + ), +]; +/// The Python model type a native type is captured as. +pub const CANONICAL_TYPES: [(&str, &str); 9] = [ + ( + "sqlbuild._native:DiscoveredSqlModelFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlModelFile", + ), + ( + "sqlbuild._native:NativeInputsView", + "sqlbuild.compiler.compile.models:CompileProjectInputs", + ), + ( + "sqlbuild._native:NativeProjectView", + "sqlbuild.compiler.compile.models:CompiledProject", + ), + ( + "sqlbuild._native:NativeModelInput", + "sqlbuild.compiler.compile.models:CompileModelInput", + ), + ( + "sqlbuild._native:NativeCompiledModel", + "sqlbuild.compiler.compile.models:CompiledModel", + ), + ( + "sqlbuild._native:NativeCompiledSource", + "sqlbuild.compiler.compile.models:CompiledSource", + ), + ( + "sqlbuild._native:NativeCompiledSeed", + "sqlbuild.compiler.compile.models:CompiledSeed", + ), + ( + "sqlbuild._native:NativeCompiledFunction", + "sqlbuild.compiler.compile.models:CompiledFunction", + ), + ( + "sqlbuild._native:NativeCompiledAudit", + "sqlbuild.compiler.compile.models:CompiledAudit", + ), +]; +/// Sequence types captured as their items. +pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; diff --git a/crates/sqlbuild-cache/src/stage_capture/main/capture_sort_key.rs b/crates/sqlbuild-cache/src/stage_capture/main/capture_sort_key.rs new file mode 100644 index 0000000000..ed8b4a2e07 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/capture_sort_key.rs @@ -0,0 +1,17 @@ +//! The key unordered capture items are sorted by. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::stage_capture::_helpers::noise::masked; + +/// `mask_capture_noise(json.dumps(item, sort_keys=True, ensure_ascii=False))`. +pub fn capture_sort_key(item: &JsonValue) -> Result { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + ensure_ascii: false, + sort_keys: true, + ..StdlibJsonOptions::new(None) + }); + Ok(masked(&dumps(item, &dialect)?)) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/main/compact_capture_json.rs b/crates/sqlbuild-cache/src/stage_capture/main/compact_capture_json.rs new file mode 100644 index 0000000000..f17f3d80aa --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/compact_capture_json.rs @@ -0,0 +1,12 @@ +//! The compact JSON text a capture subtree is shared and digested as. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::JsonValue; + +use crate::stage_capture::_helpers::dialect::capture_dialect; + +/// `json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))`. +pub fn compact_capture_json(value: &JsonValue) -> Result { + dumps(value, &capture_dialect(None, (",", ":"))) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/main/mask_capture_noise.rs b/crates/sqlbuild-cache/src/stage_capture/main/mask_capture_noise.rs new file mode 100644 index 0000000000..55698691a6 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/mask_capture_noise.rs @@ -0,0 +1,8 @@ +//! `mask_capture_noise`: the text of a capture without what varies between equal compiles. + +use crate::stage_capture::_helpers::noise::masked; + +/// Mask invocation ids, native store suffixes and refactor staging folders of a compile. +pub fn mask_capture_noise(text: &str) -> String { + masked(text) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/main/mod.rs b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs new file mode 100644 index 0000000000..96a12d57ab --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs @@ -0,0 +1,4 @@ +pub mod capture_sort_key; +pub mod compact_capture_json; +pub mod mask_capture_noise; +pub mod shared_node_digest; diff --git a/crates/sqlbuild-cache/src/stage_capture/main/shared_node_digest.rs b/crates/sqlbuild-cache/src/stage_capture/main/shared_node_digest.rs new file mode 100644 index 0000000000..37b42b5e94 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/shared_node_digest.rs @@ -0,0 +1,13 @@ +//! The digest a shared capture subtree is keyed by. + +use sha2::{Digest, Sha256}; + +use crate::stage_capture::_helpers::noise::masked; + +/// SHA-256 hex digest of the subtree's compact JSON with its noise masked. +pub fn shared_node_digest(text: &str) -> String { + Sha256::digest(masked(text).as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/sqlbuild-cache/src/stage_capture/mod.rs b/crates/sqlbuild-cache/src/stage_capture/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/stage_capture/models.rs b/crates/sqlbuild-cache/src/stage_capture/models.rs new file mode 100644 index 0000000000..5d86a91a68 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/models.rs @@ -0,0 +1,36 @@ +//! The capture document, written as shared nodes followed by its root. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::JsonValue; + +use crate::stage_capture::_helpers::dialect::capture_dialect; +use crate::stage_capture::constants::{ROOT_KEY, SHARED_NODES_KEY}; + +/// One capture document: each shared node and then the root, each on its own line. +#[derive(Debug, Default)] +pub struct CaptureDocument { + shared: bool, +} + +impl CaptureDocument { + /// The text that appends one shared node, given as compact JSON. + pub fn shared_chunk(&mut self, digest: &str, text: &str) -> String { + let prefix: String = if self.shared { + ",\n".to_owned() + } else { + format!("{{\"{SHARED_NODES_KEY}\": {{\n") + }; + self.shared = true; + format!("{prefix}\"{digest}\": {text}") + } + + /// The text that writes the root and closes the document. + pub fn final_chunk(&self, root: &JsonValue) -> Result { + if !self.shared { + return Ok(dumps(root, &capture_dialect(Some(1), (",", ": ")))? + "\n"); + } + let compact: String = dumps(root, &capture_dialect(None, (",", ":")))?; + Ok(format!("\n}},\n\"{ROOT_KEY}\": {compact}\n}}\n")) + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests.rs b/crates/sqlbuild-cache/src/stage_capture/tests.rs new file mode 100644 index 0000000000..80c6c333f9 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_capture_document.rs"] +mod capture_document; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_mask_capture_noise.rs"] +mod mask_capture_noise; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs new file mode 100644 index 0000000000..59924b9967 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs @@ -0,0 +1,9 @@ +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_capture_document.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_capture_document.rs new file mode 100644 index 0000000000..5619ea386a --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_capture_document.rs @@ -0,0 +1,66 @@ +use sqlbuild_core::json::models::JsonValue; + +use crate::stage_capture::main::capture_sort_key::capture_sort_key; +use crate::stage_capture::models::CaptureDocument; +use crate::stage_capture::tests::helpers::{integer, text}; +use crate::stage_capture::tests::test_types::{DocumentTestCase, SortKeyTestCase}; + +#[test] +fn given_shared_nodes_and_root_when_writing_document_then_matches_python_writer() { + let test_cases = [ + DocumentTestCase { + description: "a document without shared nodes is the indented root", + shared: &[], + root: JsonValue::Object(vec![( + "a".to_owned(), + JsonValue::Array(vec![integer(1), text("\u{e9}")]), + )]), + expected_document: "{\n \"a\": [\n 1,\n \"\u{e9}\"\n ]\n}\n", + }, + DocumentTestCase { + description: "shared nodes precede the compact root", + shared: &[("d1", "[1]"), ("d2", "\"x\"")], + root: JsonValue::Object(vec![( + "a".to_owned(), + JsonValue::Object(vec![("__shared__".to_owned(), text("d1"))]), + )]), + expected_document: "{\"__shared_nodes__\": {\n\"d1\": [1],\n\"d2\": \"x\"\n},\n\"__capture__\": {\"a\":{\"__shared__\":\"d1\"}}\n}\n", + }, + ]; + + for test_case in test_cases { + let mut document = CaptureDocument::default(); + let mut written = String::new(); + for (digest, node) in test_case.shared { + written.push_str(&document.shared_chunk(digest, node)); + } + written.push_str(&document.final_chunk(&test_case.root).unwrap_or_default()); + + assert_eq!( + written, test_case.expected_document, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_item_when_keying_for_sort_then_matches_python_sort_key() { + let test_cases = [SortKeyTestCase { + description: "keys are sorted, separators spaced and noise masked", + item: JsonValue::Object(vec![ + ("b".to_owned(), integer(1)), + ( + "a".to_owned(), + JsonValue::Array(vec![integer(2), text("x 20261012T223835Z_81de82a3e779")]), + ), + ]), + expected_key: "{\"a\": [2, \"x \"], \"b\": 1}", + }]; + + for test_case in test_cases { + let key = capture_sort_key(&test_case.item).unwrap_or_default(); + + assert_eq!(key, test_case.expected_key, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_mask_capture_noise.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_mask_capture_noise.rs new file mode 100644 index 0000000000..b571222625 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_mask_capture_noise.rs @@ -0,0 +1,78 @@ +use crate::stage_capture::main::mask_capture_noise::mask_capture_noise; +use crate::stage_capture::main::shared_node_digest::shared_node_digest; +use crate::stage_capture::tests::test_types::MaskTestCase; + +#[test] +fn given_capture_text_when_masking_noise_then_matches_python_patterns() { + let test_cases = [ + MaskTestCase { + description: "an invocation id between words is masked", + text: "run 20261012T223835Z_81de82a3e779 done", + expected_masked: "run done", + }, + MaskTestCase { + description: "an invocation id inside a word is kept", + text: "x20261012T223835Z_81de82a3e779", + expected_masked: "x20261012T223835Z_81de82a3e779", + }, + MaskTestCase { + description: "an invocation id followed by a word character is kept", + text: "20261012T223835Z_81de82a3e7790", + expected_masked: "20261012T223835Z_81de82a3e7790", + }, + MaskTestCase { + description: "decimal digits of any script form an invocation id", + text: "\u{663}\u{661}\u{662}\u{663}\u{661}\u{662}\u{663}\u{661}T123456Z_abcdefabcdef!", + expected_masked: "!", + }, + MaskTestCase { + description: "native store suffixes are removed only before a word boundary", + text: "target/cache/compiler-native-v1/x and -native-v12a and -native-v3", + expected_masked: "target/cache/compiler/x and -native-v12a and ", + }, + MaskTestCase { + description: "a suffix without a version is kept", + text: "-native-v", + expected_masked: "-native-v", + }, + MaskTestCase { + description: "staging folders are masked after a word boundary", + text: "/tmp/sqb-refactor-ab_9Z/models and xsqb-refactor-q and sqb-refactor-", + expected_masked: "/tmp/sqb-refactor-Z/models and xsqb-refactor-q and sqb-refactor-", + }, + MaskTestCase { + description: "a combining mark is not a word character", + text: "caf\u{e9}-native-v2\u{301} sqb-refactor-a\u{301}", + expected_masked: "caf\u{e9}\u{301} sqb-refactor-\u{301}", + }, + ]; + + for test_case in test_cases { + let masked = mask_capture_noise(test_case.text); + + assert_eq!( + masked, test_case.expected_masked, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_shared_text_when_digesting_then_masks_noise_before_hashing() { + let test_cases = [MaskTestCase { + description: "the digest of Python's masked text", + text: "{\"a\":\"20261012T223835Z_81de82a3e779\"}", + expected_masked: "ec2780ba023eb7dd5d4579e606aee7fb714e8c5373d30f2d5f3a4cdf613f6d7f", + }]; + + for test_case in test_cases { + let digest = shared_node_digest(test_case.text); + + assert_eq!( + digest, test_case.expected_masked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs new file mode 100644 index 0000000000..9962247a4d --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs @@ -0,0 +1,20 @@ +use sqlbuild_core::json::models::JsonValue; + +pub(crate) struct MaskTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_masked: &'static str, +} + +pub(crate) struct DocumentTestCase { + pub(crate) description: &'static str, + pub(crate) shared: &'static [(&'static str, &'static str)], + pub(crate) root: JsonValue, + pub(crate) expected_document: &'static str, +} + +pub(crate) struct SortKeyTestCase { + pub(crate) description: &'static str, + pub(crate) item: JsonValue, + pub(crate) expected_key: &'static str, +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/child_nodes.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/child_nodes.rs new file mode 100644 index 0000000000..ae7d58809c --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/child_nodes.rs @@ -0,0 +1,9 @@ +//! CPython's `ast.iter_child_nodes` over mirrored nodes. + +use crate::cpython_tree::_helpers::children::children; +use crate::cpython_tree::models::CpythonNode; + +/// The children of `node` that can hold other nodes, in CPython's `_fields` order. +pub(crate) fn child_nodes<'a>(node: &CpythonNode<'a>) -> Vec> { + children(node) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs index 67ecd869e5..413d360e0d 100644 --- a/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod child_nodes; pub(crate) mod joined_values; pub(crate) mod parse_cpython_module; pub(crate) mod source_position; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/models.rs b/crates/sqlbuild-pysource/src/cpython_tree/models.rs index bcbe85e2fd..7bffa5c21f 100644 --- a/crates/sqlbuild-pysource/src/cpython_tree/models.rs +++ b/crates/sqlbuild-pysource/src/cpython_tree/models.rs @@ -1,7 +1,7 @@ //! CPython `ast` nodes mirrored over the ruff tree, and the line starts of a source. use ruff_python_ast::{ - Comprehension, ElifElseClause, ExceptHandler, Expr, InterpolatedElement, + Comprehension, ElifElseClause, ExceptHandler, Expr, ExprName, InterpolatedElement, InterpolatedStringFormatSpec, Keyword, MatchCase, ModModule, Parameter, Parameters, Pattern, Stmt, TypeParam, WithItem, }; @@ -31,6 +31,24 @@ pub(crate) enum CpythonNode<'a> { FormatSpec(&'a InterpolatedStringFormatSpec), } +impl<'a> CpythonNode<'a> { + /// The `ExprName` this node is, if it is a `Name`. + pub(crate) fn name(&self) -> Option<&'a ExprName> { + match self { + Self::Expr(Expr::Name(name)) => Some(name), + _ => None, + } + } + + /// Whether a depth-first walk of runtime nodes stops here: nested functions, lambdas, classes. + pub(crate) fn opens_scope(&self) -> bool { + matches!( + self, + Self::Stmt(Stmt::FunctionDef(_) | Stmt::ClassDef(_)) | Self::Expr(Expr::Lambda(_)) + ) + } +} + /// A merged string constant and the offset CPython locates it at. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct TextConstant { diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs index 4a1551516a..ba8392c0c5 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs @@ -4,7 +4,8 @@ use ruff_python_ast::{Expr, ModModule, Stmt, StmtFunctionDef}; use ruff_text_size::Ranged; use std::collections::HashSet; -use crate::macro_modules::_helpers::python_nodes::PyNode; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::CpythonNode; use crate::macro_modules::constants::PRIVATE_PREFIX; /// The first public name a non-function top-level statement declares, if any. @@ -18,8 +19,8 @@ pub(crate) fn public_non_function_declaration(statement: &Stmt) -> Option assign .targets .iter() - .flat_map(|target| PyNode::Expr(target).walk()) - .filter_map(PyNode::name) + .flat_map(|target| walk_breadth_first(vec![CpythonNode::Expr(target)])) + .filter_map(|node| node.name()) .map(|name| name.id.as_str().to_owned()) .collect(), Stmt::AnnAssign(assign) => match assign.target.as_ref() { diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs index 03e1061fc8..f5708dad38 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs @@ -3,7 +3,9 @@ use ruff_python_ast::{Expr, ExprContext, ModModule, StmtFunctionDef}; use std::collections::{HashMap, HashSet}; -use crate::macro_modules::_helpers::python_nodes::PyNode; +use crate::cpython_tree::main::child_nodes::child_nodes; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::CpythonNode; /// The identity of an `ast.Name` node: its address in the parsed tree. pub(crate) type NodeId = *const Expr; @@ -23,20 +25,20 @@ pub(crate) struct FunctionCalls { } /// The nodes of a function body, depth first, without entering nested scopes. -pub(crate) fn runtime_nodes(function: &StmtFunctionDef) -> Vec> { - let mut nodes: Vec> = Vec::new(); - let mut pending: Vec> = function.body.iter().rev().map(PyNode::Stmt).collect(); +pub(crate) fn runtime_nodes(function: &StmtFunctionDef) -> Vec> { + let mut nodes: Vec> = Vec::new(); + let mut pending: Vec> = + function.body.iter().rev().map(CpythonNode::Stmt).collect(); while let Some(node) = pending.pop() { - nodes.push(node); - if node.opens_scope() { - continue; + if !node.opens_scope() { + pending.extend(child_nodes(&node).into_iter().rev()); } - pending.extend(node.children().into_iter().rev()); + nodes.push(node); } nodes } -fn bound_names(function: &StmtFunctionDef, nodes: &[PyNode<'_>]) -> HashSet { +fn bound_names(function: &StmtFunctionDef, nodes: &[CpythonNode<'_>]) -> HashSet { let parameters = &function.parameters; let mut names: HashSet = parameters .posonlyargs @@ -66,9 +68,9 @@ fn first_sorted<'a>(names: impl Iterator) -> Option<&'a Strin names.min() } -fn direct_call_name(node: PyNode<'_>) -> Option<&Expr> { +fn direct_call_name<'a>(node: &CpythonNode<'a>) -> Option<&'a Expr> { match node { - PyNode::Expr(Expr::Call(call)) if call.func.is_name_expr() => Some(&call.func), + CpythonNode::Expr(Expr::Call(call)) if call.func.is_name_expr() => Some(&call.func), _ => None, } } @@ -76,7 +78,7 @@ fn direct_call_name(node: PyNode<'_>) -> Option<&Expr> { /// The calls of one function, or the message of the first misuse Python reports. pub(crate) fn function_calls( function: &StmtFunctionDef, - nodes: &[PyNode<'_>], + nodes: &[CpythonNode<'_>], names: &CallNames<'_>, ) -> Result { let display: &str = names.display; @@ -93,13 +95,13 @@ pub(crate) fn function_calls( } let direct_calls: HashSet = nodes .iter() - .filter_map(|node| direct_call_name(*node)) + .filter_map(direct_call_name) .map(std::ptr::from_ref) .collect(); let mut dependencies: Vec = Vec::new(); let mut helpers: Vec = Vec::new(); for node in nodes { - if let PyNode::Expr(expression @ Expr::Name(name)) = node { + if let CpythonNode::Expr(expression @ Expr::Name(name)) = node { let direct: bool = direct_calls.contains(&std::ptr::from_ref(*expression)); if !direct && names.imported.contains_key(name.id.as_str()) { return Err(format!( @@ -116,7 +118,7 @@ pub(crate) fn function_calls( )); } } - let Some(Expr::Name(called)) = direct_call_name(*node) else { + let Some(Expr::Name(called)) = direct_call_name(node) else { continue; }; let called: &str = called.id.as_str(); @@ -154,17 +156,17 @@ pub(crate) fn validate_use_locations( runtime: &HashSet, ) -> Result<(), String> { let display: &str = names.display; - let outside: Vec<(&str, NodeId)> = PyNode::Module(module) - .walk() - .into_iter() - .filter_map(|node| match node { - PyNode::Expr(expression @ Expr::Name(name)) => { - Some((name.id.as_str(), std::ptr::from_ref(expression))) - } - _ => None, - }) - .filter(|(_, id)| !runtime.contains(id)) - .collect(); + let outside: Vec<(&str, NodeId)> = + walk_breadth_first(module.body.iter().map(CpythonNode::Stmt).collect()) + .into_iter() + .filter_map(|node| match node { + CpythonNode::Expr(expression @ Expr::Name(name)) => { + Some((name.id.as_str(), std::ptr::from_ref(expression))) + } + _ => None, + }) + .filter(|(_, id)| !runtime.contains(id)) + .collect(); if let Some((name, _)) = outside .iter() .find(|(name, _)| names.imported.contains_key(*name)) diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs index 0c53353130..18d81a3751 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs @@ -3,4 +3,3 @@ pub(crate) mod function_calls; pub(crate) mod module_analysis; pub(crate) mod module_names; pub(crate) mod project_analysis; -pub(crate) mod python_nodes; diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs index 6163f88223..a627a23261 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs @@ -5,6 +5,7 @@ use ruff_python_ast::{ModModule, Stmt, StmtFunctionDef, StmtImportFrom}; use std::collections::{HashMap, HashSet}; use std::iter::once; +use crate::cpython_tree::models::CpythonNode; use crate::macro_modules::_helpers::declarations::module_exports; use crate::macro_modules::_helpers::function_calls::{ CallNames, NodeId, function_calls, runtime_nodes, unique, validate_use_locations, @@ -12,7 +13,6 @@ use crate::macro_modules::_helpers::function_calls::{ use crate::macro_modules::_helpers::module_names::{ import_visible, imported_module_names, is_project_macro_import, resolve_import_from, }; -use crate::macro_modules::_helpers::python_nodes::PyNode; use crate::macro_modules::constants::{CYCLE_SEPARATOR, STAR_IMPORT_NAME}; use crate::macro_modules::models::MacroModuleSource; @@ -121,9 +121,9 @@ pub(crate) fn analyze_module( let mut runtime: HashSet = HashSet::new(); for name in &function_order { let function: &StmtFunctionDef = functions[name]; - let nodes: Vec> = runtime_nodes(function); + let nodes: Vec> = runtime_nodes(function); runtime.extend(nodes.iter().filter_map(|node| match node { - PyNode::Expr(expression) => Some(std::ptr::from_ref(*expression)), + CpythonNode::Expr(expression) => Some(std::ptr::from_ref(*expression)), _ => None, })); let calls = function_calls(function, &nodes, &names)?; diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs deleted file mode 100644 index 3f08569e4b..0000000000 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs +++ /dev/null @@ -1,397 +0,0 @@ -//! The ruff tree as CPython `ast` nodes in `iter_child_nodes` order, leaf nodes left out. - -use ruff_python_ast::{ - Arguments, Comprehension, Decorator, ElifElseClause, ExceptHandler, ExceptHandlerExceptHandler, - Expr, ExprName, FStringPart, InterpolatedElement, InterpolatedStringElement, - InterpolatedStringElements, InterpolatedStringFormatSpec, Keyword, MatchCase, ModModule, - Parameter, Parameters, Pattern, Stmt, TypeParam, TypeParams, WithItem, -}; -use std::collections::VecDeque; -use std::iter::once; - -/// One CPython `ast` node over the ruff tree. -#[derive(Clone, Copy)] -pub(crate) enum PyNode<'a> { - Module(&'a ModModule), - Stmt(&'a Stmt), - Expr(&'a Expr), - /// An `If` whose `orelse` CPython nests from ruff's `elif`/`else` clauses. - IfChain(&'a Expr, &'a [Stmt], &'a [ElifElseClause]), - Arguments(&'a Parameters), - Arg(&'a Parameter), - Keyword(&'a Keyword), - Comprehension(&'a Comprehension), - WithItem(&'a WithItem), - MatchCase(&'a MatchCase), - Pattern(&'a Pattern), - Handler(&'a ExceptHandlerExceptHandler), - TypeParam(&'a TypeParam), - /// A `FormattedValue` or `Interpolation`. - Formatted(&'a InterpolatedElement), - /// The `JoinedStr` of a format spec. - FormatSpec(&'a InterpolatedStringFormatSpec), -} - -impl<'a> PyNode<'a> { - /// The `ExprName` this node is, if it is a `Name`. - pub(crate) fn name(self) -> Option<&'a ExprName> { - match self { - Self::Expr(Expr::Name(name)) => Some(name), - _ => None, - } - } - - /// Whether a depth-first walk of runtime nodes stops here: nested functions, lambdas, classes. - pub(crate) fn opens_scope(self) -> bool { - matches!( - self, - Self::Stmt(Stmt::FunctionDef(_) | Stmt::ClassDef(_)) | Self::Expr(Expr::Lambda(_)) - ) - } - - /// The children `ast.iter_child_nodes` yields, in order, without leaf nodes. - pub(crate) fn children(self) -> Vec> { - match self { - Self::Module(module) => statements(&module.body).collect(), - Self::Stmt(statement) => statement_children(statement), - Self::Expr(expression) => expression_children(expression), - Self::IfChain(test, body, clauses) => if_children((test, body, clauses)), - Self::Arguments(parameters) => argument_children(parameters), - Self::Arg(parameter) => optional(parameter.annotation.as_deref()).collect(), - Self::Keyword(keyword) => vec![Self::Expr(&keyword.value)], - Self::Comprehension(comprehension) => [ - Self::Expr(&comprehension.target), - Self::Expr(&comprehension.iter), - ] - .into_iter() - .chain(expressions(&comprehension.ifs)) - .collect(), - Self::WithItem(item) => once(Self::Expr(&item.context_expr)) - .chain(optional(item.optional_vars.as_deref())) - .collect(), - Self::MatchCase(case) => once(Self::Pattern(&case.pattern)) - .chain(optional(case.guard.as_deref())) - .chain(statements(&case.body)) - .collect(), - Self::Pattern(pattern) => pattern_children(pattern), - Self::Handler(handler) => optional(handler.type_.as_deref()) - .chain(statements(&handler.body)) - .collect(), - Self::TypeParam(parameter) => type_param_children(parameter), - Self::Formatted(element) => once(Self::Expr(&element.expression)) - .chain(element.format_spec.as_deref().map(Self::FormatSpec)) - .collect(), - Self::FormatSpec(spec) => interpolations(&spec.elements).collect(), - } - } - - /// Every node `ast.walk` yields from this one, in its breadth-first order. - pub(crate) fn walk(self) -> Vec> { - let mut found: Vec> = Vec::new(); - let mut pending: VecDeque> = VecDeque::from([self]); - while let Some(node) = pending.pop_front() { - pending.extend(node.children()); - found.push(node); - } - found - } -} - -fn statements<'a>(body: &'a [Stmt]) -> impl Iterator> { - body.iter().map(PyNode::Stmt) -} - -fn expressions<'a>(values: &'a [Expr]) -> impl Iterator> { - values.iter().map(PyNode::Expr) -} - -fn optional(value: Option<&Expr>) -> impl Iterator> { - value.into_iter().map(PyNode::Expr) -} - -fn type_params(parameters: Option<&TypeParams>) -> impl Iterator> { - parameters - .into_iter() - .flat_map(|parameters| parameters.type_params.iter().map(PyNode::TypeParam)) -} - -fn decorators(decorators: &[Decorator]) -> impl Iterator> { - decorators - .iter() - .map(|decorator| PyNode::Expr(&decorator.expression)) -} - -fn handlers(handlers: &[ExceptHandler]) -> impl Iterator> { - handlers - .iter() - .map(|ExceptHandler::ExceptHandler(handler)| PyNode::Handler(handler)) -} - -fn class_bases(arguments: Option<&Arguments>) -> Vec> { - arguments.map_or_else(Vec::new, |arguments| { - expressions(&arguments.args) - .chain(arguments.keywords.iter().map(PyNode::Keyword)) - .collect() - }) -} - -fn statement_children(statement: &Stmt) -> Vec> { - match statement { - Stmt::FunctionDef(function) => once(PyNode::Arguments(&function.parameters)) - .chain(statements(&function.body)) - .chain(decorators(&function.decorator_list)) - .chain(optional(function.returns.as_deref())) - .chain(type_params(function.type_params.as_deref())) - .collect(), - Stmt::ClassDef(class) => class_bases(class.arguments.as_deref()) - .into_iter() - .chain(statements(&class.body)) - .chain(decorators(&class.decorator_list)) - .chain(type_params(class.type_params.as_deref())) - .collect(), - Stmt::Return(statement) => optional(statement.value.as_deref()).collect(), - Stmt::Delete(statement) => expressions(&statement.targets).collect(), - Stmt::TypeAlias(statement) => once(PyNode::Expr(&statement.name)) - .chain(type_params(statement.type_params.as_deref())) - .chain(once(PyNode::Expr(&statement.value))) - .collect(), - Stmt::Assign(statement) => expressions(&statement.targets) - .chain(once(PyNode::Expr(&statement.value))) - .collect(), - Stmt::AugAssign(statement) => vec![ - PyNode::Expr(&statement.target), - PyNode::Expr(&statement.value), - ], - Stmt::AnnAssign(statement) => [ - PyNode::Expr(&statement.target), - PyNode::Expr(&statement.annotation), - ] - .into_iter() - .chain(optional(statement.value.as_deref())) - .collect(), - Stmt::For(statement) => [ - PyNode::Expr(&statement.target), - PyNode::Expr(&statement.iter), - ] - .into_iter() - .chain(statements(&statement.body)) - .chain(statements(&statement.orelse)) - .collect(), - Stmt::While(statement) => once(PyNode::Expr(&statement.test)) - .chain(statements(&statement.body)) - .chain(statements(&statement.orelse)) - .collect(), - Stmt::If(statement) => if_children(( - &statement.test, - &statement.body, - &statement.elif_else_clauses, - )), - Stmt::With(statement) => statement - .items - .iter() - .map(PyNode::WithItem) - .chain(statements(&statement.body)) - .collect(), - Stmt::Match(statement) => once(PyNode::Expr(&statement.subject)) - .chain(statement.cases.iter().map(PyNode::MatchCase)) - .collect(), - Stmt::Raise(statement) => optional(statement.exc.as_deref()) - .chain(optional(statement.cause.as_deref())) - .collect(), - Stmt::Try(statement) => statements(&statement.body) - .chain(handlers(&statement.handlers)) - .chain(statements(&statement.orelse)) - .chain(statements(&statement.finalbody)) - .collect(), - Stmt::Assert(statement) => once(PyNode::Expr(&statement.test)) - .chain(optional(statement.msg.as_deref())) - .collect(), - Stmt::Expr(statement) => vec![PyNode::Expr(&statement.value)], - Stmt::Import(_) - | Stmt::ImportFrom(_) - | Stmt::Global(_) - | Stmt::Nonlocal(_) - | Stmt::Pass(_) - | Stmt::Break(_) - | Stmt::Continue(_) - | Stmt::IpyEscapeCommand(_) => Vec::new(), - } -} - -/// An `If`'s test and body, then its `orelse`: the next `elif` as a nested `If`, or `else` body. -fn if_children<'a>( - (test, body, clauses): (&'a Expr, &'a [Stmt], &'a [ElifElseClause]), -) -> Vec> { - let orelse: Vec> = clauses - .split_first() - .map_or_else(Vec::new, |(clause, rest)| match &clause.test { - Some(elif_test) => vec![PyNode::IfChain(elif_test, &clause.body, rest)], - None => statements(&clause.body).collect(), - }); - once(PyNode::Expr(test)) - .chain(statements(body)) - .chain(orelse) - .collect() -} - -/// `posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults`. -fn argument_children(parameters: &Parameters) -> Vec> { - let positional = || parameters.posonlyargs.iter().chain(¶meters.args); - positional() - .map(|parameter| PyNode::Arg(¶meter.parameter)) - .chain(parameters.vararg.as_deref().map(PyNode::Arg)) - .chain( - parameters - .kwonlyargs - .iter() - .map(|parameter| PyNode::Arg(¶meter.parameter)), - ) - .chain( - parameters - .kwonlyargs - .iter() - .filter_map(|parameter| parameter.default.as_deref().map(PyNode::Expr)), - ) - .chain(parameters.kwarg.as_deref().map(PyNode::Arg)) - .chain(positional().filter_map(|parameter| parameter.default.as_deref().map(PyNode::Expr))) - .collect() -} - -fn interpolations(elements: &InterpolatedStringElements) -> impl Iterator> { - elements.iter().filter_map(|element| match element { - InterpolatedStringElement::Interpolation(interpolation) => { - Some(PyNode::Formatted(interpolation)) - } - InterpolatedStringElement::Literal(_) => None, - }) -} - -fn fstring_parts(part: &FStringPart) -> Vec> { - match part { - FStringPart::FString(fstring) => interpolations(&fstring.elements).collect(), - FStringPart::Literal(_) => Vec::new(), - } -} - -fn comprehension_children<'a>( - leading: Vec>, - generators: &'a [Comprehension], -) -> Vec> { - leading - .into_iter() - .chain(generators.iter().map(PyNode::Comprehension)) - .collect() -} - -fn expression_children(expression: &Expr) -> Vec> { - match expression { - Expr::BoolOp(node) => expressions(&node.values).collect(), - Expr::Named(node) => vec![PyNode::Expr(&node.target), PyNode::Expr(&node.value)], - Expr::BinOp(node) => vec![PyNode::Expr(&node.left), PyNode::Expr(&node.right)], - Expr::UnaryOp(node) => vec![PyNode::Expr(&node.operand)], - Expr::Lambda(node) => node - .parameters - .as_deref() - .map(PyNode::Arguments) - .into_iter() - .chain(once(PyNode::Expr(&node.body))) - .collect(), - Expr::If(node) => vec![ - PyNode::Expr(&node.test), - PyNode::Expr(&node.body), - PyNode::Expr(&node.orelse), - ], - Expr::Dict(node) => node - .items - .iter() - .filter_map(|item| item.key.as_ref().map(PyNode::Expr)) - .chain(node.items.iter().map(|item| PyNode::Expr(&item.value))) - .collect(), - Expr::Set(node) => expressions(&node.elts).collect(), - Expr::ListComp(node) => { - comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) - } - Expr::SetComp(node) => { - comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) - } - Expr::DictComp(node) => comprehension_children( - optional(node.key.as_deref()) - .chain(once(PyNode::Expr(&node.value))) - .collect(), - &node.generators, - ), - Expr::Generator(node) => { - comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) - } - Expr::Await(node) => vec![PyNode::Expr(&node.value)], - Expr::Yield(node) => optional(node.value.as_deref()).collect(), - Expr::YieldFrom(node) => vec![PyNode::Expr(&node.value)], - Expr::Compare(node) => once(PyNode::Expr(&node.left)) - .chain(expressions(&node.comparators)) - .collect(), - Expr::Call(node) => once(PyNode::Expr(&node.func)) - .chain(expressions(&node.arguments.args)) - .chain(node.arguments.keywords.iter().map(PyNode::Keyword)) - .collect(), - Expr::FString(node) => node.value.iter().flat_map(fstring_parts).collect(), - Expr::TString(node) => node - .value - .iter() - .flat_map(|tstring| interpolations(&tstring.elements)) - .collect(), - Expr::Attribute(node) => vec![PyNode::Expr(&node.value)], - Expr::Subscript(node) => vec![PyNode::Expr(&node.value), PyNode::Expr(&node.slice)], - Expr::Starred(node) => vec![PyNode::Expr(&node.value)], - Expr::List(node) => expressions(&node.elts).collect(), - Expr::Tuple(node) => expressions(&node.elts).collect(), - Expr::Slice(node) => optional(node.lower.as_deref()) - .chain(optional(node.upper.as_deref())) - .chain(optional(node.step.as_deref())) - .collect(), - Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::NumberLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) - | Expr::EllipsisLiteral(_) - | Expr::Name(_) - | Expr::IpyEscapeCommand(_) => Vec::new(), - } -} - -fn pattern_children(pattern: &Pattern) -> Vec> { - match pattern { - Pattern::MatchValue(node) => vec![PyNode::Expr(&node.value)], - Pattern::MatchSequence(node) => node.patterns.iter().map(PyNode::Pattern).collect(), - Pattern::MatchMapping(node) => expressions(&node.keys) - .chain(node.patterns.iter().map(PyNode::Pattern)) - .collect(), - Pattern::MatchClass(node) => once(PyNode::Expr(&node.cls)) - .chain(node.arguments.patterns.iter().map(PyNode::Pattern)) - .chain( - node.arguments - .keywords - .iter() - .map(|keyword| PyNode::Pattern(&keyword.pattern)), - ) - .collect(), - Pattern::MatchAs(node) => node - .pattern - .as_deref() - .map(PyNode::Pattern) - .into_iter() - .collect(), - Pattern::MatchOr(node) => node.patterns.iter().map(PyNode::Pattern).collect(), - Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => Vec::new(), - } -} - -fn type_param_children(parameter: &TypeParam) -> Vec> { - match parameter { - TypeParam::TypeVar(node) => optional(node.bound.as_deref()) - .chain(optional(node.default.as_deref())) - .collect(), - TypeParam::TypeVarTuple(node) => optional(node.default.as_deref()).collect(), - TypeParam::ParamSpec(node) => optional(node.default.as_deref()).collect(), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 965951e4aa..5fb7a0b6f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -13,7 +13,10 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const PYTHON_ANALYSIS_MODULE: &str = "sqlbuild.compiler.compile.classes.python_model_analysis"; +const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; @@ -46,18 +49,7 @@ fn analyze_native_model_sql<'py>( } let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; - let python: Bound<'py, PyAny> = { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("model_inputs", &model_inputs)?; - arguments.set_item("inference_profile", &profile)?; - arguments.set_item( - "complete_binding_schemas", - request.getattr("complete_binding_schemas")?, - )?; - py.import(PYTHON_ANALYSIS_MODULE)? - .getattr("PythonModelAnalysis")? - .call((), Some(&arguments))? - }; + let python: Vec> = model_inputs.try_iter()?.collect::>()?; let (rules, adapter_rules) = nullability_rules(&profile)?; let session_request: Bound<'py, PyAny> = session_request(&request, &python, &catalog, rules)?; let (cache, adapter): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match adapter_rules { @@ -84,7 +76,7 @@ fn analyze_native_model_sql<'py>( /// The session request row, in the order the native session reads it. fn session_request<'py>( request: &Bound<'py, PyAny>, - python: &Bound<'py, PyAny>, + python: &[Bound<'py, PyAny>], catalog: &Bound<'py, PyAny>, rules: Vec<(String, String)>, ) -> PyResult> { @@ -111,7 +103,7 @@ fn session_request<'py>( shape_rows(&request.getattr("complete_binding_schemas")?)?.into_bound_py_any(py)?, shape_rows(&catalog.getattr("schemas")?)?.into_bound_py_any(py)?, families.into_bound_py_any(py)?, - python.call_method0("model_rows")?, + model_rows(py, python)?, ]; Ok(PyTuple::new(py, items)?.into_any()) } @@ -151,7 +143,7 @@ fn analysis_store<'py>(request: &Bound<'py, PyAny>) -> PyResult( request: &Bound<'py, PyAny>, - (python, catalog): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (python, catalog): (&[Bound<'py, PyAny>], &Bound<'py, PyAny>), session: &Bound<'py, PyAny>, analyses_class: &Bound<'py, PyAny>, ) -> PyResult> { @@ -186,9 +178,8 @@ fn run_session<'py>( record_cache(python, session)?; let positions: Bound<'py, PyDict> = PyDict::new(py); let mut names: Vec> = Vec::new(); - for analysed in python.getattr("requests")?.try_iter()? { - let name: Bound<'py, PyAny> = analysed? - .getattr("model_input")? + for model_input in python { + let name: Bound<'py, PyAny> = model_input .getattr("model_file")? .getattr("file_path")? .getattr("stem")?; @@ -252,34 +243,113 @@ fn record_catalog_changes<'py>( Ok(()) } -fn record_cache(python: &Bound<'_, PyAny>, session: &Bound<'_, PyAny>) -> PyResult<()> { +fn record_cache(python: &[Bound<'_, PyAny>], session: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = session.py(); let stats: Bound<'_, PyAny> = session.getattr("cache_stats")?; if stats.is_none() { - python.call_method0("record_uncached")?; - return Ok(()); + return record_metrics(py, (0, 0, python.len())); } - let (hits, misses, _, failure): ( - Bound<'_, PyAny>, - Bound<'_, PyAny>, - Bound<'_, PyAny>, - Bound<'_, PyAny>, - ) = stats.extract()?; + let (hits, misses, _, failure): (usize, usize, usize, Bound<'_, PyAny>) = stats.extract()?; if !failure.is_none() { - log_debug(python.py(), STORE_FAILURE_MESSAGE, &failure)?; + log_debug(py, STORE_FAILURE_MESSAGE, &failure)?; } - if hits.add(&misses)?.eq(0)? { - python.call_method0("record_uncached")?; - return Ok(()); + if hits + misses == 0 { + return record_metrics(py, (0, 0, python.len())); } - let arguments: Bound<'_, PyDict> = PyDict::new(python.py()); - arguments.set_item("hits", hits)?; + record_metrics(py, (hits, misses, 0)) +} + +/// Record the analysis cache entries a native session read, missed and bypassed. +fn record_metrics(py: Python<'_>, (hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("batch_hits", 0)?; + arguments.set_item("entry_hits", hits)?; arguments.set_item("misses", misses)?; - python - .getattr("record_cached")? + arguments.set_item("bypasses", bypasses)?; + py.import(ANALYSIS_CACHE_MODULE)? + .getattr("record_analysis_cache_metrics")? .call((), Some(&arguments))?; Ok(()) } +/// Each model's native request row, in analysis order. +fn model_rows<'py>( + py: Python<'py>, + model_inputs: &[Bound<'py, PyAny>], +) -> PyResult> { + let cursor_sql: Bound<'py, PyAny> = py + .import(CURSOR_INTRINSICS_MODULE)? + .getattr("cursor_intrinsics_analysis_sql")?; + let planner = py.import(PLANNER_TYPES_MODULE)?; + let enforced: Bound<'py, PyAny> = planner.getattr("ContractPolicy")?.getattr("ENFORCED")?; + let snapshot: Bound<'py, PyAny> = planner + .getattr("MaterializationType")? + .getattr("SNAPSHOT")?; + let mut rows: Vec> = Vec::with_capacity(model_inputs.len()); + for model_input in model_inputs { + let values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; + let get = |key: &str| values.call_method1("get", (key,)); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", model_input.getattr("query_sql")?)?; + arguments.set_item("cursor_type", get("cursor_type")?)?; + let query_sql: Bound<'py, PyAny> = cursor_sql.call((), Some(&arguments))?; + let mut placeholders: Vec<(String, String)> = Vec::new(); + let raw: Bound<'py, PyAny> = get("placeholders")?; + if raw.is_instance_of::() { + for item in raw.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + placeholders.push((key.str()?.extract()?, value.str()?.extract()?)); + } + } + let mut references: Vec<(String, String)> = Vec::new(); + for row in model_input.getattr("reference_rows")?.try_iter()? { + let row: Bound<'py, PyAny> = row?; + references.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); + } + let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + let recovers: bool = get("contract")?.eq(&enforced)? + || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?); + let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? { + Some(( + or_default(&get("valid_from_column")?, "valid_from")?, + or_default(&get("valid_to_column")?, "valid_to")?, + )) + } else { + None + }; + let families = if entry.is_none() { + Vec::new() + } else { + family_rows(&entry.getattr("dynamic_columns")?)? + }; + let row: Vec> = vec![ + model_input + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")?, + query_sql, + placeholders.into_bound_py_any(py)?, + references.into_bound_py_any(py)?, + recovers.into_bound_py_any(py)?, + snapshot_columns.into_bound_py_any(py)?, + families.into_bound_py_any(py)?, + ]; + rows.push(PyTuple::new(py, row)?.into_any()); + } + Ok(PyList::new(py, rows)?.into_any()) +} + +/// `str(value or default)`. +fn or_default(value: &Bound<'_, PyAny>, default: &str) -> PyResult { + if value.is_truthy()? { + return value.str()?.extract(); + } + Ok(default.to_owned()) +} + fn log_debug(py: Python<'_>, message: &str, failure: &Bound<'_, PyAny>) -> PyResult<()> { let arguments: Bound<'_, PyDict> = PyDict::new(py); arguments.set_item( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 3a43e2bbde..6172668169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -12,8 +12,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -const NATIVE_DECLARATIONS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.assembly.native_declarations"; const IDENTIFIER_CASE_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._identifier_case"; const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; pub(crate) const NATIVE_STAGES_MODULE: &str = @@ -147,10 +145,9 @@ pub(crate) fn declared_names<'py>( let py: Python<'py> = inputs.py(); let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; let names: Bound<'py, PyAny> = - imported(py, NATIVE_DECLARATIONS_MODULE, "known_function_names")? - .call1((functions.clone(),))?; + imported(py, NATIVE_MODULE, "known_function_names")?.call1((functions.clone(),))?; let types: Bound<'py, PyAny> = call_with( - &imported(py, NATIVE_DECLARATIONS_MODULE, "known_declared_types")?, + &imported(py, NATIVE_MODULE, "known_declared_types")?, &[ ("functions", functions), ("column_types", column_types.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 582675569c..99da866a6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -8,13 +8,13 @@ use crate::bindings::_helpers::compile_driver::assembly_analysis::{ ANALYSIS_MODELS_MODULE, AnalysisContext, model_stem, selected_for_analysis, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; const NATIVE_MODULE: &str = "sqlbuild._native"; const SYNTAX_CHECKS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.syntax_checks"; const CURSOR_INTRINSICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; -const HAND_BUILT_MODULE: &str = "sqlbuild.compiler.compile._helpers.assembly.hand_built_models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -189,14 +189,11 @@ fn compiled_model<'py>( ), ], )?; - call_with( - &imported(py, HAND_BUILT_MODULE, "hand_built_compiled_model")?, - &[ - ("model_input", model_input.clone()), - ("deps", native_model.getattr("deps")?), - ("analysis", analysis), - ("macro_deps", macro_deps), - ], + hand_built_model( + model_input, + &native_model.getattr("deps")?, + &analysis, + ¯o_deps, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs new file mode 100644 index 0000000000..ad302535b7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -0,0 +1,365 @@ +//! Deferred target relation locations, and the warehouse relations they name. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; +use sqlbuild_model_config::templates::models::DeferredSegment; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PROJECT_VARS_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; +const PRESERVE_TARGET_VALUE: &str = "preserve"; +/// `sorted(DEFERRED_TARGET_CONTEXT_KEYS)`. +const DEFERRED_TARGET_CONTEXT_KEYS: [&str; 2] = ["database", "schema"]; + +/// The deferred target's naming and the adapter defaults its locations fall back to. +struct DeferredNaming<'py> { + target_config: Bound<'py, PyAny>, + effective_vars: Bound<'py, PyAny>, + default_schema: Bound<'py, PyAny>, + default_database: Bound<'py, PyAny>, + render_qualified_name: Bound<'py, PyAny>, +} + +/// Build physical locations for all models and seeds under a deferred target. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_deferred_locations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_deferred_locations", + options, + ( + &[ + "project", + "deferred_target_config", + "effective_vars", + "default_schema", + "default_database", + "render_qualified_name", + ], + &[], + ), + )?; + deferred_locations( + &arguments.value("project")?, + &DeferredNaming { + target_config: arguments.value("deferred_target_config")?, + effective_vars: arguments.value("effective_vars")?, + default_schema: arguments.value("default_schema")?, + default_database: arguments.value("default_database")?, + render_qualified_name: arguments.value("render_qualified_name")?, + }, + ) +} + +/// `build_deferred_locations` with its arguments. +fn deferred_locations<'py>( + project: &Bound<'py, PyAny>, + naming: &DeferredNaming<'py>, +) -> PyResult> { + let locations: Bound<'py, PyDict> = PyDict::new(project.py()); + for attribute in ["models", "seeds"] { + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + let location: Bound<'py, PyAny> = + deferred_location(&resource.getattr("destination")?, naming)?; + locations.set_item(resource.getattr("name")?, location)?; + } + } + Ok(locations) +} + +/// One relation location under the deferred target's naming rules. +fn deferred_location<'py>( + target: &Bound<'py, PyAny>, + naming: &DeferredNaming<'py>, +) -> PyResult> { + let py: Python<'py> = target.py(); + let logical_schema: Bound<'py, PyAny> = target.getattr("logical_schema")?; + let logical_database: Bound<'py, PyAny> = target.getattr("logical_database")?; + let mut schema: Bound<'py, PyAny> = target_field( + &naming.target_config.getattr("schema")?, + &logical_schema, + &naming.effective_vars, + )?; + let mut database: Bound<'py, PyAny> = target_field( + &naming.target_config.getattr("database")?, + &logical_database, + &naming.effective_vars, + )?; + if schema.is_none() { + schema = naming.default_schema.clone(); + } + if database.is_none() { + database = naming.default_database.clone(); + } + let name: Bound<'py, PyAny> = target.getattr("name")?; + let qualified_name: Bound<'py, PyAny> = call_with( + &naming.render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledRelationLocation")?, + &[ + ("database", database), + ("schema", schema), + ("name", name), + ("qualified_name", qualified_name), + ("logical_schema", logical_schema), + ("logical_database", logical_database), + ], + ) +} + +/// One target schema or database field resolved against the logical value. +fn target_field<'py>( + target_value: &Bound<'py, PyAny>, + logical_value: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_value.py(); + if target_value.is_none() || target_value.eq(PRESERVE_TARGET_VALUE)? { + return Ok(logical_value.clone()); + } + let template: String = target_value.extract()?; + let logical: String = if logical_value.is_none() { + String::new() + } else { + logical_value.extract()? + }; + let mut variables: Vec = Vec::new(); + for name in effective_vars.try_iter()? { + if let Ok(name) = name?.downcast::() { + variables.push(name.to_cow()?.into_owned()); + } + } + let context_keys: Vec = DEFERRED_TARGET_CONTEXT_KEYS + .iter() + .map(|key| (*key).to_owned()) + .collect(); + let render: Bound<'py, PyAny> = imported(py, PROJECT_VARS_MODULE, "render_project_var_text")?; + let mut resolved: String = String::new(); + for segment in deferred_target_segments(&template, &logical, &context_keys, &variables) { + match segment { + DeferredSegment::Text(text) => resolved.push_str(&text), + DeferredSegment::Variable(name) => { + let text: String = call_with( + &render, + &[ + ("value", effective_vars.get_item(name.as_str())?), + ( + "label", + PyString::new(py, &format!("deferred target variable '${{{name}}}'")) + .into_any(), + ), + ], + )? + .extract()?; + resolved.push_str(&text); + } + } + } + Ok(PyString::new(py, &resolved).into_any()) +} + +/// A `PlannerInputError` with `message` and the default code. +fn planner_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Validate and resolve the deferred target config. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_deferred_target_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_deferred_target_config", + options, + ( + &["discovered_inputs", "defer_to", "current_target_name"], + &[], + ), + )?; + deferred_target_config( + &arguments.value("discovered_inputs")?, + &arguments.value("defer_to")?, + &arguments.value("current_target_name")?, + ) +} + +/// `resolve_deferred_target_config` with its arguments. +fn deferred_target_config<'py>( + discovered: &Bound<'py, PyAny>, + defer_to: &Bound<'py, PyAny>, + current_target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let targets: Bound<'py, PyAny> = discovered.getattr("project_config")?.getattr("targets")?; + let name: String = defer_to.str()?.to_cow()?.into_owned(); + if !targets.contains(defer_to)? { + return Err(planner_error( + py, + format!("Unknown deferred target '{name}'"), + )); + } + if defer_to.eq(current_target_name)? { + return Err(planner_error( + py, + format!("Cannot defer to the current target '{name}'"), + )); + } + targets.get_item(defer_to) +} + +/// Gather existing relations from the deferred target's relation-location schemas. +#[pyfunction] +#[pyo3(signature = (**options))] +fn gather_deferred_relations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "gather_deferred_relations", + options, + (&["adapter", "connection", "deferred_locations"], &[]), + )?; + deferred_relations( + &arguments.value("adapter")?, + &arguments.value("connection")?, + &arguments.value("deferred_locations")?, + ) +} + +/// `gather_deferred_relations` with its arguments. +fn deferred_relations<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + locations: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let mut schemas: Vec = Vec::new(); + let mut database: Bound<'py, PyAny> = py.None().into_bound(py); + for location in locations.call_method0("values")?.try_iter()? { + let location: Bound<'py, PyAny> = location?; + let schema: Bound<'py, PyAny> = location.getattr("schema")?; + if !schema.is_none() { + let schema: String = schema.extract()?; + if !schemas.contains(&schema) { + schemas.push(schema); + } + } + let location_database: Bound<'py, PyAny> = location.getattr("database")?; + if !location_database.is_none() && database.is_none() { + database = location_database; + } + } + let relations_by_name: Bound<'py, PyDict> = PyDict::new(py); + if schemas.is_empty() { + return Ok(relations_by_name); + } + schemas.sort(); + let relations: Bound<'py, PyAny> = call_with( + &adapter.getattr("list_relations")?, + &[ + ("connection", connection.clone()), + ("database", database), + ("schemas", PyTuple::new(py, schemas)?.into_any()), + ], + )?; + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + relations_by_name.set_item(relation.getattr("name")?, relation)?; + } + Ok(relations_by_name) +} + +/// Resolve native target locations and warehouse relations for model deferral. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_model_deferral_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_model_deferral_inputs", + options, + ( + &[ + "project", + "discovered_inputs", + "adapter", + "connection", + "defer_to", + ], + &[], + ), + )?; + let inputs: Bound<'py, PyAny> = imported(py, PLANNER_MODELS_MODULE, "DeferralInputs")?; + let defer_to: Bound<'py, PyAny> = arguments.value("defer_to")?; + if defer_to.is_none() { + return inputs.call0(); + } + let project: Bound<'py, PyAny> = arguments.value("project")?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let target_config: Bound<'py, PyAny> = deferred_target_config( + &arguments.value("discovered_inputs")?, + &defer_to, + &project.getattr("effective_target_name")?, + )?; + let locations: Bound<'py, PyDict> = deferred_locations( + &project, + &DeferredNaming { + target_config, + effective_vars: project.getattr("effective_vars")?, + default_schema: adapter.call_method0("default_schema")?, + default_database: adapter.call_method0("default_database")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + }, + )?; + let relations: Bound<'py, PyDict> = deferred_relations( + &adapter, + &arguments.value("connection")?, + locations.as_any(), + )?; + call_with( + &inputs, + &[ + ("deferred_locations", locations.into_any()), + ("deferred_relations", relations.into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_deferred_locations, module)?)?; + module.add_function(wrap_pyfunction!(resolve_deferred_target_config, module)?)?; + module.add_function(wrap_pyfunction!(gather_deferred_relations, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_deferral_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs new file mode 100644 index 0000000000..e2d3a56ffa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -0,0 +1,240 @@ +//! The diff pipeline: both environments compiled, then diff selectors resolved to models. + +use std::collections::HashSet; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::diff_selection::errors::SelectorError; +use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_diff_selection; +use sqlbuild_analysis::diff_selection::models::DiffGraph; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LINEAGE_VIEWS_MODULE: &str = "sqlbuild.compiler.graph.main.project_lineage_views"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; + +/// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. +struct KeyIndexes<'py> { + indexes: Bound<'py, PyDict>, +} + +impl<'py> KeyIndexes<'py> { + fn index(&self, key: &Bound<'py, PyAny>) -> PyResult { + if let Some(index) = self.indexes.get_item(key)? { + return index.extract(); + } + let index: usize = self.indexes.len(); + self.indexes.set_item(key, index)?; + Ok(index) + } + + fn indexes(&self, keys: &Bound<'py, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| self.index(&key?)).collect() + } +} + +/// `edges` as one list of neighbour indexes per resource index. +fn adjacency<'py>( + edges: &Bound<'py, PyAny>, + keys: &KeyIndexes<'py>, +) -> PyResult)>> { + let mut rows: Vec<(usize, Vec)> = Vec::new(); + for item in edges.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + rows.push(( + keys.index(&item.get_item(0)?)?, + keys.indexes(&item.get_item(1)?)?, + )); + } + Ok(rows) +} + +/// The project's names, lineage edges and selector indexes over resource indexes. +fn diff_graph<'py>(project: &Bound<'py, PyAny>, keys: &KeyIndexes<'py>) -> PyResult { + let py: Python<'py> = project.py(); + let lineage: Bound<'py, PyAny> = + imported(py, LINEAGE_VIEWS_MODULE, "project_lineage_views")?.call1((project,))?; + let mut graph: DiffGraph = DiffGraph::default(); + for attribute in ["models", "sources", "seeds"] { + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + let name: String = resource.getattr("name")?.extract()?; + graph + .names + .push((name, keys.index(&resource.getattr("key")?)?)); + } + } + let upstream: Vec<(usize, Vec)> = adjacency(&lineage.getattr("upstream_deps")?, keys)?; + let downstream: Vec<(usize, Vec)> = + adjacency(&lineage.getattr("downstream_deps")?, keys)?; + for item in lineage + .getattr("tag_index")? + .call_method0("items")? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + graph.tags.push(( + item.get_item(0)?.extract()?, + keys.indexes(&item.get_item(1)?)?, + )); + } + for item in lineage + .getattr("path_index")? + .call_method0("items")? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + graph.paths.push(( + keys.index(&item.get_item(0)?)?, + item.get_item(1)?.extract()?, + )); + } + let size: usize = keys.indexes.len(); + graph.upstream = vec![Vec::new(); size]; + graph.downstream = vec![Vec::new(); size]; + for (key, neighbours) in upstream { + graph.upstream[key] = neighbours; + } + for (key, neighbours) in downstream { + graph.downstream[key] = neighbours; + } + Ok(graph) +} + +/// The `PlannerInputError` a rejected selector raises. +fn selector_error(py: Python<'_>, error: SelectorError) -> PyErr { + let raised: PyResult> = (|| { + imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")?.call( + (error.message,), + Some(&keywords( + py, + &[("code", PyString::new(py, error.code).into_any())], + )?), + ) + })(); + match raised { + Ok(raised) => PyErr::from_value(raised), + Err(raised) => raised, + } +} + +/// Resolve diff selectors to model names only. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_diff_model_names<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_diff_model_names", + options, + (&["project", "select", "exclude"], &[]), + )?; + diff_model_names( + &arguments.value("project")?, + &arguments.value("select")?, + &arguments.value("exclude")?, + ) +} + +/// `resolve_diff_model_names` with its arguments. +fn diff_model_names<'py>( + project: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let keys: KeyIndexes<'py> = KeyIndexes { + indexes: PyDict::new(py), + }; + let graph: DiffGraph = diff_graph(project, &keys)?; + let select: Vec = select.extract()?; + let exclude: Vec = exclude.extract()?; + let selected: HashSet = resolve_diff_selection(&graph, (&select, &exclude)) + .map_err(|error| selector_error(py, error))?; + let model_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let mut names: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let key: Bound<'py, PyAny> = model.getattr("key")?; + let chosen: bool = match keys.indexes.get_item(&key)? { + Some(index) => selected.contains(&index.extract::()?), + None => false, + }; + if chosen && key.getattr("resource_type")?.eq(&model_type)? { + names.push(model.getattr("name")?); + } + } + PyTuple::new(py, names) +} + +/// Compile both environments and resolve selected diffable model names. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_diff_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_diff_pipeline", + options, + ( + &[ + "discovered_inputs", + "adapter", + "from_target", + "to_target", + "resolved_connection", + "no_sql_validation", + "select", + "exclude", + ], + &["cli_vars", "external_sql_reference_resolver"], + ), + )?; + let build: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "build_compiled_project")?; + let compile = |target: &str| -> PyResult> { + call_with( + &build, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("selected_target", arguments.value(target)?), + ( + "resolved_connection", + arguments.value("resolved_connection")?, + ), + ("no_sql_validation", arguments.value("no_sql_validation")?), + ("cli_vars", arguments.value("cli_vars")?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ], + ) + }; + let left: Bound<'py, PyAny> = compile("from_target")?; + let right: Bound<'py, PyAny> = compile("to_target")?; + let names: Bound<'py, PyTuple> = diff_model_names( + &right, + &arguments.value("select")?, + &arguments.value("exclude")?, + )?; + PyTuple::new(py, [left, right, names.into_any()]) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_diff_model_names, module)?)?; + module.add_function(wrap_pyfunction!(run_diff_pipeline, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 94aa6db407..82c8c76ae2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::discovery::yaml_records::{ discover_schema_files, discover_source_files, }; -const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; +const FRONTIER_MODULE: &str = "sqlbuild._native"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const PROJECT_YAML_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.project"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 5641901225..17e7d4196c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -5,10 +5,17 @@ pub(crate) mod assembly_checks; pub(crate) mod assembly_models; pub(crate) mod assembly_project; pub(crate) mod command; +pub(crate) mod deferral; +pub(crate) mod diff; pub(crate) mod discovery; pub(crate) mod inputs; pub(crate) mod phases; pub(crate) mod pipeline; +pub(crate) mod pipeline_entries; pub(crate) mod project; pub(crate) mod python_calls; +pub(crate) mod python_plan_entries; pub(crate) mod registration; +pub(crate) mod relation_targets; +pub(crate) mod target_defaults; +pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 482282183c..ffb6416eb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -5,6 +5,9 @@ use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::pipeline_entries::{ + compile_analysis_selection, project_graph, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -16,7 +19,6 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; -const GRAPH_HELPERS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.graph"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; @@ -99,17 +101,7 @@ fn compiled_phase<'py>( progress(on_progress, "Compiling project...")?; let started: f64 = monotonic(py)?; let compiled: PyResult> = (|| { - let selection: Bound<'py, PyAny> = call_with( - &imported( - py, - ANALYSIS_SELECTION_MODULE, - "resolve_compile_analysis_selection", - )?, - &[ - ("options", options.clone()), - ("discovered_inputs", discovered.clone()), - ], - )?; + let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; call_with( &imported(py, NATIVE_MODULE, "build_compiled_project")?, &[ @@ -243,8 +235,7 @@ fn run_compile_pipeline<'py>( let compiled: Bound<'py, PyAny> = compiled_phase((&discovered, &adapter), &options, &on_progress)?; let project: Bound<'py, PyAny> = compiled.getattr("project")?; - let graph: Bound<'py, PyAny> = - imported(py, GRAPH_HELPERS_MODULE, "build_project_graph_impl")?.call1((&project,))?; + let graph: Bound<'py, PyAny> = project_graph(&project)?; run_rules_gate( (&discovered, &adapter), (&options, &on_progress), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs new file mode 100644 index 0000000000..af827ae3a3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -0,0 +1,217 @@ +//! Small pipeline entry points: analysis selection, project graphs and pipeline results. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFloat}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// Discovered Python node functions whose selectors need the Python graph. +const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; +/// Plan entry lists that are runtime work. +const PLAN_WORK_ENTRIES: [&str; 6] = [ + "model_entries", + "seed_entries", + "source_load_entries", + "function_entries", + "audit_entries", + "test_entries", +]; + +/// Limit analysis when selectors cannot resolve through Python graph nodes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_compile_analysis_selection<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_compile_analysis_selection", + options, + (&["options", "discovered_inputs"], &[]), + )?; + compile_analysis_selection( + &arguments.value("options")?, + &arguments.value("discovered_inputs")?, + ) +} + +/// `resolve_compile_analysis_selection` with its arguments. +pub(crate) fn compile_analysis_selection<'py>( + options: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = options.py(); + let mut has_python_nodes: bool = false; + for name in PYTHON_NODE_FUNCTIONS { + if discovered.getattr(name)?.is_truthy()? { + has_python_nodes = true; + break; + } + } + let selection: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?; + let no_cache: Bound<'py, PyAny> = options.getattr("no_cache")?; + if (options + .getattr("resolve_python_run_selectors")? + .is_truthy()? + && has_python_nodes) + || options + .getattr("sql_test_selection")? + .getattr("names")? + .is_truthy()? + { + return call_with(&selection, &[("no_cache", no_cache)]); + } + call_with( + &selection, + &[ + ("select", options.getattr("select")?), + ("exclude", options.getattr("exclude")?), + ("auto_load_sources", options.getattr("auto_load_sources")?), + ("no_cache", no_cache), + ], + ) +} + +/// Build a static dependency graph for an already-compiled project. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_project_graph_from_compiled_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_graph_from_compiled_project", + options, + (&["project"], &[]), + )?; + project_graph(&arguments.value("project")?) +} + +/// The project's natively held graph; dict views are built only when a boundary reads them. +pub(crate) fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(project.py(), PIPELINE_MODELS_MODULE, "ProjectGraph")?, + &[ + ("project", project.clone()), + ("native", project.getattr("lineage_graph")?), + ], + ) +} + +/// Return whether a planned build has runtime work to execute. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_has_executable_work<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "plan_has_executable_work", + options, + (&["plan"], &["python_plan_entries"]), + )?; + let plan: Bound<'py, PyAny> = arguments.value("plan")?; + for name in PLAN_WORK_ENTRIES { + if plan.getattr(name)?.is_truthy()? { + return Ok(true); + } + } + if !arguments.passed("python_plan_entries")? { + return Ok(false); + } + arguments.value("python_plan_entries")?.is_truthy() +} + +/// Wrap a focused command projection in the established pipeline envelope. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_static_pipeline_result<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_static_pipeline_result", + options, + (&["context", "plan_output"], &[]), + )?; + let context: Bound<'py, PyAny> = arguments.value("context")?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineResult")?, + &[ + ("project", context.getattr("project")?), + ("plan_output", arguments.value("plan_output")?), + ("compile_seconds", context.getattr("compile_seconds")?), + ("planning_seconds", PyFloat::new(py, 0.0).into_any()), + ], + ) +} + +/// Compile a project for consumers that construct their own focused plans. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_compile_only_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_compile_only_pipeline", + options, + ( + &["discovered_inputs", "adapter", "options"], + &["on_progress"], + ), + )?; + let compiled: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "compile_project_phase")?, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("options", arguments.value("options")?), + ("on_progress", arguments.value("on_progress")?), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineResult")?, + &[ + ("project", compiled.getattr("project")?), + ( + "plan_output", + imported(py, PLANNER_MODELS_MODULE, "PlanOutput")?.call0()?, + ), + ("compile_seconds", compiled.getattr("compile_seconds")?), + ("planning_seconds", PyFloat::new(py, 0.0).into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + resolve_compile_analysis_selection, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_project_graph_from_compiled_project, + module + )?)?; + module.add_function(wrap_pyfunction!(plan_has_executable_work, module)?)?; + module.add_function(wrap_pyfunction!(build_static_pipeline_result, module)?)?; + module.add_function(wrap_pyfunction!(run_compile_only_pipeline, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 4ed96a7291..cdd8a42d09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -5,11 +5,19 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::compile_driver::target_defaults::{ + TargetDefaults, apply_target_defaults, +}; +use crate::bindings::_helpers::compile_driver::targets::{ + managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, + project_targets, +}; -const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; +const FRONTIER_MODULE: &str = "sqlbuild._native"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; @@ -19,9 +27,6 @@ const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_confl const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; -const TARGET_DEFAULTS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.target_defaults"; -const TARGET_VALIDATION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.target_validation"; -const GRAPH_HELPERS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.graph"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const COLLECTION_RENDERING_MODULE: &str = @@ -143,33 +148,8 @@ fn python_compiled_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - call_with( - &imported( - py, - TARGET_VALIDATION_MODULE, - "validate_named_target_schema_strategy", - )?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", request.selected_target.bind(py).clone()), - ], - )?; - call_with( - &imported( - py, - TARGET_VALIDATION_MODULE, - "validate_managed_loader_target_isolation", - )?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ( - "resolved_connection", - request.resolved_connection.bind(py).clone(), - ), - ], - )?; + named_target_schema_strategy(discovered, adapter, request.selected_target.bind(py))?; + managed_loader_target_isolation(discovered, adapter, request.resolved_connection.bind(py))?; let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); let no_cache: bool = !selection.is_none() && selection.getattr("no_cache")?.is_truthy()?; let timing: Bound<'py, PyAny> = @@ -227,39 +207,21 @@ fn targeted_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - call_with( - &imported( - py, - TARGET_VALIDATION_MODULE, - "validate_managed_write_schemas", - )?, - &[("adapter", adapter.clone()), ("project", project.clone())], - )?; + managed_write_schemas(adapter, project)?; let effective_schema: Bound<'py, PyAny> = project.getattr("effective_target_schema")?; let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { effective_schema } else { adapter.call_method0("default_schema")? }; - let project: Bound<'py, PyAny> = call_with( - &imported(py, TARGET_DEFAULTS_MODULE, "apply_target_defaults")?, - &[ - ("project", project.clone()), - ("default_schema", default_schema), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ( - "python_functions_inherit_default_namespace", - adapter.call_method0("python_functions_inherit_default_namespace")?, - ), - ], - )?; + let defaults: TargetDefaults<'py> = TargetDefaults { + default_schema, + default_database: adapter.call_method0("default_database")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + python_functions_inherit_default_namespace: adapter + .call_method0("python_functions_inherit_default_namespace")?, + }; + let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; let adapter_name: Bound<'py, PyAny> = call_with( @@ -269,10 +231,7 @@ fn targeted_project<'py>( ("local_config", local_config.clone()), ], )?; - call_with( - &imported(py, TARGET_VALIDATION_MODULE, "validate_project_targets")?, - &[("adapter_name", adapter_name), ("project", project.clone())], - )?; + project_targets(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = imported( py, DISCOVERY_CONSTANTS_MODULE, @@ -444,7 +403,7 @@ fn selected_graph<'py>( let elapsed: f64 = monotonic(py)? - started; on_progress.call1((format!("Compiled project. ({elapsed:.2}s)"),))?; } - imported(py, GRAPH_HELPERS_MODULE, "build_project_graph_impl")?.call1((project,)) + project_graph(&project) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs new file mode 100644 index 0000000000..083dba3be3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -0,0 +1,395 @@ +//! Display-ready Python plan entries and Python-aware selection warnings of a plan. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::python_plan::main::ordered_python_names::ordered_python_names; +use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main.run_lifecycle"; +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; +const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; +const SKIPPED_UPSTREAM_CODE: &str = "P501"; + +/// The Python node kinds and plan phases entries are built from. +struct NodeKinds<'py> { + task: Bound<'py, PyAny>, + asset: Bound<'py, PyAny>, +} + +impl<'py> NodeKinds<'py> { + fn load(py: Python<'py>) -> PyResult { + let kinds: Bound<'py, PyAny> = imported(py, PYTHON_NODE_TYPES_MODULE, "PythonNodeKind")?; + Ok(Self { + task: kinds.getattr("TASK")?, + asset: kinds.getattr("ASSET")?, + }) + } + + /// Whether `kind` is a task or an asset. + fn runs(&self, kind: &Bound<'py, PyAny>) -> PyResult { + Ok(kind.eq(&self.task)? || kind.eq(&self.asset)?) + } +} + +/// `python_graph.upstream_deps` as names. +fn upstream_names(graph: &Bound<'_, PyAny>) -> PyResult>> { + let mut upstream: HashMap> = HashMap::new(); + for item in graph + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let item: Bound<'_, PyAny> = item?; + let names: Vec = item + .get_item(1)? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + upstream.insert(item.get_item(0)?.extract()?, names); + } + Ok(upstream) +} + +/// The discovered Python node graph. +fn node_graph<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported( + discovered.py(), + NODE_GRAPH_MODULE, + "build_discovered_python_node_graph", + )?, + &[("discovered_inputs", discovered.clone())], + ) +} + +/// Return sources declared by the selected Python nodes and their Python upstreams. +#[pyfunction] +#[pyo3(signature = (**options))] +fn python_read_source_names_for<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "python_read_source_names_for", + options, + (&["discovered_inputs", "python_node_names"], &[]), + )?; + let frozenset: Bound<'py, PyAny> = imported(py, "builtins", "frozenset")?; + let selected: Bound<'py, PyAny> = arguments.value("python_node_names")?; + if !selected.is_truthy()? { + return frozenset.call0(); + } + let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; + let source: Bound<'py, PyAny> = + imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?.getattr("SOURCE")?; + let mut names: Vec> = Vec::new(); + for reference in call_with( + &graph.getattr("selected_sql_refs")?, + &[("selected_names", selected)], + )? + .try_iter()? + { + let reference: Bound<'py, PyAny> = reference?; + if reference.getattr("kind")?.eq(&source)? { + names.push(reference.getattr("name")?); + } + } + frozenset.call1((PyTuple::new(py, names)?,)) +} + +/// Attach python-aware selection warnings and plan entries to the plan output. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_python_run_plan_outputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_python_run_plan_outputs", + options, + ( + &[ + "discovered_inputs", + "plan_output", + "run_selection", + "selected_python_node_names", + ], + &[], + ), + )?; + let outputs: Bound<'py, PyAny> = imported(py, PIPELINE_MODELS_MODULE, "PythonRunPlanOutputs")?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + let run_selection: Bound<'py, PyAny> = arguments.value("run_selection")?; + let selected: Bound<'py, PyAny> = arguments.value("selected_python_node_names")?; + if run_selection.is_none() { + return call_with( + &outputs, + &[ + ("plan_output", plan_output), + ("python_plan_entries", PyTuple::empty(py).into_any()), + ("selected_python_node_names", selected), + ], + ); + } + let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; + let kinds: NodeKinds<'py> = NodeKinds::load(py)?; + let mut warnings: Vec> = plan_output + .getattr("warnings")? + .try_iter()? + .collect::>()?; + warnings.extend(skipped_ingress_warnings( + &plan_output, + &run_selection, + (&graph, &kinds), + )?); + let plan_output: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?.call( + (plan_output,), + Some(&keywords( + py, + &[("warnings", PyTuple::new(py, warnings)?.into_any())], + )?), + )?; + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PYTHON_NODE_MODELS_MODULE, "PythonSqlRunSelection")?, + &[ + ("sql_keys", plan_output.getattr("selected_keys")?), + ("python_node_names", selected.clone()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &[("selection", selection), ("python_graph", graph.clone())], + )?; + let entries: Bound<'py, PyTuple> = plan_entries( + &lifecycle, + (&graph, &kinds), + &plan_output.getattr("python_identity_fingerprints")?, + )?; + call_with( + &outputs, + &[ + ("plan_output", plan_output), + ("python_plan_entries", entries.into_any()), + ("selected_python_node_names", selected), + ], + ) +} + +/// Return task/asset plan entries ordered by lifecycle dependency readiness. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_python_plan_entries<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_python_plan_entries", + options, + ( + &["lifecycle_plan", "python_graph"], + &["previous_identities"], + ), + )?; + let previous: Bound<'py, PyAny> = arguments.value("previous_identities")?; + let previous: Bound<'py, PyAny> = if previous.is_none() { + PyDict::new(py).into_any() + } else { + previous + }; + plan_entries( + &arguments.value("lifecycle_plan")?, + (&arguments.value("python_graph")?, &NodeKinds::load(py)?), + &previous, + ) +} + +/// The entries of the lifecycle's ingress and then read-side tasks and assets. +fn plan_entries<'py>( + lifecycle: &Bound<'py, PyAny>, + (graph, kinds): (&Bound<'py, PyAny>, &NodeKinds<'py>), + previous_identities: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = lifecycle.py(); + let phases: Bound<'py, PyAny> = imported(py, PYTHON_NODE_TYPES_MODULE, "PythonRunPhase")?; + let entry_class: Bound<'py, PyAny> = imported(py, PIPELINE_MODELS_MODULE, "PythonPlanEntry")?; + let upstream: HashMap> = upstream_names(graph)?; + let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; + let mut entries: Vec> = Vec::new(); + for (attribute, phase) in [ + ("ingress_python_node_names", "PRE_SQL_INGRESS"), + ("read_side_python_node_names", "READ_SIDE"), + ] { + let selected: Vec = lifecycle + .getattr(attribute)? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + for name in ordered_python_names(&selected, &upstream) { + let node: Bound<'py, PyAny> = nodes.get_item(name.as_str())?; + let kind: Bound<'py, PyAny> = node.getattr("kind")?; + if !kinds.runs(&kind)? { + continue; + } + let identity: Bound<'py, PyAny> = node.getattr("identity")?; + let previous: Bound<'py, PyAny> = previous_identity(&identity, previous_identities)?; + let attribute_or_none = |value: &Bound<'py, PyAny>, name: &str| { + if value.is_none() { + Ok(py.None().into_bound(py)) + } else { + value.getattr(name) + } + }; + entries.push(call_with( + &entry_class, + &[ + ("name", node.getattr("name")?), + ("kind", kind), + ("phase", phases.getattr(phase)?), + ( + "identity_status", + identity_status(&identity, previous_identities)?, + ), + ( + "current_definition_json", + attribute_or_none(&identity, "definition_json")?, + ), + ( + "previous_definition_json", + attribute_or_none(&previous, "definition")?, + ), + ( + "current_metadata_json", + attribute_or_none(&identity, "metadata_json")?, + ), + ( + "previous_metadata_json", + attribute_or_none(&previous, "metadata_json")?, + ), + ("provider_usages", node.getattr("provider_usages")?), + ], + )?); + } + } + PyTuple::new(py, entries) +} + +/// `_previous_identity`: the stored fingerprint of a node's identity, or `None`. +fn previous_identity<'py>( + identity: &Bound<'py, PyAny>, + previous_identities: &Bound<'py, PyAny>, +) -> PyResult> { + if identity.is_none() { + return Ok(identity.clone()); + } + let key: Bound<'py, PyTuple> = PyTuple::new( + identity.py(), + [ + identity.getattr("node_type")?, + identity.getattr("node_name")?, + ], + )?; + previous_identities.call_method1("get", (key,)) +} + +/// `_identity_status`: whether a node's identity is unknown, new, unchanged or changed. +fn identity_status<'py>( + identity: &Bound<'py, PyAny>, + previous_identities: &Bound<'py, PyAny>, +) -> PyResult> { + let statuses: Bound<'py, PyAny> = imported( + identity.py(), + PYTHON_NODE_TYPES_MODULE, + "PythonIdentityStatus", + )?; + if identity.is_none() { + return statuses.getattr("UNKNOWN"); + } + let previous: Bound<'py, PyAny> = previous_identity(identity, previous_identities)?; + if previous.is_none() { + return statuses.getattr("NEW"); + } + if previous + .getattr("version_hash")? + .eq(identity.getattr("version_hash")?)? + { + return statuses.getattr("UNCHANGED"); + } + statuses.getattr("CHANGED") +} + +/// Warnings for skipped task/asset deps of planned source loaders. +fn skipped_ingress_warnings<'py>( + plan_output: &Bound<'py, PyAny>, + run_selection: &Bound<'py, PyAny>, + (graph, kinds): (&Bound<'py, PyAny>, &NodeKinds<'py>), +) -> PyResult>> { + let py: Python<'py> = plan_output.py(); + let mut loaders: Vec = Vec::new(); + for entry in plan_output.getattr("source_load_entries")?.try_iter()? { + let loader: String = entry?.getattr("loader")?.extract()?; + if !loaders.contains(&loader) { + loaders.push(loader); + } + } + loaders.sort(); + let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; + let selected: Bound<'py, PyAny> = run_selection.getattr("python_node_names")?; + let upstream: HashMap> = upstream_names(graph)?; + let warning_class: Bound<'py, PyAny> = imported(py, PLANNER_MODELS_MODULE, "PlanWarning")?; + let severity: Bound<'py, PyAny> = + imported(py, PLANNER_TYPES_MODULE, "WarningSeverity")?.getattr("WARNING")?; + let mut warnings: Vec> = Vec::new(); + for loader in loaders { + if !nodes.contains(loader.as_str())? { + continue; + } + for upstream_name in upstream_closure(&loader, &upstream) { + if selected.contains(upstream_name.as_str())? { + continue; + } + let kind: Bound<'py, PyAny> = + nodes.get_item(upstream_name.as_str())?.getattr("kind")?; + if !kinds.runs(&kind)? { + continue; + } + let message: String = format!( + "Source loader '{loader}' has unselected upstream {} '{upstream_name}'. SQLBuild \ + will not run it and cannot verify side effects or payload availability; use \ + +source:{loader} to refresh upstream ingress dependencies.", + kind.getattr("value")?.str()? + ); + warnings.push(call_with( + &warning_class, + &[ + ("model_name", py.None().into_bound(py)), + ("severity", severity.clone()), + ("message", PyString::new(py, &message).into_any()), + ("code", PyString::new(py, SKIPPED_UPSTREAM_CODE).into_any()), + ], + )?); + } + } + Ok(warnings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(python_read_source_names_for, module)?)?; + module.add_function(wrap_pyfunction!(build_python_run_plan_outputs, module)?)?; + module.add_function(wrap_pyfunction!(build_python_plan_entries, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 46cbaf85e4..ae7039d6f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ - analysis, assembly, command, discovery, inputs, pipeline, project, + analysis, assembly, command, deferral, diff, discovery, inputs, pipeline, pipeline_entries, + project, python_plan_entries, relation_targets, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -13,5 +14,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project::register(module)?; inputs::register(module)?; discovery::register(module)?; - pipeline::register(module) + pipeline::register(module)?; + pipeline_entries::register(module)?; + deferral::register(module)?; + diff::register(module)?; + python_plan_entries::register(module)?; + relation_targets::register(module)?; + targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs new file mode 100644 index 0000000000..c2d3cf40b0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -0,0 +1,238 @@ +//! Runtime SQL relation maps for Python node contexts and hard-coded name guards. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; +const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; +const QUALIFIED_NAME_MODULE: &str = + "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; +const SOURCE_RELATION_MODULE: &str = "sqlbuild.compiler.references.main.render_source_relation"; +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +/// The `SqlResourceRefKind` members a relation map distinguishes. +struct RefKinds<'py> { + model: Bound<'py, PyAny>, + seed: Bound<'py, PyAny>, + source: Bound<'py, PyAny>, +} + +impl<'py> RefKinds<'py> { + fn load(py: Python<'py>) -> PyResult { + let kinds: Bound<'py, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; + Ok(Self { + model: kinds.getattr("MODEL")?, + seed: kinds.getattr("SEED")?, + source: kinds.getattr("SOURCE")?, + }) + } +} + +/// A `SharedInputError` with `message`. +fn shared_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Return adapter-qualified runtime relations required by selected Python nodes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_python_relation_targets<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_python_relation_targets", + options, + (&["adapter", "project", "plan_output"], &["required_refs"]), + )?; + let kinds: RefKinds<'py> = RefKinds::load(py)?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let project: Bound<'py, PyAny> = arguments.value("project")?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + let required: Bound<'py, PyAny> = arguments.value("required_refs")?; + let refs: Bound<'py, PyAny> = if required.is_none() { + planned_relation_refs(&plan_output, &kinds)? + } else { + required + }; + let targets: Bound<'py, PyDict> = PyDict::new(py); + for reference in refs.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + let relation: Bound<'py, PyAny> = + resolved_relation((&adapter, &project, &plan_output), &reference, &kinds)?; + targets.set_item(reference, relation)?; + } + Ok(targets) +} + +/// Return every planned project relation for hard-coded name warnings, or None when off. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_project_relation_targets<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_relation_targets", + options, + (&["adapter", "plan_output"], &[]), + )?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + if !plan_output + .getattr("enforce_explicit_references")? + .is_truthy()? + { + return Ok(None); + } + let kinds: RefKinds<'py> = RefKinds::load(py)?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let targets: Bound<'py, PyDict> = PyDict::new(py); + for reference in planned_relation_refs(&plan_output, &kinds)?.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + let relation: Bound<'py, PyAny> = + planned_relation(&adapter, &plan_output, &reference, &kinds)?; + if !relation.is_none() { + targets.set_item(reference, relation)?; + } + } + Ok(Some(targets)) +} + +/// The model, seed and Python source refs of a plan, as the frozenset Python builds. +fn planned_relation_refs<'py>( + plan_output: &Bound<'py, PyAny>, + kinds: &RefKinds<'py>, +) -> PyResult> { + let py: Python<'py> = plan_output.py(); + let reference: Bound<'py, PyAny> = imported(py, REF_MODELS_MODULE, "SqlResourceRef")?; + let refs: Bound<'py, PySet> = PySet::empty(py)?; + for (attribute, kind) in [ + ("model_locations", &kinds.model), + ("seed_locations", &kinds.seed), + ("python_source_entries", &kinds.source), + ] { + for name in plan_output.getattr(attribute)?.try_iter()? { + refs.add(call_with( + &reference, + &[("kind", kind.clone()), ("name", name?)], + )?)?; + } + } + imported(py, "builtins", "frozenset")?.call1((refs,)) +} + +/// `_resolve_relation`: the planned relation, or the compiled location's qualified name. +fn resolved_relation<'py>( + (adapter, project, plan_output): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + reference: &Bound<'py, PyAny>, + kinds: &RefKinds<'py>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let planned: Bound<'py, PyAny> = planned_relation(adapter, plan_output, reference, kinds)?; + if !planned.is_none() { + return Ok(planned); + } + let kind: Bound<'py, PyAny> = reference.getattr("kind")?; + let name: Bound<'py, PyAny> = reference.getattr("name")?; + if kind.eq(&kinds.source)? { + return Err(shared_error( + py, + format!( + "Python node source '{}' is missing from the resolved plan source map", + name.str()? + ), + )); + } + let resources: Bound<'py, PyAny> = project.getattr(if kind.eq(&kinds.model)? { + "models" + } else { + "seeds" + })?; + let mut location: Option> = None; + for resource in resources.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + if resource.getattr("name")?.eq(&name)? { + location = Some(resource.getattr("destination")?); + break; + } + } + let Some(location) = location else { + return Err(shared_error( + py, + format!( + "Python node references unknown {} '{}'", + kind.getattr("value")?.str()?, + name.str()? + ), + )); + }; + qualified_name(adapter, &location) +} + +/// `resolve_relation_location_qualified_name(adapter=adapter, location=location)`. +fn qualified_name<'py>( + adapter: &Bound<'py, PyAny>, + location: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + adapter.py(), + QUALIFIED_NAME_MODULE, + "resolve_relation_location_qualified_name", + )?, + &[("adapter", adapter.clone()), ("location", location.clone())], + ) +} + +/// `_planned_relation`: the relation the plan resolved for `reference`, or `None`. +fn planned_relation<'py>( + adapter: &Bound<'py, PyAny>, + plan_output: &Bound<'py, PyAny>, + reference: &Bound<'py, PyAny>, + kinds: &RefKinds<'py>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let kind: Bound<'py, PyAny> = reference.getattr("kind")?; + let name: Bound<'py, PyAny> = reference.getattr("name")?; + if kind.eq(&kinds.source)? { + let source: Bound<'py, PyAny> = plan_output + .getattr("python_source_entries")? + .call_method1("get", (name,))?; + if source.is_none() { + return Ok(source); + } + return call_with( + &imported(py, SOURCE_RELATION_MODULE, "render_source_relation")?, + &[("entry", source), ("adapter", adapter.clone())], + ); + } + let locations: Bound<'py, PyAny> = plan_output.getattr(if kind.eq(&kinds.model)? { + "model_locations" + } else { + "seed_locations" + })?; + let location: Bound<'py, PyAny> = locations.call_method1("get", (name,))?; + if location.is_none() { + return Ok(location); + } + qualified_name(adapter, &location) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_relation_targets, module)?)?; + module.add_function(wrap_pyfunction!(build_project_relation_targets, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs new file mode 100644 index 0000000000..c0cbc5c347 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -0,0 +1,169 @@ +//! Adapter default schema and database applied to compiled relation locations. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// The adapter defaults every compiled location falls back to. +pub(crate) struct TargetDefaults<'py> { + pub(crate) default_schema: Bound<'py, PyAny>, + pub(crate) default_database: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, + pub(crate) python_functions_inherit_default_namespace: Bound<'py, PyAny>, +} + +impl<'py> TargetDefaults<'py> { + /// `(default_schema, default_database)`. + fn pair(&self) -> PyResult> { + PyTuple::new( + self.default_schema.py(), + [self.default_schema.clone(), self.default_database.clone()], + ) + } +} + +/// `left or right`. +fn or_else<'py>(left: Bound<'py, PyAny>, right: &Bound<'py, PyAny>) -> PyResult> { + Ok(if left.is_truthy()? { + left + } else { + right.clone() + }) +} + +/// Apply adapter default schema/database to compiled project targets. +pub(crate) fn apply_target_defaults<'py>( + project: &Bound<'py, PyAny>, + defaults: &TargetDefaults<'py>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let compiled_model: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledModel")?; + let native_seed: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeCompiledSeed")?; + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let mut models: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + models.push(if model.is_instance(&compiled_model)? { + replace.call( + (&model,), + Some(&keywords( + py, + &[( + "destination", + resolved_target(&model.getattr("destination")?, defaults)?, + )], + )?), + )? + } else { + model.call_method1( + "with_target_defaults", + (defaults.pair()?, &defaults.render_qualified_name), + )? + }); + } + let mut seeds: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + seeds.push(if seed.is_instance(&native_seed)? { + seed.call_method1( + "with_target_defaults", + (defaults.pair()?, &defaults.render_qualified_name), + )? + } else { + replace.call( + (&seed,), + Some(&keywords( + py, + &[( + "destination", + resolved_target(&seed.getattr("destination")?, defaults)?, + )], + )?), + )? + }); + } + let mut functions: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + functions.push(function?.call_method1( + "with_target_defaults", + ( + defaults.pair()?, + &defaults.render_qualified_name, + &defaults.python_functions_inherit_default_namespace, + ), + )?); + } + call_with( + &imported(py, NATIVE_MODULE, "project_with")?, + &[ + ("project", project.clone()), + ( + "effective_target_database", + or_else( + project.getattr("effective_target_database")?, + &defaults.default_database, + )?, + ), + ( + "effective_target_schema", + or_else( + project.getattr("effective_target_schema")?, + &defaults.default_schema, + )?, + ), + ("models", PyTuple::new(py, models)?.into_any()), + ("seeds", PyTuple::new(py, seeds)?.into_any()), + ("functions", PyTuple::new(py, functions)?.into_any()), + ], + ) +} + +/// `_resolve_target`: `target` with adapter defaults for a missing schema or database. +fn resolved_target<'py>( + target: &Bound<'py, PyAny>, + defaults: &TargetDefaults<'py>, +) -> PyResult> { + let py: Python<'py> = target.py(); + let target_schema: Bound<'py, PyAny> = target.getattr("schema")?; + let target_database: Bound<'py, PyAny> = target.getattr("database")?; + let schema: Bound<'py, PyAny> = if target_schema.is_none() { + defaults.default_schema.clone() + } else { + target_schema.clone() + }; + let database: Bound<'py, PyAny> = if target_database.is_none() { + defaults.default_database.clone() + } else { + target_database.clone() + }; + let name: Bound<'py, PyAny> = target.getattr("name")?; + let qualified_name: Bound<'py, PyAny> = call_with( + &defaults.render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )?; + if schema.eq(&target_schema)? + && database.eq(&target_database)? + && qualified_name.eq(target.getattr("qualified_name")?)? + { + return Ok(target.clone()); + } + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledRelationLocation")?, + &[ + ("database", database), + ("schema", schema), + ("name", name), + ("qualified_name", qualified_name), + ("logical_schema", target.getattr("logical_schema")?), + ("logical_database", target.getattr("logical_database")?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs new file mode 100644 index 0000000000..919ddc74f3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -0,0 +1,595 @@ +//! Adapter-aware target namespace validation of a compile. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::assembly::target_checks::constants::{ + LOADER_COLLISION_CODE, MANAGED_WRITE_SCHEMA_CODE, MISSING_TARGET_PARTS_CODE, + NAMED_TARGET_SCHEMA_CODE, +}; +use sqlbuild_analysis::assembly::target_checks::errors::TargetError; +use sqlbuild_analysis::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; +use sqlbuild_analysis::assembly::target_checks::main::first_shared_namespace::first_shared_namespace; +use sqlbuild_analysis::assembly::target_checks::main::missing_target_parts::missing_target_parts; +use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; +use sqlbuild_analysis::assembly::target_checks::models::{LoaderCollision, MissingParts}; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; +/// Connection settings that identify the warehouse two targets write to. +const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ + "account", + "host", + "port", + "project", + "server", + "server_hostname", + "workspace_url", +]; +/// Adapters whose execution needs every model and seed database and schema resolved. +const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; +const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; +const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; +const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +const DEFAULTS_SECTION: &str = "defaults"; + +/// One optional `(database, schema)` namespace. +type Namespace = (Option, Option); +/// A target's sorted connection identity and the namespaces its managed sources write. +type TargetNamespaces = (Vec<(String, String)>, Vec); + +/// A `PlannerInputError` with `message`, `code` and optional `help`. +fn planner_error(py: Python<'_>, message: String, code: &str, help: Option) -> PyErr { + let raised: PyResult> = (|| { + imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")?.call( + (message,), + Some(&keywords( + py, + &[ + ("code", PyString::new(py, code).into_any()), + ( + "help", + help.map_or_else( + || py.None().into_bound(py), + |help| PyString::new(py, &help).into_any(), + ), + ), + ], + )?), + ) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `value` as an optional string: `None`, or a `str`'s text. +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// `value` when it is a `str`, else `None`. +fn text_or_none(value: &Bound<'_, PyAny>) -> PyResult> { + match value.downcast::() { + Ok(text) => Ok(Some(text.to_cow()?.into_owned())), + Err(_) => Ok(None), + } +} + +/// Require shared-warehouse named targets to choose a schema strategy. +#[pyfunction] +#[pyo3(signature = (**options))] +fn validate_named_target_schema_strategy<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "validate_named_target_schema_strategy", + options, + (&["discovered_inputs", "adapter", "selected_target"], &[]), + )?; + named_target_schema_strategy( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + &arguments.value("selected_target")?, + ) +} + +/// `validate_named_target_schema_strategy` with its arguments. +pub(crate) fn named_target_schema_strategy( + discovered: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + if adapter + .getattr("allows_implicit_managed_write_schema")? + .is_truthy()? + { + return Ok(()); + } + let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let target_name: Bound<'_, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + if target_name.is_none() { + return Ok(()); + } + let target_config: Bound<'_, PyAny> = call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", project_config), + ("local_config", local_config), + ("target_name", target_name.clone()), + ], + )?; + let schema: Option = text_or_none(&target_config.getattr("schema")?)?; + if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { + return Ok(()); + } + Err(planner_error( + py, + target_error_message(&TargetError::NamedTargetSchema( + target_name.str()?.to_cow()?.into_owned(), + )), + NAMED_TARGET_SCHEMA_CODE, + None, + )) +} + +/// Reject managed loader write namespaces shared by multiple targets. +#[pyfunction] +#[pyo3(signature = (**options))] +fn validate_managed_loader_target_isolation<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "validate_managed_loader_target_isolation", + options, + (&["discovered_inputs", "adapter"], &["resolved_connection"]), + )?; + managed_loader_target_isolation( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + &arguments.value("resolved_connection")?, + ) +} + +/// `validate_managed_loader_target_isolation` with its arguments. +pub(crate) fn managed_loader_target_isolation( + discovered: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + resolved_connection: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let mut managed_sources: Vec> = Vec::new(); + for source_file in discovered.getattr("source_files")?.try_iter()? { + for entry in source_file?.getattr("source_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if entry.getattr("managed")?.is_truthy()? { + managed_sources.push(entry); + } + } + } + if managed_sources.is_empty() { + return Ok(()); + } + let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let mut target_names: Vec = Vec::new(); + for config in [&project_config, &local_config] { + for name in config.getattr("targets")?.try_iter()? { + let name: String = name?.extract()?; + if !target_names.contains(&name) { + target_names.push(name); + } + } + } + target_names.sort(); + let mut targets: Vec = Vec::new(); + for target_name in &target_names { + targets.push(target_namespaces( + (discovered, adapter, resolved_connection), + target_name, + &managed_sources, + )?); + } + for (index, left_target) in target_names.iter().enumerate() { + for (offset, right_target) in target_names[index + 1..].iter().enumerate() { + let (left, right) = (&targets[index], &targets[index + 1 + offset]); + if left.0 != right.0 { + continue; + } + let shared: Vec = left + .1 + .iter() + .filter(|namespace| right.1.contains(namespace)) + .cloned() + .collect(); + let Some((database, schema)) = first_shared_namespace(&shared) else { + continue; + }; + return Err(planner_error( + py, + target_error_message(&TargetError::LoaderCollision(LoaderCollision { + left_target: left_target.clone(), + right_target: right_target.clone(), + database, + schema, + })), + LOADER_COLLISION_CODE, + None, + )); + } + } + Ok(()) +} + +/// One target's connection identity and the namespaces its managed sources write. +fn target_namespaces( + (discovered, adapter, resolved_connection): ( + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + ), + target_name: &str, + managed_sources: &[Bound<'_, PyAny>], +) -> PyResult { + let py: Python<'_> = discovered.py(); + let name: Bound<'_, PyAny> = PyString::new(py, target_name).into_any(); + let target_config: Bound<'_, PyAny> = call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_name", name.clone()), + ], + )?; + let runtime: Bound<'_, PyAny> = call_with( + &imported( + py, + EFFECTIVE_RUNTIME_MODULE, + "build_effective_runtime_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", name.clone()), + ], + )?; + let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + runtime.extract()?; + let connection: Bound<'_, PyAny> = if resolved_connection.is_none() { + call_with( + &imported( + py, + EFFECTIVE_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", name), + ], + )? + } else { + resolved_connection.clone() + }; + let mut identity: Vec<(String, String)> = Vec::new(); + for item in connection.call_method0("items")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let key: Bound<'_, PyAny> = item.get_item(0)?; + let Some(key) = text_or_none(&key)? else { + continue; + }; + if CONNECTION_IDENTITY_KEYS.contains(&key.as_str()) { + identity.push((key, item.get_item(1)?.repr()?.to_cow()?.into_owned())); + } + } + identity.sort(); + let mut namespaces: Vec = Vec::new(); + for source_entry in managed_sources { + let namespace: Namespace = managed_source_namespace( + source_entry, + (&target_config, &connection, &effective_vars), + adapter, + )?; + if !namespaces.contains(&namespace) { + namespaces.push(namespace); + } + } + Ok((identity, namespaces)) +} + +/// The first truthy value of `candidates`, else `fallback()`. +fn first_truthy<'py>( + candidates: &[Bound<'py, PyAny>], + fallback: impl FnOnce() -> PyResult>, +) -> PyResult> { + for candidate in candidates { + if candidate.is_truthy()? { + return Ok(candidate.clone()); + } + } + fallback() +} + +/// `_managed_source_namespace`: the database and schema a managed source loads into. +fn managed_source_namespace( + source_entry: &Bound<'_, PyAny>, + (target_config, connection, effective_vars): ( + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + ), + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = source_entry.py(); + let database: Bound<'_, PyAny> = first_truthy( + &[ + source_entry.getattr("database")?, + target_config.getattr("database")?, + connection.call_method1("get", ("database",))?, + ], + || adapter.call_method0("default_database"), + )?; + let schema: Bound<'_, PyAny> = first_truthy( + &[ + source_entry.getattr("schema")?, + target_config.getattr("loader_schema")?, + target_config.getattr("schema")?, + connection.call_method1("get", ("schema",))?, + ], + || adapter.call_method0("default_schema"), + )?; + let value: Bound<'_, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&value, "database", database)?; + pyo3::types::PyDictMethods::set_item(&value, "schema", schema)?; + let false_value: Bound<'_, PyAny> = pyo3::types::PyBool::new(py, false).to_owned().into_any(); + let expanded: Bound<'_, PyAny> = call_with( + &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, + &[ + ("value", value.into_any()), + ("variables", effective_vars.clone()), + ("context_values", PyDict::new(py).into_any()), + ( + "context_label", + PyString::new(py, LOADER_NAMESPACE_LABEL).into_any(), + ), + ("allow_context", false_value.clone()), + ("preserve_context_tokens", false_value.clone()), + ("preserve_unknown_context", false_value), + ], + )?; + let part = |key: &str| -> PyResult> { + let value: Bound<'_, PyAny> = expanded.get_item(key)?; + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_cow()?.into_owned())) + }; + Ok((part("database")?, part("schema")?)) +} + +/// Validate compiled model and seed locations for the effective adapter. +#[pyfunction] +#[pyo3(signature = (**options))] +fn validate_project_targets<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "validate_project_targets", + options, + (&["adapter_name", "project"], &[]), + )?; + project_targets( + &arguments.value("adapter_name")?, + &arguments.value("project")?, + ) +} + +/// `validate_project_targets` with its arguments. +pub(crate) fn project_targets( + adapter_name: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult<()> { + let name: Option = text_or_none(adapter_name)?; + if !name + .as_deref() + .is_some_and(|name| EXPLICIT_TARGET_ADAPTERS.contains(&name)) + { + return Ok(()); + } + let adapter_label: String = adapter_name.str()?.to_cow()?.into_owned(); + for (resource_kind, attribute) in [("model", "models"), ("seed", "seeds")] { + let mut targets: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let resource_name: String = resource.getattr("name")?.str()?.to_cow()?.into_owned(); + let destination: Bound<'_, PyAny> = resource.getattr("destination")?; + match targets + .iter_mut() + .find(|(existing, _)| *existing == resource_name) + { + Some(entry) => entry.1 = destination, + None => targets.push((resource_name, destination)), + } + } + for (resource_name, target) in targets { + required_target_parts((&adapter_label, resource_kind), resource_name, &target)?; + } + } + Ok(()) +} + +/// Raise when `target` misses its database or schema. +fn required_target_parts( + (adapter_name, resource_kind): (&str, &str), + resource_name: String, + target: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target.py(); + let database: Option = optional_text(&target.getattr("database")?)?; + let schema: Option = optional_text(&target.getattr("schema")?)?; + let missing: Vec<&str> = missing_target_parts(database.as_deref(), schema.as_deref()); + let Some(first) = missing.first() else { + return Ok(()); + }; + let help: String = call_with( + &imported(py, SETTING_HELP_MODULE, "setting_help")?, + &[ + ( + "purpose", + PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), + ), + ( + "file_name", + PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), + ), + ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), + ("key", PyString::new(py, first).into_any()), + ( + "value", + PyString::new(py, &format!("my_{first}")).into_any(), + ), + ], + )? + .extract()?; + Err(planner_error( + py, + target_error_message(&TargetError::MissingParts(MissingParts { + adapter_name: adapter_name.to_owned(), + resource_kind: resource_kind.to_owned(), + resource_name, + database, + schema, + })), + MISSING_TARGET_PARTS_CODE, + Some(help), + )) +} + +/// Require an explicitly resolved physical schema for every managed warehouse write. +#[pyfunction] +#[pyo3(signature = (**options))] +fn validate_managed_write_schemas<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "validate_managed_write_schemas", + options, + (&["adapter", "project"], &[]), + )?; + managed_write_schemas(&arguments.value("adapter")?, &arguments.value("project")?) +} + +/// `validate_managed_write_schemas` with its arguments. +pub(crate) fn managed_write_schemas( + adapter: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult<()> { + if adapter + .getattr("allows_implicit_managed_write_schema")? + .is_truthy()? + { + return Ok(()); + } + let connection_schema: Option = explicit_write_schema( + text_or_none( + &project + .getattr("effective_connection")? + .call_method1("get", ("schema",))?, + )? + .as_deref(), + ); + for (label, attribute) in [ + ("Model", "models"), + ("Seed", "seeds"), + ("Function", "functions"), + ] { + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + managed_write_schema( + &format!("{label} '{}'", resource.getattr("name")?.str()?.to_cow()?), + &resource.getattr("destination")?.getattr("schema")?, + connection_schema.as_deref(), + )?; + } + } + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; + if !entry.getattr("managed")?.is_truthy()? { + continue; + } + managed_write_schema( + &format!( + "Managed source '{}'", + source.getattr("name")?.str()?.to_cow()? + ), + &entry.getattr("schema")?, + connection_schema.as_deref(), + )?; + } + Ok(()) +} + +/// Raise when neither the resource nor the connection names a physical write schema. +fn managed_write_schema( + resource_label: &str, + schema: &Bound<'_, PyAny>, + connection_schema: Option<&str>, +) -> PyResult<()> { + if explicit_write_schema(text_or_none(schema)?.as_deref()).is_some() + || connection_schema.is_some() + { + return Ok(()); + } + Err(planner_error( + schema.py(), + target_error_message(&TargetError::ManagedWriteSchema(resource_label.to_owned())), + MANAGED_WRITE_SCHEMA_CODE, + None, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + validate_named_target_schema_strategy, + module + )?)?; + module.add_function(wrap_pyfunction!( + validate_managed_loader_target_isolation, + module + )?)?; + module.add_function(wrap_pyfunction!(validate_project_targets, module)?)?; + module.add_function(wrap_pyfunction!(validate_managed_write_schemas, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs new file mode 100644 index 0000000000..0c6f8031a5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs @@ -0,0 +1,61 @@ +//! The single seam between the compiler stages and the native frontier. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::frontier::stage_capture::write_stage_capture; + +const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge"; + +/// Produce the frontier object for `until`, capturing it on request. +#[pyfunction] +#[pyo3(signature = (**options))] +fn compile_frontier<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "compile_frontier", + options, + (&["until", "python_stage"], &["native_stage"]), + )?; + let until: Bound<'py, PyAny> = arguments.value("until")?; + let result: Bound<'py, PyAny> = native_frontier( + &until, + &arguments.value("python_stage")?, + &arguments.value("native_stage")?, + )?; + write_stage_capture(&until, &result)?; + Ok(result) +} + +/// The frontier object for `until`; stages not yet native run in Python. +fn native_frontier<'py>( + until: &Bound<'py, PyAny>, + python_stage: &Bound<'py, PyAny>, + native_stage: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = until.py(); + if !native_stage.is_none() { + return native_stage.call0(); + } + let inputs_stage: Bound<'py, PyAny> = + imported(py, FRONTIER_TYPES_MODULE, "CompilerStage")?.getattr("COMPILE_PROJECT_INPUTS")?; + if until.is(&inputs_stage) { + return call_with( + &imported(py, RUN_BRIDGE_MODULE, "run_with_macro_bridge")?, + &[("stage", python_stage.clone())], + ); + } + python_stage.call0() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_frontier, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs new file mode 100644 index 0000000000..b5512b437b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs @@ -0,0 +1,442 @@ +//! Encode a frontier object graph as canonical JSON, sharing large subtrees by digest. + +use std::collections::{HashMap, HashSet}; + +use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, +}; + +use sqlbuild_cache::stage_capture::constants::{ + CANONICAL_TYPES, DECODED_SEQUENCES, OMITTED_ATTRIBUTES, PROJECTED_ATTRIBUTES, RESERVED_KEYS, + SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, UNORDERED_ATTRIBUTES, +}; +use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; +use sqlbuild_cache::stage_capture::main::compact_capture_json::compact_capture_json; +use sqlbuild_cache::stage_capture::main::shared_node_digest::shared_node_digest; +use sqlbuild_cache::stage_capture::models::CaptureDocument; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::frontier::python_types::PythonTypes; + +/// An instance's attributes, by name in first-set order. +type Attributes<'py> = Vec<(String, Bound<'py, PyAny>)>; + +/// Nesting at which encoding stops, as Python's recursion limit stops the Python encoder. +const MAX_DEPTH: usize = 900; +const RECURSION_MESSAGE: &str = "maximum recursion depth exceeded"; +const SURROGATE_MESSAGE: &str = "'utf-8' codec can't encode characters: surrogates not allowed"; + +/// The Python error a capture serializer failure stands for. +pub(crate) fn emit_error(error: JsonEmitError) -> PyErr { + match error { + JsonEmitError::LoneSurrogate => PyUnicodeEncodeError::new_err(SURROGATE_MESSAGE), + JsonEmitError::NestingTooDeep => PyRecursionError::new_err(RECURSION_MESSAGE), + other => PyValueError::new_err(format!("{other:?}")), + } +} + +/// `{"key": value}`. +fn tagged(key: &str, value: JsonValue) -> JsonValue { + JsonValue::Object(vec![(key.to_owned(), value)]) +} + +/// Set `key` in `entries`, keeping the position of a key already present, as a dict does. +fn with_entry(mut entries: Vec<(String, T)>, key: String, value: T) -> Vec<(String, T)> { + match entries.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => entries.push((key, value)), + } + entries +} + +/// `table[name]` of a per-type table. +fn lookup<'a, T: Copy>(table: &'a [(&'a str, T)], name: &str) -> Option { + table + .iter() + .find(|(type_name, _)| *type_name == name) + .map(|(_, value)| *value) +} + +/// `module:qualname` of a callable or class, as Python's `qualified_name` writes it. +pub(crate) fn qualified_name(value: &Bound<'_, PyAny>) -> PyResult { + let module: Bound<'_, PyAny> = attribute_or_none(value, "__module__")?; + let mut name: Bound<'_, PyAny> = attribute_or_none(value, "__qualname__")?; + if !name.is_truthy()? { + name = attribute_or_none(value, "__name__")?; + } + if name.is_none() { + return qualified_name(&value.get_type().into_any()); + } + let name: String = name.str()?.to_str()?.to_owned(); + match module.downcast::() { + Ok(module) => Ok(format!("{}:{name}", module.to_str()?)), + Err(_) => Ok(name), + } +} + +/// `getattr(value, name, None)`. +fn attribute_or_none<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + let py: Python<'py> = value.py(); + match value.getattr(name) { + Ok(attribute) => Ok(attribute), + Err(error) if error.is_instance_of::(py) => { + Ok(py.None().into_bound(py)) + } + Err(error) => Err(error), + } +} + +/// A Python `str` as JSON text; one holding a lone surrogate as its UTF-16 code units. +fn text_value(text: &Bound<'_, PyString>) -> PyResult { + if let Ok(text) = text.to_str() { + return Ok(JsonValue::String(text.to_owned())); + } + let encoded: Bound<'_, PyBytes> = text + .call_method1("encode", ("utf-16-le", "surrogatepass"))? + .downcast_into::()?; + Ok(JsonValue::Utf16( + encoded + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + )) +} + +/// The capture encoder with the document its shared nodes are written to. +pub(crate) struct CaptureEncoder<'py> { + types: PythonTypes<'py>, + active: HashSet, + cycles: usize, + encoded: HashMap, JsonValue)>, + digests: HashSet, + depth: usize, + pub(crate) document: CaptureDocument, + /// The document text written so far: the shared nodes, as Python streams them. + pub(crate) written: String, +} + +impl<'py> CaptureEncoder<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + types: PythonTypes::load(py)?, + active: HashSet::new(), + cycles: 0, + encoded: HashMap::new(), + digests: HashSet::new(), + depth: 0, + document: CaptureDocument::default(), + written: String::new(), + }) + } + + /// The JSON value of `value`, keeping insertion order and sorting only sets. + pub(crate) fn encode(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + if self.depth >= MAX_DEPTH { + return Err(PyRecursionError::new_err(RECURSION_MESSAGE)); + } + self.depth += 1; + let encoded: PyResult = self.encode_value(value); + self.depth -= 1; + encoded + } + + fn encode_value(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + if value.is_instance(&self.types.enumeration)? { + let name: String = value.getattr("name")?.str()?.to_str()?.to_owned(); + return Ok(tagged( + "__enum__", + JsonValue::String(format!( + "{}.{name}", + qualified_name(&value.get_type().into_any())? + )), + )); + } + if let Ok(text) = value.downcast::() { + let encoded: JsonValue = text_value(text)?; + return if value.len()? < SHARED_MIN_CHARS { + Ok(encoded) + } else { + self.share(encoded) + }; + } + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + let decimal: String = imported(value.py(), "builtins", "int")? + .call_method1("__repr__", (value,))? + .extract()?; + return Ok(JsonInteger::parse(&decimal).map_or(JsonValue::Null, JsonValue::Integer)); + } + if value.is_instance_of::() { + let number: f64 = value.extract()?; + return Ok(if number.is_finite() { + JsonValue::Float(number) + } else { + tagged( + "__float__", + JsonValue::String(value.repr()?.to_str()?.to_owned()), + ) + }); + } + if let Some(scalar) = self.types.scalar(value)? { + return Ok(scalar); + } + let identity: usize = value.as_ptr() as usize; + if self.active.contains(&identity) { + self.cycles += 1; + return Ok(tagged( + "__cycle__", + JsonValue::String(qualified_name(&value.get_type().into_any())?), + )); + } + if let Some((_, known)) = self.encoded.get(&identity) { + return Ok(known.clone()); + } + let cycles: usize = self.cycles; + self.active.insert(identity); + let container: PyResult = self.encode_container(value); + let encoded: PyResult = container.and_then(|container| self.share(container)); + self.active.remove(&identity); + let encoded: JsonValue = encoded?; + if self.cycles == cycles { + self.encoded + .insert(identity, (value.clone(), encoded.clone())); + } + Ok(encoded) + } + + /// The shared-node marker for a large object, array or string, writing the node once. + fn share(&mut self, encoded: JsonValue) -> PyResult { + if !matches!( + encoded, + JsonValue::Object(_) | JsonValue::Array(_) | JsonValue::String(_) | JsonValue::Utf16(_) + ) { + return Ok(encoded); + } + let text: String = compact_capture_json(&encoded).map_err(emit_error)?; + if text.chars().count() < SHARED_MIN_CHARS { + return Ok(encoded); + } + let digest: String = shared_node_digest(&text); + if self.digests.insert(digest.clone()) { + let chunk: String = self.document.shared_chunk(&digest, &text); + self.written.push_str(&chunk); + } + Ok(tagged(SHARED_MARKER, JsonValue::String(digest))) + } + + fn encode_container(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + let type_name: String = qualified_name(&value.get_type().into_any())?; + if DECODED_SEQUENCES.contains(&type_name.as_str()) + && value.is_instance(&self.types.sequence)? + { + return self.encode_items(value); + } + if self.types.dataclass_instance_check(value)? { + return self.encode_dataclass(value, type_name); + } + if value.is_instance(&self.types.mapping)? { + return self.encode_mapping(value); + } + if value.is_instance(&self.types.abstract_set)? { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(self.encode(&item?)?); + } + return Ok(tagged("__set__", JsonValue::Array(sorted_items(items)?))); + } + if value.is_instance_of::() || value.is_instance_of::() { + return self.encode_items(value); + } + if value.is_instance(&self.types.partial)? { + return Ok(JsonValue::Object(vec![ + ( + "__partial__".to_owned(), + self.encode(&value.getattr("func")?)?, + ), + ("args".to_owned(), self.encode(&value.getattr("args")?)?), + ( + "keywords".to_owned(), + self.encode(&value.getattr("keywords")?)?, + ), + ])); + } + self.encode_instance(value, type_name) + } + + fn encode_items(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(self.encode(&item?)?); + } + Ok(JsonValue::Array(items)) + } + + fn encode_dataclass( + &mut self, + value: &Bound<'py, PyAny>, + type_name: String, + ) -> PyResult { + let omitted: &[&str] = lookup(&OMITTED_ATTRIBUTES, &type_name).unwrap_or(&[]); + let mut entries: Vec<(String, JsonValue)> = + vec![("__type__".to_owned(), JsonValue::String(type_name))]; + for field in self.types.fields.call1((value,))?.try_iter()? { + let name: String = field?.getattr("name")?.extract()?; + if omitted.contains(&name.as_str()) { + continue; + } + let item: Bound<'py, PyAny> = attribute_or_none(value, &name)?; + let encoded: JsonValue = self.encode(&item)?; + entries = with_entry(entries, name, encoded); + } + Ok(JsonValue::Object(entries)) + } + + fn encode_mapping(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + let mut plain: bool = true; + for key in value.try_iter()? { + if !key?.is_instance_of::() { + plain = false; + break; + } + } + if plain { + for key in value.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if RESERVED_KEYS.contains(&key.downcast::()?.to_cow()?.as_ref()) { + plain = false; + break; + } + } + } + if !plain { + return Ok(tagged( + "__mapping__", + JsonValue::Array(self.mapping_pairs(value)?), + )); + } + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for item in value.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let key: String = item.get_item(0)?.str()?.to_str()?.to_owned(); + let encoded: JsonValue = self.encode(&item.get_item(1)?)?; + entries = with_entry(entries, key, encoded); + } + Ok(JsonValue::Object(entries)) + } + + fn mapping_pairs(&mut self, value: &Bound<'py, PyAny>) -> PyResult> { + let mut pairs: Vec = Vec::new(); + for item in value.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let key: JsonValue = self.encode(&item.get_item(0)?)?; + let encoded: JsonValue = self.encode(&item.get_item(1)?)?; + pairs.push(JsonValue::Array(vec![key, encoded])); + } + Ok(pairs) + } + + fn encode_instance( + &mut self, + value: &Bound<'py, PyAny>, + type_name: String, + ) -> PyResult { + let attributes: Option)>> = + match lookup(&PROJECTED_ATTRIBUTES, &type_name) { + Some(names) => Some( + names + .iter() + .map(|name| Ok(((*name).to_owned(), value.getattr(*name)?))) + .collect::>()?, + ), + None => instance_attributes(value)?, + }; + let Some(attributes) = attributes else { + return Ok(tagged("__opaque__", JsonValue::String(type_name))); + }; + let unordered: &[&str] = lookup(&UNORDERED_ATTRIBUTES, &type_name).unwrap_or(&[]); + let omitted: &[&str] = lookup(&OMITTED_ATTRIBUTES, &type_name).unwrap_or(&[]); + let canonical: String = + lookup(&CANONICAL_TYPES, &type_name).map_or(type_name.clone(), str::to_owned); + let mut entries: Vec<(String, JsonValue)> = + vec![("__type__".to_owned(), JsonValue::String(canonical))]; + for (name, item) in attributes { + if omitted.contains(&name.as_str()) { + continue; + } + let encoded: JsonValue = + if unordered.contains(&name.as_str()) && item.is_instance(&self.types.mapping)? { + tagged( + "__unordered_mapping__", + JsonValue::Array(sorted_items(self.mapping_pairs(&item)?)?), + ) + } else { + self.encode(&item)? + }; + entries = with_entry(entries, name, encoded); + } + Ok(JsonValue::Object(entries)) + } +} + +/// `items` sorted by their masked, key-sorted JSON text, as Python's `sorted` orders them. +fn sorted_items(items: Vec) -> PyResult> { + let mut keyed: Vec<(String, JsonValue)> = items + .into_iter() + .map(|item| Ok((capture_sort_key(&item).map_err(emit_error)?, item))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, item)| item).collect()) +} + +/// `_instance_attributes`: the instance `__dict__` and the slots set on `value`, or `None`. +fn instance_attributes<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + let instance_dict: Bound<'py, PyAny> = attribute_or_none(value, "__dict__")?; + let has_dict: bool = instance_dict.is_exact_instance_of::() + || instance_dict.is_instance_of::(); + let mut attributes: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + if has_dict { + for item in instance_dict.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let name: String = item.get_item(0)?.str()?.to_str()?.to_owned(); + attributes = with_entry(attributes, name, item.get_item(1)?); + } + } + for cls in value.get_type().getattr("__mro__")?.try_iter()? { + let slots: Bound<'py, PyAny> = cls? + .getattr("__dict__")? + .call_method1("get", ("__slots__", PyTuple::empty(value.py())))?; + let names: Vec> = if slots.is_instance_of::() { + vec![slots] + } else { + slots.try_iter()?.collect::>()? + }; + for name in names { + let Ok(text) = name.downcast::() else { + continue; + }; + let text: String = text.to_str()?.to_owned(); + if SKIPPED_SLOTS.contains(&text.as_str()) || !value.hasattr(text.as_str())? { + continue; + } + let item: Bound<'py, PyAny> = value.getattr(text.as_str())?; + attributes = with_entry(attributes, text, item); + } + } + if attributes.is_empty() && !has_dict { + return Ok(None); + } + Ok(Some(attributes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs new file mode 100644 index 0000000000..fa17bbe460 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod compile_frontier; +pub(crate) mod encoder; +pub(crate) mod python_types; +pub(crate) mod registration; +pub(crate) mod stage_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs new file mode 100644 index 0000000000..4cfe429e65 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs @@ -0,0 +1,134 @@ +//! The Python types the capture encoder distinguishes, and its scalar encodings. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyBytesMethods, PyStringMethods, PyTuple}; + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::frontier::encoder::qualified_name; + +/// The Python classes and functions the encoder checks values against. +pub(crate) struct PythonTypes<'py> { + pub(crate) enumeration: Bound<'py, PyAny>, + pub(crate) mapping: Bound<'py, PyAny>, + pub(crate) abstract_set: Bound<'py, PyAny>, + pub(crate) sequence: Bound<'py, PyAny>, + pub(crate) partial: Bound<'py, PyAny>, + pub(crate) fields: Bound<'py, PyAny>, + is_dataclass: Bound<'py, PyAny>, + pure_path: Bound<'py, PyAny>, + decimal: Bound<'py, PyAny>, + dates: Bound<'py, PyTuple>, + bytes_like: Bound<'py, PyTuple>, + pattern: Bound<'py, PyAny>, + module: Bound<'py, PyAny>, + callables: Bound<'py, PyTuple>, + sha256: Bound<'py, PyAny>, +} + +impl<'py> PythonTypes<'py> { + pub(crate) fn load(py: Python<'py>) -> PyResult { + let types = |name: &str| imported(py, "types", name); + Ok(Self { + enumeration: imported(py, "enum", "Enum")?, + mapping: imported(py, "collections.abc", "Mapping")?, + abstract_set: imported(py, "collections.abc", "Set")?, + sequence: imported(py, "collections.abc", "Sequence")?, + partial: imported(py, "functools", "partial")?, + fields: imported(py, "dataclasses", "fields")?, + is_dataclass: imported(py, "dataclasses", "is_dataclass")?, + pure_path: imported(py, "pathlib", "PurePath")?, + decimal: imported(py, "decimal", "Decimal")?, + dates: PyTuple::new( + py, + [ + imported(py, "datetime", "date")?, + imported(py, "datetime", "time")?, + ], + )?, + bytes_like: PyTuple::new( + py, + [ + imported(py, "builtins", "bytes")?, + imported(py, "builtins", "bytearray")?, + imported(py, "builtins", "memoryview")?, + ], + )?, + pattern: imported(py, "re", "Pattern")?, + module: types("ModuleType")?, + callables: PyTuple::new( + py, + [ + types("FunctionType")?, + types("BuiltinFunctionType")?, + types("MethodType")?, + types("BuiltinMethodType")?, + imported(py, "builtins", "type")?, + ], + )?, + sha256: imported(py, "hashlib", "sha256")?, + }) + } + + /// Whether `value` is a dataclass instance rather than a dataclass. + pub(crate) fn dataclass_instance_check(&self, value: &Bound<'py, PyAny>) -> PyResult { + Ok(self.is_dataclass.call1((value,))?.is_truthy()? + && !value.is_instance(&imported(value.py(), "builtins", "type")?)?) + } + + /// `_encode_scalar`: paths, decimals, dates, bytes, patterns, modules and callables. + pub(crate) fn scalar(&self, value: &Bound<'py, PyAny>) -> PyResult> { + let text = |key: &str, text: String| { + JsonValue::Object(vec![(key.to_owned(), JsonValue::String(text))]) + }; + if value.is_instance(&self.pure_path)? { + return Ok(Some(text( + "__path__", + value.call_method0("as_posix")?.extract()?, + ))); + } + if value.is_instance(&self.decimal)? { + return Ok(Some(text("__decimal__", value.str()?.to_str()?.to_owned()))); + } + if value.is_instance(self.dates.as_any())? { + return Ok(Some(text( + "__datetime__", + value.call_method0("isoformat")?.extract()?, + ))); + } + if value.is_instance(self.bytes_like.as_any())? { + let data: Bound<'py, PyBytes> = imported(value.py(), "builtins", "bytes")? + .call1((value,))? + .downcast_into::()?; + let digest: String = self + .sha256 + .call1((&data,))? + .call_method0("hexdigest")? + .extract()?; + return Ok(Some(JsonValue::Object(vec![ + ( + "__bytes__".to_owned(), + JsonValue::Integer(JsonInteger::from(data.as_bytes().len() as u64)), + ), + ("sha256".to_owned(), JsonValue::String(digest)), + ]))); + } + if value.is_instance(&self.pattern)? { + return Ok(Some(text( + "__pattern__", + value.getattr("pattern")?.str()?.to_str()?.to_owned(), + ))); + } + if value.is_instance(&self.module)? { + return Ok(Some(text( + "__module__", + value.getattr("__name__")?.extract()?, + ))); + } + if value.is_instance(self.callables.as_any())? { + return Ok(Some(text("__callable__", qualified_name(value)?))); + } + Ok(None) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs new file mode 100644 index 0000000000..82b9d86c7e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs @@ -0,0 +1,10 @@ +//! Register the compiler frontier and its stage captures. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::frontier::{compile_frontier, stage_capture}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + compile_frontier::register(module)?; + stage_capture::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs new file mode 100644 index 0000000000..dec05b79b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -0,0 +1,117 @@ +//! Stage captures: frontier objects written as canonical JSON so compiles compare stage by stage. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; +use sqlbuild_cache::stage_capture::main::mask_capture_noise::mask_capture_noise as masked; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + call_with, imported, keywords, within, +}; +use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; + +const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; +const CAPTURE_SUFFIX: &str = ".json"; +const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; + +/// The capture document of `value`, or the shared nodes written before encoding failed. +fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result { + let mut encoder: CaptureEncoder<'_> = + CaptureEncoder::new(py).map_err(|error| (String::new(), error))?; + let root = match encoder.encode(value) { + Ok(root) => root, + Err(error) => return Err((encoder.written, error)), + }; + match encoder.document.final_chunk(&root) { + Ok(chunk) => Ok(encoder.written + &chunk), + Err(error) => Err((encoder.written, emit_error(error))), + } +} + +/// Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory. +pub(crate) fn write_stage_capture( + stage: &Bound<'_, PyAny>, + value: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = stage.py(); + let directory: Bound<'_, PyAny> = + imported(py, "os", "environ")?.call_method1("get", (CAPTURE_DIR_ENV_VAR,))?; + if !directory.is_truthy()? { + return Ok(()); + } + let number: u64 = imported(py, "builtins", "next")? + .call1((imported( + py, + FRONTIER_CONSTANTS_MODULE, + "STAGE_CAPTURE_SEQUENCE", + )?,))? + .extract()?; + let stage_name: String = stage.getattr("value")?.str()?.extract()?; + let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? + .call1((directory,))? + .call_method1( + "__truediv__", + (format!("{number:03}-{stage_name}{CAPTURE_SUFFIX}"),), + )?; + call_with( + &path.getattr("parent")?.getattr("mkdir")?, + &[ + ( + "parents", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ( + "exist_ok", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ], + )?; + let stream: Bound<'_, PyAny> = path.call_method( + "open", + ("w",), + Some(&keywords( + py, + &[ + ( + "encoding", + pyo3::types::PyString::new(py, "utf-8").into_any(), + ), + ("newline", pyo3::types::PyString::new(py, "\n").into_any()), + ], + )?), + )?; + within(&stream, |opened| match capture_text(py, value) { + Ok(text) => opened.call_method1("write", (text,)).map(|_| ()), + Err((written, error)) => { + if !written.is_empty() { + opened.call_method1("write", (written,))?; + } + Err(error) + } + }) +} + +/// Render a value as the JSON document a stage capture writes. +#[pyfunction] +fn render_stage_capture(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + capture_text(py, value).map_err(|(_, error)| error) +} + +/// Mask invocation ids, native store suffixes and refactor staging folders of a compile. +#[pyfunction] +fn mask_capture_noise(text: &str) -> String { + masked(text) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let omitted: Bound<'_, PyDict> = PyDict::new(py); + for (type_name, names) in OMITTED_ATTRIBUTES { + omitted.set_item(type_name, PyFrozenSet::new(py, names.iter())?)?; + } + module.add("STAGE_CAPTURE_OMITTED_ATTRIBUTES", omitted)?; + module.add_function(wrap_pyfunction!(render_stage_capture, module)?)?; + module.add_function(wrap_pyfunction!(mask_capture_noise, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs new file mode 100644 index 0000000000..e8320dac1d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -0,0 +1,327 @@ +//! A `CompiledModel` for a model input built outside the native compile-input stage. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; + +type Object<'py> = Bound<'py, PyAny>; + +/// The compiled model of `model_input` from its session outcome, fallback rows and proof. +pub(crate) fn compiled_model<'py>( + model_input: &Object<'py>, + deps: &Object<'py>, + analysis: &Object<'py>, + macro_deps: &Object<'py>, +) -> PyResult> { + let py: Python<'py> = model_input.py(); + let native = py.import(NATIVE_MODULE)?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let types = py.import(COMPILE_TYPES_MODULE)?; + let model_file: Object<'py> = model_input.getattr("model_file")?; + let model_name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; + let none: Object<'py> = py.None().into_bound(py); + let mut inferred: Object<'py> = none.clone(); + let mut lineage: Object<'py> = none.clone(); + let mut has_star: Object<'py> = false_object(py); + let mut star_resolved: Object<'py> = false_object(py); + let mut diagnostics: Object<'py> = PyTuple::empty(py).into_any(); + let mut validated: Object<'py> = false_object(py); + let mut proof: Object<'py> = native + .getattr("dynamic_contract_proof")? + .call1((analysis.getattr("standalone_proof")?,))?; + let session: Object<'py> = analysis.getattr("session")?; + if !session.is_none() { + let (session, position): (Object<'py>, Object<'py>) = session.extract()?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("session", session)?; + arguments.set_item("position", position)?; + arguments.set_item("placeholders", &none)?; + let outcome: Object<'py> = native + .getattr("session_outcome_analysis")? + .call((), Some(&arguments))?; + let result: Object<'py> = outcome.getattr("polyglot_analysis")?; + if result.getattr("analysis_succeeded")?.is_truthy()? { + inferred = result.getattr("columns")?; + lineage = result.getattr("lineage_columns")?; + has_star = result.getattr("has_star")?; + star_resolved = result.getattr("star_resolved")?; + } else { + inferred = fallback_columns(&analysis.getattr("fallback")?)?; + } + let session_proof: Object<'py> = outcome.getattr("dynamic_column_contract")?; + if session_proof.is_truthy()? { + proof = session_proof; + } + validated = result.getattr("binding_validated")?; + let cleaned: Object<'py> = outcome.getattr("cleaned_sql")?; + let cleaned: Object<'py> = if cleaned.is_truthy()? { + cleaned + } else { + PyString::new(py, "").into_any() + }; + diagnostics = located( + model_input, + &result.getattr("binding_diagnostics")?, + &cleaned, + )?; + } + if !proof.is_none() && proof.getattr("output_proven")?.is_truthy()? { + inferred = proof.getattr("fixed_columns")?; + has_star = true_object(py); + } + let resource_type: Object<'py> = types.getattr("CompiledResourceType")?; + let key_class: Object<'py> = models.getattr("CompiledObjectKey")?; + let mut dependency_keys: Vec> = Vec::new(); + for dependency in deps.try_iter()? { + let (kind, name): (Object<'py>, Object<'py>) = dependency?.extract()?; + dependency_keys.push(call( + &key_class, + &[ + ("resource_type", resource_type.call1((kind,))?), + ("name", name), + ], + )?); + } + let config: Object<'py> = model_input.getattr("config")?; + let key: Object<'py> = call( + &key_class, + &[ + ("resource_type", resource_type.getattr("MODEL")?), + ("name", model_name.clone()), + ], + )?; + call( + &models.getattr("CompiledModel")?, + &[ + ("key", key), + ("deps", PyTuple::new(py, dependency_keys)?.into_any()), + ("name", model_name.clone()), + ("relative_path", model_file.getattr("relative_path")?), + ("query_sql", model_input.getattr("query_sql")?), + ("config", config.clone()), + ("destination", destination(&config, &model_name)?), + ("references", model_input.getattr("references")?), + ("schema_entry", model_input.getattr("schema_entry")?), + ("inferred_columns", inferred), + ("fast_lineage_columns", lineage), + ("fast_lineage_has_star", has_star), + ("fast_lineage_star_resolved", star_resolved), + ("authored_sql", model_file.getattr("contents")?), + ("authored_query_sql", model_file.getattr("query_sql")?), + ( + "output_column_locations", + model_file.getattr("output_column_locations")?, + ), + ( + "extract_implicit_alias_columns", + model_file.getattr("extract_implicit_alias_columns")?, + ), + ("macro_deps", macro_deps.clone()), + ( + "enum_declarations", + model_input.getattr("enum_declarations")?, + ), + ( + "constant_declarations", + model_input.getattr("constant_declarations")?, + ), + ("enum_columns", model_input.getattr("enum_columns")?), + ("binding_diagnostics", diagnostics), + ("binding_validated", validated), + ("dynamic_column_contract", proof), + ( + "rejected_sql_analysis_opt_out", + model_input.getattr("rejected_sql_analysis_opt_out")?, + ), + ], + ) +} + +fn false_object(py: Python<'_>) -> Object<'_> { + pyo3::types::PyBool::new(py, false).to_owned().into_any() +} + +fn true_object(py: Python<'_>) -> Object<'_> { + pyo3::types::PyBool::new(py, true).to_owned().into_any() +} + +/// A model's configured physical and logical destination. +fn destination<'py>(config: &Object<'py>, model_name: &Object<'py>) -> PyResult> { + let py: Python<'py> = config.py(); + let values: Object<'py> = config.getattr("values")?; + let text = |key: &str| -> PyResult>> { + let value: Object<'py> = values.call_method1("get", (key,))?; + Ok(value.is_instance_of::().then_some(value)) + }; + let none: Object<'py> = py.None().into_bound(py); + call( + &py.import(COMPILE_MODELS_MODULE)? + .getattr("CompiledRelationLocation")?, + &[ + ( + "database", + text("database")?.unwrap_or_else(|| none.clone()), + ), + ("schema", text("schema")?.unwrap_or_else(|| none.clone())), + ("name", text("alias")?.unwrap_or_else(|| model_name.clone())), + ("qualified_name", none.clone()), + ("logical_schema", config.getattr("logical_schema")?), + ("logical_database", config.getattr("logical_database")?), + ], + ) +} + +/// A failed analysis's parse-mode column rows as inferred columns, None without rows. +fn fallback_columns<'py>(rows: &Object<'py>) -> PyResult> { + let py: Python<'py> = rows.py(); + if rows.is_none() { + return Ok(rows.clone()); + } + let column_class: Object<'py> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability: Object<'py> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut columns: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (name, data_type, value): (Object<'py>, Object<'py>, Object<'py>) = row?.extract()?; + columns.push(call( + &column_class, + &[ + ("name", name), + ("type", data_type), + ("nullability", nullability.call1((value,))?), + ], + )?); + } + Ok(PyTuple::new(py, columns)?.into_any()) +} + +/// Binding diagnostics located in the authored model file. +fn located<'py>( + model_input: &Object<'py>, + diagnostics: &Object<'py>, + cleaned: &Object<'py>, +) -> PyResult> { + let py: Python<'py> = model_input.py(); + if !diagnostics.is_truthy()? { + return Ok(PyTuple::empty(py).into_any()); + } + let model_file: Object<'py> = model_input.getattr("model_file")?; + let path: Object<'py> = model_file.getattr("relative_path")?; + let mut rows: Vec> = Vec::new(); + for item in diagnostics.try_iter()? { + let item: Object<'py> = item?; + let fields: Vec> = [ + "code", "message", "line", "column", "start", "end", "severity", + ] + .into_iter() + .map(|name| item.getattr(name)) + .collect::>()?; + rows.push(PyTuple::new(py, fields)?.into_any()); + } + let texts: Object<'py> = PyTuple::new( + py, + [ + model_file.getattr("contents")?, + model_file.getattr("query_sql")?, + cleaned.clone(), + ], + )? + .into_any(); + let expansion: Object<'py> = model_input.getattr("sql_expansion")?; + let expansion_row: Object<'py> = if expansion.is_none() { + expansion + } else { + let mut passes: Vec> = Vec::new(); + for spans in expansion.getattr("passes")?.try_iter()? { + let mut rows: Vec> = Vec::new(); + for span in spans?.try_iter()? { + let span: Object<'py> = span?; + let fields: Vec> = + ["source_start", "source_end", "output_start", "output_end"] + .into_iter() + .map(|name| span.getattr(name)) + .collect::>()?; + rows.push(PyTuple::new(py, fields)?.into_any()); + } + passes.push(PyList::new(py, rows)?.into_any()); + } + PyTuple::new( + py, + [ + expansion.getattr("expanded_sql")?, + PyList::new(py, passes)?.into_any(), + ], + )? + .into_any() + }; + let located: Object<'py> = py + .import(NATIVE_MODULE)? + .getattr("locate_model_binding_diagnostics")? + .call1((PyList::new(py, rows)?, texts, expansion_row))?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let types = py.import(COMPILE_TYPES_MODULE)?; + let diagnostic_class: Object<'py> = models.getattr("CompilerDiagnostic")?; + let location_class: Object<'py> = py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?; + let phase: Object<'py> = types.getattr("DiagnosticPhase")?.getattr("COMPILE")?; + let model: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; + let name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; + let mut built: Vec> = Vec::new(); + for row in located.try_iter()? { + let (code, message, severity, line, column, span): ( + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + ) = row?.extract()?; + let location: Object<'py> = if span.is_none() { + span + } else { + call( + &location_class, + &[ + ("path", path.clone()), + ("line", span.get_item(0)?), + ("column", span.get_item(1)?), + ("end_line", span.get_item(2)?), + ("end_column", span.get_item(3)?), + ], + )? + }; + built.push(call( + &diagnostic_class, + &[ + ("phase", phase.clone()), + ("severity", severity), + ("code", code), + ("message", message), + ("resource_type", model.clone()), + ("resource_name", name.clone()), + ("path", path.clone()), + ("line", line), + ("column", column), + ("location", location), + ], + )?); + } + Ok(PyTuple::new(py, built)?.into_any()) +} + +fn call<'py>(callable: &Object<'py>, pairs: &[(&str, Object<'py>)]) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + arguments.set_item(*name, value)?; + } + callable.call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/mod.rs new file mode 100644 index 0000000000..2c8fe3d950 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/mod.rs @@ -0,0 +1 @@ +pub(crate) mod compiled_model; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs new file mode 100644 index 0000000000..008b35b5c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod module_digests; +pub(crate) mod registration; +pub(crate) mod store_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs new file mode 100644 index 0000000000..664899d77d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs @@ -0,0 +1,132 @@ +//! The module file digests a macro call store records and checks before reuse. + +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_cache::module_digests::main::module_digests_metadata::module_digests_metadata as metadata_text; +use sqlbuild_core::json::models::JsonValue; + +use crate::bindings::_helpers::cache::native_store::digest_files; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::macro_bridge::store_keys::text; + +/// The fields of one stored row: a path and its digest. +const MODULE_DIGEST_ROW_FIELDS: usize = 2; + +/// Content digest of each module file, or None when any file cannot be read. +#[pyfunction] +pub(crate) fn digest_module_files<'py>( + py: Python<'py>, + paths: &Bound<'py, PyAny>, +) -> PyResult>> { + let ordered: Bound<'py, PyDict> = PyDict::new(py); + for path in paths.try_iter()? { + ordered.set_item(path?, py.None())?; + } + let mut files: Vec = Vec::with_capacity(ordered.len()); + for path in ordered.keys() { + files.push(path.extract()?); + } + let digests: Vec> = digest_files(py, files)?; + if digests.iter().any(Option::is_none) { + return Ok(None); + } + let result: Bound<'py, PyDict> = PyDict::new(py); + for (path, digest) in ordered.keys().iter().zip(digests) { + if let Some(digest) = digest.filter(|digest| !digest.is_empty()) { + result.set_item(path, digest)?; + } + } + Ok(Some(result)) +} + +/// Encode module file digests for the store's metadata. +#[pyfunction] +fn module_digests_metadata<'py>( + py: Python<'py>, + digests: &Bound<'py, PyAny>, +) -> PyResult> { + let mut rows: Vec<(JsonValue, JsonValue)> = Vec::new(); + for item in digests.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + rows.push(( + text(item.get_item(0)?.downcast::()?)?, + text(item.get_item(1)?.downcast::()?)?, + )); + } + let encoded: String = metadata_text(rows).map_err(emit_error)?; + Ok(PyBytes::new(py, encoded.as_bytes())) +} + +/// Whether a decoded metadata row is a list of a path and a digest. +fn digest_row_check(row: &Bound<'_, PyAny>) -> PyResult { + let Ok(row) = row.downcast::() else { + return Ok(false); + }; + if row.len() != MODULE_DIGEST_ROW_FIELDS { + return Ok(false); + } + for field in row.iter() { + if !field.is_instance_of::() { + return Ok(false); + } + } + Ok(true) +} + +/// Return the stored module digests when every module file still has them, else None. +#[pyfunction] +#[pyo3(signature = (*, metadata, known))] +fn unchanged_module_digests<'py>( + py: Python<'py>, + metadata: &Bound<'py, PyAny>, + known: &Bound<'py, PyAny>, +) -> PyResult>> { + let rows: Bound<'py, PyAny> = match imported(py, "json", "loads")?.call1((metadata,)) { + Ok(rows) => rows, + Err(error) if error.is_instance_of::(py) => return Ok(None), + Err(error) => return Err(error), + }; + let Ok(rows) = rows.downcast::() else { + return Ok(None); + }; + for row in rows.iter() { + if !digest_row_check(&row)? { + return Ok(None); + } + } + let stored: Bound<'py, PyDict> = PyDict::new(py); + for row in rows.iter() { + stored.set_item(row.get_item(0)?, row.get_item(1)?)?; + } + let mut unknown: Vec> = Vec::new(); + for path in stored.keys() { + if !known.contains(&path)? { + unknown.push(path); + } + } + let Some(current) = digest_module_files(py, PyList::new(py, unknown)?.as_any())? else { + return Ok(None); + }; + for path in stored.keys() { + if known.contains(&path)? { + current.set_item(&path, known.get_item(&path)?)?; + } + } + Ok(if current.eq(&stored)? { + Some(stored) + } else { + None + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(digest_module_files, module)?)?; + module.add_function(wrap_pyfunction!(module_digests_metadata, module)?)?; + module.add_function(wrap_pyfunction!(unchanged_module_digests, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs new file mode 100644 index 0000000000..b483dec47b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs @@ -0,0 +1,10 @@ +//! Register the macro bridge's persistent store keys and module digests. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::macro_bridge::{module_digests, store_keys}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module_digests::register(module)?; + store_keys::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs new file mode 100644 index 0000000000..3a257f062e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs @@ -0,0 +1,332 @@ +//! Value-based text of everything a stored macro call result depends on besides its call text. + +use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyBool, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyStringMethods, + PyTuple, PyTypeMethods, +}; +use pyo3::{create_exception, pyfunction, wrap_pyfunction}; + +use sqlbuild_cache::macro_store_keys::main::canonical_sort_text::canonical_sort_text; +use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; +use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::frontier::encoder::emit_error; + +const VALUE_RENDERER_FIELD: &str = "_value_renderer"; +/// Nesting at which a value counts as too deep to key, as Python's recursion limit stops it. +const MAX_DEPTH: usize = 900; + +create_exception!(_native, UncacheableValue, pyo3::exceptions::PyException); + +/// Whether `error` is one that makes a value unkeyable rather than a failure to report. +fn unstable(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) +} + +/// `f"{kind.__module__}.{kind.__qualname__}"`. +fn type_name(kind: &Bound<'_, PyAny>) -> PyResult { + Ok(format!( + "{}.{}", + kind.getattr("__module__")?.str()?.to_cow()?, + kind.getattr("__qualname__")?.str()?.to_cow()? + )) +} + +/// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. +pub(crate) fn text(value: &Bound<'_, PyString>) -> PyResult { + if let Ok(text) = value.to_str() { + return Ok(JsonValue::String(text.to_owned())); + } + let encoded: Bound<'_, PyBytes> = value + .call_method1("encode", ("utf-16-le", "surrogatepass"))? + .downcast_into::()?; + Ok(JsonValue::Utf16( + pyo3::types::PyBytesMethods::as_bytes(&encoded) + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + )) +} + +/// `[tag, value]`. +fn tagged(tag: String, value: JsonValue) -> JsonValue { + JsonValue::Array(vec![JsonValue::String(tag), value]) +} + +/// The Python types `_canonical` distinguishes. +struct CanonicalTypes<'py> { + enumeration: Bound<'py, PyAny>, + decimal: Bound<'py, PyAny>, + dates: Bound<'py, PyTuple>, + timedelta: Bound<'py, PyAny>, + pure_path: Bound<'py, PyAny>, + mapping_proxy: Bound<'py, PyAny>, + is_dataclass: Bound<'py, PyAny>, + fields: Bound<'py, PyAny>, + type_type: Bound<'py, PyAny>, +} + +impl<'py> CanonicalTypes<'py> { + fn load(py: Python<'py>) -> PyResult { + Ok(Self { + enumeration: imported(py, "enum", "Enum")?, + decimal: imported(py, "decimal", "Decimal")?, + dates: PyTuple::new( + py, + [ + imported(py, "datetime", "date")?, + imported(py, "datetime", "time")?, + ], + )?, + timedelta: imported(py, "datetime", "timedelta")?, + pure_path: imported(py, "pathlib", "PurePath")?, + mapping_proxy: imported(py, "types", "MappingProxyType")?, + is_dataclass: imported(py, "dataclasses", "is_dataclass")?, + fields: imported(py, "dataclasses", "fields")?, + type_type: imported(py, "builtins", "type")?, + }) + } + + /// `_canonical(value)`. + fn canonical(&self, value: &Bound<'py, PyAny>, depth: usize) -> PyResult { + if depth >= MAX_DEPTH { + return Err(PyRecursionError::new_err( + "maximum recursion depth exceeded", + )); + } + let kind: Bound<'py, PyAny> = value.get_type().into_any(); + if value.is_instance(&self.enumeration)? { + return Ok(JsonValue::Array(vec![ + JsonValue::String("enum".to_owned()), + JsonValue::String(type_name(&kind)?), + self.canonical(&value.getattr("value")?, depth + 1)?, + ])); + } + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(pyo3::types::PyBoolMethods::is_true(flag))); + } + if let Ok(string) = value.downcast::() { + return text(string); + } + if value.is_instance_of::() { + return Ok(tagged( + "int".to_owned(), + JsonValue::String(value.str()?.to_cow()?.into_owned()), + )); + } + if value.is_instance_of::() { + return Ok(tagged( + "float".to_owned(), + JsonValue::String(value.call_method0("hex")?.extract()?), + )); + } + if value.is_instance(&self.decimal)? { + return Ok(tagged( + "decimal".to_owned(), + JsonValue::String(value.str()?.to_cow()?.into_owned()), + )); + } + if value.is_instance(self.dates.as_any())? { + return Ok(tagged( + type_name(&kind)?, + JsonValue::String(value.call_method0("isoformat")?.extract()?), + )); + } + if value.is_instance(&self.timedelta)? { + let mut items: Vec = vec![JsonValue::String("timedelta".to_owned())]; + for part in ["days", "seconds", "microseconds"] { + items.push(JsonValue::Integer(JsonInteger::from( + value.getattr(part)?.extract::()?, + ))); + } + return Ok(JsonValue::Array(items)); + } + if value.is_instance_of::() { + return Ok(tagged( + "bytes".to_owned(), + JsonValue::String(value.call_method0("hex")?.extract()?), + )); + } + if value.is_instance(&self.pure_path)? { + return Ok(tagged( + "path".to_owned(), + JsonValue::String(value.call_method0("as_posix")?.extract()?), + )); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(self.canonical(&item?, depth + 1)?); + } + return Ok(tagged(type_name(&kind)?, JsonValue::Array(items))); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut members: Vec = Vec::new(); + for item in value.try_iter()? { + let canonical: JsonValue = self.canonical(&item?, depth + 1)?; + members.push(canonical_sort_text(&canonical).map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!("{error:?}")) + })?); + } + members.sort(); + return Ok(tagged( + type_name(&kind)?, + JsonValue::Array(members.into_iter().map(JsonValue::String).collect()), + )); + } + if value.is_instance_of::() || value.is_instance(&self.mapping_proxy)? { + let mut pairs: Vec = Vec::new(); + for item in value.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + pairs.push(JsonValue::Array(vec![ + self.canonical(&item.get_item(0)?, depth + 1)?, + self.canonical(&item.get_item(1)?, depth + 1)?, + ])); + } + return Ok(tagged("mapping".to_owned(), JsonValue::Array(pairs))); + } + if self.is_dataclass.call1((value,))?.is_truthy()? && !value.is_instance(&self.type_type)? { + let mut fields: Vec = Vec::new(); + for field in self.fields.call1((value,))?.try_iter()? { + let name: Bound<'py, PyAny> = field?.getattr("name")?; + let item: Bound<'py, PyAny> = value.getattr(name.downcast::()?)?; + fields.push(JsonValue::Array(vec![ + text(name.downcast::()?)?, + self.canonical(&item, depth + 1)?, + ])); + } + return Ok(tagged(type_name(&kind)?, JsonValue::Array(fields))); + } + Err(UncacheableValue::new_err(type_name(&kind)?)) + } +} + +/// Name a macro by its name, file, identity and exact source, or None when it has no key. +#[pyfunction] +#[pyo3(signature = (**options))] +fn macro_store_token<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "macro_store_token", + options, + (&["loaded", "identity"], &[]), + )?; + let loaded: Bound<'py, PyAny> = arguments.value("loaded")?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let name: JsonValue = text(loaded.getattr("name")?.downcast::()?)?; + let path: JsonValue = text( + loaded + .getattr("relative_path")? + .call_method0("as_posix")? + .downcast::()?, + )?; + let identity: JsonValue = types.canonical(&arguments.value("identity")?, 0)?; + let raw_source: String = loaded.getattr("raw_source")?.extract()?; + macro_key((name, path), identity, &raw_source).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } +} + +/// Digest the whole context a macro may read, or None when a value has no stable text. +#[pyfunction] +#[pyo3(signature = (**options))] +fn context_store_token<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "context_store_token", + options, + (&["macro_context", "declarations"], &[]), + )?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let context: JsonValue = canonical_context(&types, &arguments.value("macro_context")?)?; + let declarations: Bound<'py, PyAny> = arguments.value("declarations")?; + let declared: JsonValue = if declarations.is_none() { + JsonValue::Null + } else { + let sorted = imported(py, "builtins", "sorted")?; + let mut parts: Vec = vec![ + types.canonical(&declarations.getattr("constants")?, 0)?, + types.canonical(&declarations.getattr("enums")?, 0)?, + ]; + for name in ["inaccessible_constants", "inaccessible_enums"] { + let mut items: Vec = Vec::new(); + for item in sorted.call1((declarations.getattr(name)?,))?.try_iter()? { + items.push(plain_json(&item?)?); + } + parts.push(JsonValue::Array(items)); + } + JsonValue::Array(parts) + }; + context_key(&JsonValue::Array(vec![context, declared])).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } +} + +/// `_canonical_context`: the context's type and each field, the value renderer by its type. +fn canonical_context<'py>( + types: &CanonicalTypes<'py>, + macro_context: &Bound<'py, PyAny>, +) -> PyResult { + let mut fields: Vec = Vec::new(); + for field in types.fields.call1((macro_context,))?.try_iter()? { + let name: Bound<'py, PyAny> = field?.getattr("name")?; + let name_text: String = name.extract()?; + let value: Bound<'py, PyAny> = macro_context.getattr(name_text.as_str())?; + let canonical: JsonValue = if name_text == VALUE_RENDERER_FIELD { + JsonValue::String(type_name(&value.get_type().into_any())?) + } else { + types.canonical(&value, 0)? + }; + fields.push(JsonValue::Array(vec![ + text(name.downcast::()?)?, + canonical, + ])); + } + Ok(JsonValue::Array(vec![ + JsonValue::String(type_name(¯o_context.get_type().into_any())?), + JsonValue::Array(fields), + ])) +} + +/// An inaccessible declaration name, which is always a string, as JSON. +fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { + match value.downcast::() { + Ok(string) => text(string), + Err(_) => Err(pyo3::exceptions::PyTypeError::new_err(format!( + "Object of type {} is not JSON serializable", + value.get_type().name()? + ))), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(macro_store_token, module)?)?; + module.add_function(wrap_pyfunction!(context_store_token, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 64b6b9f9d4..268eaac38c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -12,8 +12,11 @@ pub(crate) mod contracts; pub(crate) mod dag_artifact; pub(crate) mod discovery; pub(crate) mod fingerprints; +pub(crate) mod frontier; pub(crate) mod graph; +pub(crate) mod hand_built; pub(crate) mod lineage; +pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod project_assembly; @@ -23,6 +26,7 @@ pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; pub(crate) mod semantic_checks; +pub(crate) mod semantic_shapes; pub(crate) mod sql_test_glue; pub(crate) mod sqltext; pub(crate) mod type_system; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs deleted file mode 100644 index f75e461918..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Deferred target schema and database templates split natively around known variables. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments as segments; -use sqlbuild_model_config::templates::models::DeferredSegment; - -/// `(is_variable, text)` pieces of a deferred target field after context substitution. -#[pyfunction] -fn deferred_target_segments( - template: &str, - logical: &str, - context_keys: Vec, - variables: Vec, -) -> Vec<(bool, String)> { - segments(template, logical, &context_keys, &variables) - .into_iter() - .map(|segment| match segment { - DeferredSegment::Text(text) => (false, text), - DeferredSegment::Variable(name) => (true, name), - }) - .collect() -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(deferred_target_segments, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index ef4f1e6b7c..2f0dab4dca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod authored_nodes; pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; -pub(crate) mod deferred_targets; pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index afe3c15955..951a98b94b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,15 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_templates, deferred_targets, described_models, - model_config_builder, model_validator, + column_migrations, config_errors, config_templates, described_models, model_config_builder, + model_validator, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; config_templates::register(module)?; - deferred_targets::register(module)?; described_models::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index 64e06a54a3..a853c62be7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::semantic_checks::type_recovery::{ LineageInput, StageShared, diagnostic_ids, id_of, inferred_names, items, lineage_outputs, lineage_payload, }; +use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; const RECOVERED_CODE: &str = "B002"; @@ -43,7 +44,7 @@ pub(crate) fn completed_diagnostics<'py>( let shapes: ShapeRows = if diagnostics.is_empty() { Vec::new() } else { - shape_rows(&classes.inputs.call_method1("shapes", (project,))?)? + shape_rows(project_shapes(project, None)?.as_any())? }; let mut semantic: Vec = Vec::with_capacity(diagnostics.len()); for item in &diagnostics { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs new file mode 100644 index 0000000000..f8527f82c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -0,0 +1,155 @@ +//! Compile-owned inputs of the semantic completion stage: shapes, requests and SQL test columns. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const BINDING_REQUESTS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; +const IDENTIFIER_CASE_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._identifier_case"; +const RESOURCE_SQL_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.resource_sql"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +/// The column names a metadata config value lists: a name, or the names of a list or tuple. +pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); + } + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::new(); + for item in value.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if item.is_instance_of::() { + names.push(item.extract()?); + } + } + Ok(names) +} + +/// The inferred output column names of every model SQL test CTE body, by body. +pub(crate) fn sql_test_columns<'py>( + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, + model_test_payload: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let bodies: Bound<'py, PyDict> = PyDict::new(py); + for test in project.getattr("sql_tests")?.try_iter()? { + let payload: Bound<'py, PyAny> = test?.getattr("payload")?; + if !payload.is_instance(model_test_payload)? { + continue; + } + for cte in payload + .getattr("authored_ctes")? + .try_iter()? + .chain(payload.getattr("expected_ctes")?.try_iter()?) + { + bodies.set_item(cte?.getattr("sql_body")?, py.None())?; + } + } + let columns: Bound<'py, PyDict> = PyDict::new(py); + if bodies.is_empty() { + return Ok(columns); + } + let query_class: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeColumnQuery")?; + let mut queries: Vec> = Vec::with_capacity(bodies.len()); + for sql in bodies.keys() { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", sql)?; + arguments.set_item("mode", "batch")?; + arguments.set_item("recover_cte_facts", false)?; + queries.push(query_class.call((), Some(&arguments))?); + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("queries", PyTuple::new(py, queries)?)?; + arguments.set_item("profile", profile)?; + let analyses: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("infer_native_query_columns")? + .call((), Some(&arguments))?; + for (sql, analysis) in bodies.keys().iter().zip(analyses.try_iter()?) { + let analysis: Bound<'py, PyAny> = analysis?; + let mut names: Vec> = Vec::new(); + let inferred: Bound<'py, PyAny> = analysis.getattr("columns")?; + if !inferred.is_none() { + for column in inferred.try_iter()? { + names.push(column?.getattr("name")?); + } + } + columns.set_item(sql, PyTuple::new(py, names)?)?; + } + Ok(columns) +} + +/// Type recovery's request revalidating one failed model against `shapes`. +pub(crate) fn revalidation_request<'py>( + project: &Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let dialect: Bound<'py, PyAny> = project.getattr("sql_analysis_dialect")?; + let functions: Bound<'py, PyAny> = project.getattr("functions")?; + let schema: Bound<'py, PyDict> = PyDict::new(py); + for reference in model.getattr("references")?.try_iter()? { + let name: Bound<'py, PyAny> = reference?.getattr("ref_name")?; + let shape: Bound<'py, PyAny> = match shapes.get_item(&name)? { + Some(shape) => shape, + None => PyDict::new(py).into_any(), + }; + schema.set_item(name, shape)?; + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("query_sql", model.getattr("query_sql")?)?; + arguments.set_item("placeholders", py.None())?; + arguments.set_item("dialect", &dialect)?; + arguments.set_item( + "known_functions", + PyTuple::new(py, function_names(&functions)?)?, + )?; + arguments.set_item( + "known_types", + PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?, + )?; + arguments.set_item("binding_schema", schema)?; + let request: Bound<'py, PyAny> = py + .import(BINDING_REQUESTS_MODULE)? + .getattr("get_complete_schema_binding_request")? + .call((), Some(&arguments))?; + let case: Bound<'py, PyDict> = PyDict::new(py); + case.set_item("connection", project.getattr("effective_connection")?)?; + case.set_item("dialect", &dialect)?; + let ignores: Bound<'py, PyAny> = py + .import(IDENTIFIER_CASE_MODULE)? + .getattr("ignores_quoted_case")? + .call((), Some(&case))?; + let replaced: Bound<'py, PyDict> = PyDict::new(py); + replaced.set_item("catalog", project.getattr("binding_catalog")?)?; + replaced.set_item("quoted_identifiers_ignore_case", ignores)?; + py.import(DATACLASSES_MODULE)? + .getattr("replace")? + .call((request,), Some(&replaced)) +} + +/// The compile-time SQL checks of audits, SQL tests and SQL hooks. +pub(crate) fn resource_sql_diagnostics<'py>( + project: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("project", project)?; + arguments.set_item("shapes", shapes)?; + arguments.set_item("profile", profile)?; + py.import(RESOURCE_SQL_MODULE)? + .getattr("get_resource_sql_diagnostics")? + .call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index d155831cc8..b8446a8998 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -14,8 +14,12 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; +use crate::bindings::_helpers::semantic_checks::inputs::{ + column_names, resource_sql_diagnostics, sql_test_columns, +}; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::items; +use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; const METADATA_CONFIG_KEYS: [&str; 4] = [ @@ -93,14 +97,8 @@ pub(crate) fn metadata_diagnostics<'py>( )?); } if resource_sql_analysis { - let resource: Bound<'py, PyAny> = classes.call( - &classes.inputs.getattr("resource_sql_diagnostics")?, - &[ - ("project", project), - ("shapes", &inputs.shapes), - ("profile", profile), - ], - )?; + let resource: Bound<'py, PyAny> = + resource_sql_diagnostics(project, &inputs.shapes, profile)?; diagnostics.extend(items(&resource)?); } Ok(diagnostics) @@ -111,10 +109,7 @@ fn metadata_request<'py>( project: &Bound<'py, PyAny>, profile: &Bound<'py, PyAny>, ) -> PyResult<(MetadataInputs<'py>, MetadataRequest)> { - let shapes: Bound<'py, PyAny> = classes.call( - &classes.inputs.getattr("metadata_shapes")?, - &[("project", project), ("profile", profile)], - )?; + let shapes: Bound<'py, PyAny> = project_shapes(project, Some(profile.clone()))?.into_any(); let mut tests: Vec> = Vec::new(); for test in project.getattr("sql_tests")?.try_iter()? { let test: Bound<'py, PyAny> = test?; @@ -125,10 +120,8 @@ fn metadata_request<'py>( tests.push(test); } } - let columns_by_sql: Bound<'py, PyAny> = classes.call( - &classes.inputs.getattr("sql_test_columns")?, - &[("project", project), ("profile", profile)], - )?; + let columns_by_sql: Bound<'py, PyAny> = + sql_test_columns(project, profile, &classes.model_test_payload)?.into_any(); let functions: Vec = metadata_functions(project)?; let sources: Vec> = items(&project.getattr("sources")?)?; let (files, contents) = file_contents(classes.py(), &sources, &tests)?; @@ -423,29 +416,17 @@ fn text(value: &Bound<'_, PyAny>) -> PyResult> { Ok(None) } -fn column_names(classes: &StageClasses<'_>, value: &Bound<'_, PyAny>) -> PyResult> { - classes - .inputs - .call_method1("column_names", (value,))? - .try_iter()? - .map(|name| name?.extract()) - .collect() -} - fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyResult { let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; let get = |key: &str| values.call_method1("get", (key,)); let mut references: Vec = Vec::new(); for key in METADATA_CONFIG_KEYS { - references.push((key.to_owned(), column_names(classes, &get(key)?)?)); + references.push((key.to_owned(), column_names(&get(key)?)?)); } let custom: Bound<'_, PyAny> = get("config")?; if custom.is_instance_of::() { let partition: Bound<'_, PyAny> = custom.call_method1("get", ("partition_column",))?; - references.push(( - "partition_column".to_owned(), - column_names(classes, &partition)?, - )); + references.push(("partition_column".to_owned(), column_names(&partition)?)); } let tolerances: Bound<'_, PyAny> = get("row_diff_tolerances")?; if tolerances.is_instance_of::() { @@ -463,7 +444,7 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes if cursor_inputs.is_instance_of::() { for item in cursor_inputs.call_method0("items")?.try_iter()? { let (upstream, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - upstreams.push((upstream.str()?.extract()?, column_names(classes, &columns)?)); + upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } let mut calls_functions: bool = false; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index c5c67b1a80..5075c7d3f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod completion; pub(crate) mod failures; pub(crate) mod help_catalogue; +pub(crate) mod inputs; pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod resource_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 1fd931636b..f34834adf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -10,8 +10,6 @@ const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; -const COMPLETION_INPUTS_MODULE: &str = - "sqlbuild.compiler.compile.classes.semantic_completion_inputs"; const SCHEMA_VALIDATION_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._schema_validation"; const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -34,7 +32,6 @@ pub(crate) struct StageClasses<'py> { pub(crate) project_with: Bound<'py, PyAny>, pub(crate) with_type_recovery: Bound<'py, PyAny>, pub(crate) with_binding_diagnostics: Bound<'py, PyAny>, - pub(crate) inputs: Bound<'py, PyAny>, pub(crate) schema_validations: Bound<'py, PyAny>, pub(crate) normalize_type: Bound<'py, PyAny>, pub(crate) sql_analysis_key: Bound<'py, PyAny>, @@ -74,9 +71,6 @@ impl<'py> StageClasses<'py> { with_binding_diagnostics: py .import(BINDING_DIAGNOSTICS_MODULE)? .getattr("with_binding_diagnostics")?, - inputs: py - .import(COMPLETION_INPUTS_MODULE)? - .getattr("SemanticCompletionInputs")?, schema_validations: py .import(SCHEMA_VALIDATION_MODULE)? .getattr("get_schema_validations")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 85df3f79b7..1509c5c736 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -15,10 +15,12 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::{internal_error, semantic_error}; +use crate::bindings::_helpers::semantic_checks::inputs::revalidation_request; use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; const UNCHANGED: &str = "unchanged"; @@ -283,7 +285,7 @@ fn revalidated<'py>( return Ok(Vec::new()); } let py: Python<'py> = classes.py(); - let shapes: Bound<'py, PyAny> = classes.inputs.call_method1("shapes", (project,))?; + let shapes: Bound<'py, PyDict> = project_shapes(project, None)?; let unknown: Bound<'py, PyDict> = PyDict::new(py); for item in shapes.call_method0("items")?.try_iter()? { let (name, columns): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; @@ -301,14 +303,8 @@ fn revalidated<'py>( let project_models: Bound<'py, PyAny> = project.getattr("models")?; let mut revised: Vec> = Vec::with_capacity(models.len()); for index in models { - let request: Bound<'py, PyAny> = classes.call( - &classes.inputs.getattr("revalidation_request")?, - &[ - ("project", project), - ("model", &project_models.get_item(*index)?), - ("shapes", &unknown.clone().into_any()), - ], - )?; + let request: Bound<'py, PyAny> = + revalidation_request(project, &project_models.get_item(*index)?, &unknown)?; let result: Bound<'py, PyAny> = classes .call( &classes.schema_validations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs new file mode 100644 index 0000000000..eef36dc840 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -0,0 +1,128 @@ +//! Per-model reasons a semantic check was only partial, for machine reports. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; + +const TYPE_CHECKED_DIALECTS: [&str; 4] = ["duckdb", "snowflake", "postgresql", "bigquery"]; +const SOURCE_KIND: &str = "source"; +const OPEN_KINDS: [&str; 3] = [SOURCE_KIND, "ref", "dbt_ref"]; + +/// Partial-check reasons by model name; empty when SQL analysis is off. +#[pyfunction] +fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let reasons: Bound<'py, PyDict> = PyDict::new(py); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(reasons); + } + let shapes: Bound<'py, PyDict> = project_shapes(project, None)?; + let models: Vec> = project + .getattr("models")? + .try_iter()? + .collect::>()?; + let mut names: Vec = Vec::with_capacity(models.len()); + let mut references: Vec> = Vec::with_capacity(models.len()); + let mut blocked_by: HashMap> = HashMap::new(); + for model in &models { + let name: String = model.getattr("name")?.extract()?; + let mut rows: Vec<(String, String)> = Vec::new(); + for row in model.getattr("reference_rows")?.try_iter()? { + let row: Bound<'py, PyAny> = row?; + rows.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); + } + for item in model.getattr("binding_diagnostics")?.try_iter()? { + if item?.getattr("is_error")?.is_truthy()? { + blocked_by.insert(name.clone(), HashSet::from([name.clone()])); + break; + } + } + names.push(name); + references.push(rows); + } + if !blocked_by.is_empty() { + for _ in &names { + let mut changed: bool = false; + for (name, rows) in names.iter().zip(&references) { + let mut causes: HashSet = blocked_by.get(name).cloned().unwrap_or_default(); + for (_, upstream) in rows { + causes.extend(blocked_by.get(upstream).cloned().unwrap_or_default()); + } + if !causes.is_empty() && blocked_by.get(name) != Some(&causes) { + blocked_by.insert(name.clone(), causes); + changed = true; + } + } + if !changed { + break; + } + } + } + let dialect: Bound<'py, PyAny> = project.getattr("sql_analysis_dialect")?; + let dialect: String = if dialect.is_truthy()? { + dialect.extract()? + } else { + "generic".to_owned() + }; + for ((model, name), rows) in models.iter().zip(&names).zip(&references) { + let opted: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("sql_analysis",))?; + if opted.is(PyBool::new(py, false)) { + reasons.set_item(name, PyTuple::new(py, ["SQL analysis disabled"])?)?; + continue; + } + let mut partial: BTreeSet = BTreeSet::new(); + for (kind, upstream) in rows { + if OPEN_KINDS.contains(&kind.as_str()) && !shapes.contains(upstream)? { + partial.insert(if kind == SOURCE_KIND { + format!("open source: {upstream}") + } else { + format!("open input: {upstream}") + }); + } + } + if let Some(causes) = blocked_by.get(name) { + for cause in causes.iter().filter(|cause| *cause != name) { + partial.insert(format!("depends on invalid model {cause}")); + } + } + let inferred: Bound<'py, PyAny> = model.getattr("inferred_rows")?; + if !inferred.is_truthy()? { + partial.insert("output shape could not be inferred".to_owned()); + } else if model.getattr("fast_lineage_has_star")?.is_truthy()? && !shapes.contains(name)? { + partial.insert("unresolved star over open inputs".to_owned()); + } + if inferred.is_truthy()? { + for row in inferred.try_iter()? { + if row?.get_item(1)?.is_none() { + partial.insert("some output types are unknown".to_owned()); + break; + } + } + } + if !TYPE_CHECKED_DIALECTS.contains(&dialect.as_str()) { + partial.insert("type checking awaits dialect coercion support".to_owned()); + } + if !partial.is_empty() { + reasons.set_item(name, PyTuple::new(py, partial)?)?; + } + } + Ok(reasons) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_coverage, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs new file mode 100644 index 0000000000..5d1b6239e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs @@ -0,0 +1,74 @@ +//! The function names and declared types native binding treats as known. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const UNKNOWN: &str = "UNKNOWN"; + +/// Each declared function's name and its UDF and table-function analysis names, sorted. +pub(crate) fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for function in functions.try_iter()? { + let name: String = function?.getattr("name")?.extract()?; + names.insert(format!("__sqlbuild_udf_{name}")); + names.insert(format!("__sqlbuild_table_function_{name}")); + names.insert(name); + } + Ok(names.into_iter().collect()) +} + +/// Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. +pub(crate) fn declared_types( + functions: &Bound<'_, PyAny>, + column_types: &Bound<'_, PyAny>, +) -> PyResult> { + let mut types: BTreeSet = BTreeSet::new(); + for columns in column_types.call_method0("values")?.try_iter()? { + for column_type in columns?.call_method0("values")?.try_iter()? { + types.insert(column_type?.extract()?); + } + } + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + for argument in function.getattr("arguments")?.try_iter()? { + types.insert(argument?.getattr("type")?.extract()?); + } + let returns: Bound<'_, PyAny> = function.getattr("return_columns")?; + for column in returns.try_iter()? { + types.insert(column?.getattr("type")?.extract()?); + } + if !returns.is_truthy()? { + types.insert(function.getattr("returns")?.extract()?); + } + } + types.remove(UNKNOWN); + Ok(types.into_iter().collect()) +} + +/// `known_function_names` as a sorted tuple. +#[pyfunction] +fn known_function_names<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { + PyTuple::new(functions.py(), function_names(functions)?) +} + +/// `known_declared_types(*, functions, column_types)` as a sorted tuple. +#[pyfunction] +#[pyo3(signature = (**request))] +fn known_declared_types<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("known_declared_types", request)?; + let types: Vec = + declared_types(&request.get("functions")?, &request.get("column_types")?)?; + PyTuple::new(request.py(), types) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(known_function_names, module)?)?; + module.add_function(wrap_pyfunction!(known_declared_types, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs new file mode 100644 index 0000000000..9afb6d4c00 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod coverage; +pub(crate) mod declarations; +pub(crate) mod registration; +pub(crate) mod shapes; +pub(crate) mod source_bindings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs new file mode 100644 index 0000000000..8d3deb3d24 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs @@ -0,0 +1,12 @@ +//! Register the native relation shape, coverage and declaration functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::semantic_shapes::{coverage, declarations, shapes, source_bindings}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + shapes::register(module)?; + coverage::register(module)?; + source_bindings::register(module)?; + declarations::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs new file mode 100644 index 0000000000..3ba4880c79 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -0,0 +1,420 @@ +//! Closed relation shapes of a compiled project, as binding and semantic checks read them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const UNKNOWN: &str = "UNKNOWN"; +const GENERIC_DIALECT: &str = "generic"; +const VALID_FROM_DEFAULT: &str = "valid_from"; +const VALID_TO_DEFAULT: &str = "valid_to"; +const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; +const UDF_KIND: &str = "udf"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; + +/// The relation names `(kind, name, package, count)` reference rows bind, UDFs excluded. +pub(crate) fn relation_row_names<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let mut names: Vec = Vec::new(); + for row in rows.try_iter()? { + let row: Bound<'py, PyAny> = row?; + let kind: String = row.get_item(0)?.str()?.extract()?; + let name: String = row.get_item(1)?.extract()?; + if kind == UDF_KIND { + continue; + } + let name: String = if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_PREFIX}{name}") + } else { + name + }; + if !names.contains(&name) { + names.push(name); + } + } + Ok(names) +} + +/// `{name: shapes.get(name, {})}` for each relation a model's reference rows bind. +pub(crate) fn model_inputs<'py>( + model: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let inputs: Bound<'py, PyDict> = PyDict::new(py); + for name in relation_row_names(&model.getattr("reference_rows")?)? { + let shape: Bound<'py, PyAny> = match shapes.get_item(&name)? { + Some(shape) => shape, + None => PyDict::new(py).into_any(), + }; + inputs.set_item(name, shape)?; + } + Ok(inputs) +} + +/// A model relation's shape, with snapshot validity columns. +pub(crate) fn published_shape<'py>( + model: &Bound<'py, PyAny>, + columns: &Bound<'py, PyDict>, + inputs: &Bound<'py, PyDict>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let shape: Bound<'py, PyDict> = binding_shape(model, columns, inputs, profile)?; + let values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; + let snapshot: Bound<'py, PyAny> = py + .import(PLANNER_TYPES_MODULE)? + .getattr("MaterializationType")? + .getattr("SNAPSHOT")?; + if values + .call_method1("get", ("materialized",))? + .eq(snapshot)? + { + for (key, default) in [ + ("valid_from_column", VALID_FROM_DEFAULT), + ("valid_to_column", VALID_TO_DEFAULT), + ] { + let name: Bound<'py, PyAny> = values.call_method1("get", (key,))?; + let name: Bound<'py, PyAny> = if name.is_truthy()? { + name + } else { + PyString::new(py, default).into_any() + }; + shape.set_item(name.str()?, UNKNOWN)?; + } + } + Ok(shape) +} + +/// Inferred columns, re-quoted from authored SQL where the dialect keeps quoted case. +fn binding_shape<'py>( + model: &Bound<'py, PyAny>, + columns: &Bound<'py, PyDict>, + inputs: &Bound<'py, PyDict>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let sensitive: bool = py + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("CASE_SENSITIVE_BINDING_DIALECTS")? + .contains(&dialect)?; + if profile + .getattr("quoted_identifiers_ignore_case")? + .is_truthy()? + || !sensitive + { + return Ok(columns.clone()); + } + let mut catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; + if !catalog.is_truthy()? { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("dialect", or_generic(&dialect)?)?; + arguments.set_item("quoted_ignore_case", false)?; + arguments.set_item( + "known_functions", + profile.getattr("semantic_known_functions")?, + )?; + arguments.set_item("known_types", profile.getattr("semantic_known_types")?)?; + arguments.set_item("relations", inputs)?; + catalog = py + .import(BINDING_CATALOG_MODULE)? + .getattr("create_binding_catalog")? + .call((), Some(&arguments))?; + } + let request: Bound<'py, PyDict> = PyDict::new(py); + request.set_item("sql", model.getattr("query_sql")?)?; + request.set_item("dialect", or_generic(&dialect)?)?; + request.set_item("stubs", PyDict::new(py))?; + request.set_item("placeholders", PyDict::new(py))?; + let sql: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("normalize_analysis_sql")? + .call1((request,))?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", sql)?; + arguments.set_item("columns", columns)?; + arguments.set_item("inputs", inputs)?; + Ok(catalog + .getattr("inferred_schema")? + .call((), Some(&arguments))? + .downcast_into::()?) +} + +fn or_generic<'py>(dialect: &Bound<'py, PyAny>) -> PyResult> { + if dialect.is_truthy()? { + return Ok(dialect.clone()); + } + Ok(PyString::new(dialect.py(), GENERIC_DIALECT).into_any()) +} + +/// `{name: type or "UNKNOWN"}` over `(name, type, ...)` rows. +fn typed_columns<'py>(py: Python<'py>, rows: &Bound<'py, PyAny>) -> PyResult> { + let columns: Bound<'py, PyDict> = PyDict::new(py); + for row in rows.try_iter()? { + let row: Bound<'py, PyAny> = row?; + columns.set_item(row.get_item(0)?, or_unknown(&row.get_item(1)?)?)?; + } + Ok(columns) +} + +fn or_unknown<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_truthy()? { + return Ok(value.clone()); + } + Ok(PyString::new(value.py(), UNKNOWN).into_any()) +} + +/// `{column.name: column.type or "UNKNOWN"}`. +fn declared_columns<'py>( + py: Python<'py>, + columns: &Bound<'py, PyAny>, +) -> PyResult> { + let shape: Bound<'py, PyDict> = PyDict::new(py); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + shape.set_item( + column.getattr("name")?, + or_unknown(&column.getattr("type")?)?, + )?; + } + Ok(shape) +} + +/// The profile shapes are read under, bound to the project's catalog. +pub(crate) fn project_profile<'py>( + project: &Bound<'py, PyAny>, + profile: Option>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let profile: Bound<'py, PyAny> = match profile { + Some(profile) if profile.is_truthy()? => profile, + _ => { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item( + "sql_analysis_dialect", + project.getattr("sql_analysis_dialect")?, + )?; + py.import(ADAPTER_MODELS_MODULE)? + .getattr("ExpressionInferenceProfile")? + .call((), Some(&arguments))? + } + }; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("binding_catalog", project.getattr("binding_catalog")?)?; + py.import(DATACLASSES_MODULE)? + .getattr("replace")? + .call((profile,), Some(&arguments)) +} + +/// Closed shapes by relation name; absence means open, never an empty physical table. +pub(crate) fn project_shapes<'py>( + project: &Bound<'py, PyAny>, + profile: Option>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let profile: Bound<'py, PyAny> = project_profile(project, profile)?; + let shapes: Bound<'py, PyDict> = PyDict::new(py); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + shapes.set_item( + seed.getattr("name")?, + declared_columns(py, &seed.getattr("schema_entry")?.getattr("columns")?)?, + )?; + } + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let returns: Bound<'py, PyAny> = function.getattr("return_columns")?; + if returns.is_truthy()? { + let shape: Bound<'py, PyDict> = PyDict::new(py); + for column in returns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + shape.set_item(column.getattr("name")?, column.getattr("type")?)?; + } + let name: String = function.getattr("name")?.extract()?; + shapes.set_item(format!("{TABLE_FUNCTION_PREFIX}{name}"), shape)?; + } + } + let enforced: Bound<'py, PyAny> = py + .import(PLANNER_TYPES_MODULE)? + .getattr("ContractPolicy")? + .getattr("ENFORCED")?; + let mut expression_names: Vec> = Vec::new(); + let mut expressions: Vec> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + let entry: Bound<'py, PyAny> = source.getattr("source_entry")?; + if entry.getattr("contract")?.eq(&enforced)? { + shapes.set_item( + source.getattr("name")?, + declared_columns(py, &entry.getattr("columns")?)?, + )?; + } else { + let expression: Bound<'py, PyAny> = entry.getattr("expression")?; + if expression.is_truthy()? { + expression_names.push(source.getattr("name")?); + expressions.push(expression); + } + } + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("expressions", PyTuple::new(py, expressions)?)?; + arguments.set_item("profile", &profile)?; + let source_shapes: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("infer_native_expression_source_shapes")? + .call((), Some(&arguments))?; + for (name, shape) in expression_names.iter().zip(source_shapes.try_iter()?) { + let shape: Bound<'py, PyAny> = shape?; + if !shape.is_none() { + shapes.set_item(name, shape)?; + } + } + model_shapes(project, &profile, &enforced, &shapes)?; + Ok(shapes) +} + +/// Publish model shapes in dependency rounds until no pending model can close. +fn model_shapes<'py>( + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, + enforced: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = project.py(); + let mut pending: Vec> = project + .getattr("models")? + .try_iter()? + .collect::>()?; + while !pending.is_empty() { + let mut remaining: Vec> = Vec::new(); + for model in &pending { + let declared: Bound<'py, PyAny> = model.getattr("schema_column_rows")?; + let inferred: Bound<'py, PyAny> = model.getattr("inferred_rows")?; + let rows: Bound<'py, PyAny> = model.getattr("reference_rows")?; + let relations: Vec = relation_row_names(&rows)?; + let contract: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("contract",))?; + if contract.eq(enforced)? + && !declared.is_none() + && !model.getattr("has_dynamic_columns")?.is_truthy()? + { + let unchecked: Bound<'py, PyAny> = model.getattr("unchecked_output_columns")?; + let shape: Bound<'py, PyDict> = PyDict::new(py); + for row in declared.try_iter()? { + let row: Bound<'py, PyAny> = row?; + let name: Bound<'py, PyAny> = row.get_item(0)?; + let column_type: Bound<'py, PyAny> = if unchecked.contains(&name)? { + PyString::new(py, UNKNOWN).into_any() + } else { + or_unknown(&row.get_item(1)?)? + }; + shape.set_item(name, column_type)?; + } + shapes.set_item(model.getattr("name")?, shape)?; + continue; + } + let closes: bool = inferred.is_truthy()? + && (!model.getattr("fast_lineage_has_star")?.is_truthy()? + || (model.getattr("fast_lineage_star_resolved")?.is_truthy()? + && rows.is_truthy()? + && all_known(&relations, shapes)?)); + if closes { + let inputs: Bound<'py, PyDict> = model_inputs(model, shapes)?; + let columns: Bound<'py, PyDict> = typed_columns(py, &inferred)?; + shapes.set_item( + model.getattr("name")?, + published_shape(model, &columns, &inputs, profile)?, + )?; + } else { + remaining.push(model.clone()); + } + } + if remaining.len() == pending.len() { + break; + } + pending = remaining; + } + Ok(()) +} + +fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { + for name in names { + if !shapes.contains(name)? { + return Ok(false); + } + } + Ok(true) +} + +/// Closed relation shapes; `profile` defaults to the project's dialect. +#[pyfunction] +#[pyo3(signature = (**request))] +fn semantic_shapes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("semantic_shapes", request)?; + project_shapes(&request.get("project")?, request.optional("profile")?) +} + +/// The analysis names the references bind, UDFs excluded. +#[pyfunction] +fn binding_relation_names<'py>( + references: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = references.py(); + let mut rows: Vec> = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + rows.push( + PyTuple::new( + py, + [ + reference.getattr("ref_kind")?.str()?.into_any(), + reference.getattr("ref_name")?, + ], + )? + .into_any(), + ); + } + let names: Vec = relation_row_names(&PyTuple::new(py, rows)?.into_any())?; + PyFrozenSet::new(py, names) +} + +/// A model relation's published shape: `published_model_shape(*, model, columns, inputs, profile)`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn published_model_shape<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("published_model_shape", request)?; + published_shape( + &request.get("model")?, + request.get("columns")?.downcast()?, + request.get("inputs")?.downcast()?, + &request.get("profile")?, + ) +} + +/// `{name: shapes.get(name, {})}` for each relation the model's reference rows bind. +#[pyfunction] +fn model_binding_inputs<'py>( + model: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, +) -> PyResult> { + model_inputs(model, shapes) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_shapes, module)?)?; + module.add_function(wrap_pyfunction!(published_model_shape, module)?)?; + module.add_function(wrap_pyfunction!(model_binding_inputs, module)?)?; + module.add_function(wrap_pyfunction!(binding_relation_names, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs new file mode 100644 index 0000000000..7e7e6c2622 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -0,0 +1,506 @@ +//! Rebind selected models to source columns the planner already inspected. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; +use crate::bindings::_helpers::semantic_shapes::shapes::{ + model_inputs, project_shapes, published_shape, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const IDENTIFIER_CASE_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._identifier_case"; +const SCHEMA_VALIDATION_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._schema_validation"; +const BINDING_REQUESTS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const GRAPHLIB_MODULE: &str = "graphlib"; +const REF_KIND: &str = "ref"; +const SOURCE_KIND: &str = "source"; +const UNKNOWN: &str = "UNKNOWN"; +const GENERIC_DIALECT: &str = "generic"; + +type Object<'py> = Bound<'py, PyAny>; + +/// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn source_binding_diagnostics<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("source_binding_diagnostics", request)?; + let py: Python<'py> = request.py(); + let project: Object<'py> = request.get("project")?; + let columns: Object<'py> = request.get("columns")?; + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + || !columns.is_truthy()? + { + return Ok(PyTuple::empty(py)); + } + let constants = py.import(SQL_ANALYSIS_CONSTANTS_MODULE)?; + let profile: Object<'py> = request.get("profile")?; + let raw_dialect: Object<'py> = profile.getattr("sql_analysis_dialect")?; + let key: Object<'py> = if raw_dialect.is_truthy()? { + raw_dialect.clone() + } else { + PyString::new(py, GENERIC_DIALECT).into_any() + }; + let dialect: Object<'py> = constants + .getattr("NATIVE_DIALECT_ALIASES")? + .call_method1("get", (key, &raw_dialect))?; + let ignore_case: Object<'py> = ignores_quoted_case(&project, &raw_dialect)?; + let profile: Object<'py> = replaced( + &profile, + &[ + ("sql_analysis_dialect", dialect.clone()), + ("quoted_identifiers_ignore_case", ignore_case.clone()), + ], + )?; + let shapes: Bound<'py, PyDict> = project_shapes(&project, Some(profile.clone()))?; + let sensitive: bool = constants + .getattr("CASE_SENSITIVE_BINDING_DIALECTS")? + .contains(&dialect)? + && !ignore_case.is_truthy()?; + let mut changed: HashSet = HashSet::new(); + for item in columns.call_method0("items")?.try_iter()? { + let (name, value): (String, Object<'py>) = item?.extract()?; + if !value.is_truthy()? { + continue; + } + let physical: Vec<(String, Option)> = value + .try_iter()? + .map(|column| { + let column: Object<'py> = column?; + Ok(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )) + }) + .collect::>()?; + if sensitive || adds_physical_evidence(&physical, shapes.get_item(&name)?.as_ref())? { + changed.insert(name.clone()); + } + let quoted: Bound<'py, PyDict> = PyDict::new(py); + for (column, column_type) in physical { + quoted.set_item(format!("\"{}\"", column.replace('"', "\"\"")), column_type)?; + } + shapes.set_item(name, quoted)?; + } + let binding_catalog: Object<'py> = project.getattr("binding_catalog")?; + let mut physical_catalog: Object<'py> = if binding_catalog.is_none() { + binding_catalog + } else { + binding_catalog.call_method1("with_relations", (&shapes,))? + }; + let profile: Object<'py> = + replaced(&profile, &[("binding_catalog", physical_catalog.clone())])?; + let models: Vec> = rebound_models( + &project, + &request.get("selected_keys")?, + (&changed, &shapes), + &profile, + )?; + if !physical_catalog.is_none() { + physical_catalog = physical_catalog.call_method1("with_relations", (&shapes,))?; + } + validated_diagnostics(&project, &models, (&shapes, &physical_catalog), &dialect) +} + +/// Selected models reading changed sources, re-analysed in dependency order; shapes updated. +fn rebound_models<'py>( + project: &Object<'py>, + selected_keys: &Object<'py>, + (changed, shapes): (&HashSet, &Bound<'py, PyDict>), + profile: &Object<'py>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + let by_name: Bound<'py, PyDict> = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model: Object<'py> = model?; + by_name.set_item(model.getattr("name")?, &model)?; + } + let graph: Bound<'py, PyDict> = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model: Object<'py> = model?; + let mut upstream: Vec> = Vec::new(); + for reference in model.getattr("references")?.try_iter()? { + let reference: Object<'py> = reference?; + let name: Object<'py> = reference.getattr("ref_name")?; + if reference.getattr("ref_kind")?.eq(REF_KIND)? && by_name.contains(&name)? { + upstream.push(name); + } + } + graph.set_item(model.getattr("name")?, PyTuple::new(py, upstream)?)?; + } + let order: Object<'py> = py + .import(GRAPHLIB_MODULE)? + .getattr("TopologicalSorter")? + .call1((graph,))? + .call_method0("static_order")?; + let query_class: Object<'py> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeColumnQuery")?; + let unknown: Object<'py> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")? + .getattr("UNKNOWN")?; + let infer: Object<'py> = py + .import(NATIVE_MODULE)? + .getattr("infer_native_query_columns")?; + let mut affected: HashSet = HashSet::new(); + let mut models: Vec> = Vec::new(); + for name in order.try_iter()? { + let model: Object<'py> = by_name.as_any().get_item(name?)?; + let opted: Object<'py> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("sql_analysis",))?; + if !selected_keys.contains(model.getattr("key")?)? || opted.is(PyBool::new(py, false)) { + continue; + } + let mut reads_changed: bool = false; + for reference in model.getattr("references")?.try_iter()? { + let reference: Object<'py> = reference?; + let kind: String = reference.getattr("ref_kind")?.str()?.extract()?; + let upstream: String = reference.getattr("ref_name")?.extract()?; + if (kind == SOURCE_KIND && changed.contains(&upstream)) + || (kind == REF_KIND && affected.contains(&upstream)) + { + reads_changed = true; + break; + } + } + if !reads_changed { + continue; + } + let model_name: String = model.getattr("name")?.extract()?; + affected.insert(model_name.clone()); + models.push(model.clone()); + let inputs: Bound<'py, PyDict> = model_inputs(&model, shapes)?; + let nullability: Bound<'py, PyDict> = PyDict::new(py); + for (table, shape) in inputs.iter() { + let defaults: Object<'py> = PyDict::new(py) + .into_any() + .getattr("fromkeys")? + .call1((shape, &unknown))?; + nullability.set_item(table, defaults)?; + } + let query: Object<'py> = call( + &query_class, + &[ + ("sql", model.getattr("query_sql")?), + ("mode", PyString::new(py, "reanalysis").into_any()), + ("placeholders", placeholders(&model)?), + ("references", model.getattr("references")?), + ], + )?; + let analysis: Object<'py> = call( + &infer, + &[ + ("queries", PyTuple::new(py, [query])?.into_any()), + ("profile", profile.clone()), + ("column_types_by_table", inputs.clone().into_any()), + ("column_nullability_by_table", nullability.into_any()), + ], + )? + .get_item(0)?; + let inferred: Object<'py> = analysis.getattr("columns")?; + let mut all_inputs_closed: bool = true; + for shape in inputs.values() { + if !shape.is_truthy()? { + all_inputs_closed = false; + break; + } + } + if inferred.is_truthy()? + && (!analysis.getattr("has_star")?.is_truthy()? || all_inputs_closed) + { + let columns: Bound<'py, PyDict> = PyDict::new(py); + for column in inferred.try_iter()? { + let column: Object<'py> = column?; + let column_type: Object<'py> = column.getattr("type")?; + let column_type: Object<'py> = if column_type.is_truthy()? { + column_type + } else { + PyString::new(py, UNKNOWN).into_any() + }; + columns.set_item(column.getattr("name")?, column_type)?; + } + shapes.set_item( + &model_name, + published_shape(&model, &columns, &inputs, profile)?, + )?; + } else if shapes.contains(&model_name)? { + shapes.del_item(&model_name)?; + } + } + Ok(models) +} + +/// Each rebound model's binding diagnostics, located in its authored file. +fn validated_diagnostics<'py>( + project: &Object<'py>, + models: &[Object<'py>], + (shapes, catalog): (&Bound<'py, PyDict>, &Object<'py>), + dialect: &Object<'py>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let functions: Object<'py> = project.getattr("functions")?; + let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); + let known_types: Object<'py> = + PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); + let request_builder: Object<'py> = py + .import(BINDING_REQUESTS_MODULE)? + .getattr("get_complete_schema_binding_request")?; + let cursor_sql: Object<'py> = py + .import(CURSOR_INTRINSICS_MODULE)? + .getattr("cursor_intrinsics_analysis_sql")?; + let ignores: Object<'py> = ignores_quoted_case(project, dialect)?; + let mut requests: Vec> = Vec::with_capacity(models.len()); + for model in models { + let query_sql: Object<'py> = call( + &cursor_sql, + &[ + ("sql", model.getattr("query_sql")?), + ( + "cursor_type", + model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("cursor_type",))?, + ), + ], + )?; + let request: Object<'py> = call( + &request_builder, + &[ + ("known_functions", known_functions.clone()), + ("known_types", known_types.clone()), + ("query_sql", query_sql), + ("placeholders", placeholders(model)?), + ("dialect", dialect.clone()), + ("binding_schema", model_inputs(model, shapes)?.into_any()), + ], + )?; + requests.push(replaced( + &request, + &[ + ("catalog", catalog.clone()), + ("quoted_identifiers_ignore_case", ignores.clone()), + ], + )?); + } + let results: Object<'py> = call( + &py.import(SCHEMA_VALIDATION_MODULE)? + .getattr("get_schema_validations")?, + &[("requests", PyTuple::new(py, &requests)?.into_any())], + )?; + let expansions: Bound<'py, PyDict> = PyDict::new(py); + for item in project + .getattr("sql_expansions")? + .call_method0("items")? + .try_iter()? + { + let (path, expansion): (Object<'py>, Object<'py>) = item?.extract()?; + expansions.set_item(path.getattr("stem")?, expansion)?; + } + let skipped: Object<'py> = py + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("BINDING_UNKNOWN_TABLE_INTERNAL_CODE")?; + let models_module = py.import(COMPILE_MODELS_MODULE)?; + let types = py.import(COMPILE_TYPES_MODULE)?; + let diagnostic_class: Object<'py> = models_module.getattr("CompilerDiagnostic")?; + let severity_class: Object<'py> = types.getattr("DiagnosticSeverity")?; + let phase: Object<'py> = types.getattr("DiagnosticPhase")?.getattr("COMPILE")?; + let model_type: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; + let mut diagnostics: Vec> = Vec::new(); + for ((model, request), result) in models.iter().zip(&requests).zip(results.try_iter()?) { + let result: Object<'py> = result?; + let expansion: Option> = expansions.get_item(model.getattr("name")?)?; + let mut mapper: Option> = None; + for diagnostic in result.getattr("diagnostics")?.try_iter()? { + let diagnostic: Object<'py> = diagnostic?; + if diagnostic.getattr("code")?.eq(&skipped)? { + continue; + } + let built: Object<'py> = match mapper.take() { + Some(built) => built, + None => positions(model, &request.getattr("sql")?, expansion.as_ref())?, + }; + let position: Object<'py> = built.getattr("position")?; + mapper = Some(built); + let (line, column): (Object<'py>, Object<'py>) = call( + &position, + &[ + ("start", diagnostic.getattr("start")?), + ("line", diagnostic.getattr("line")?), + ("column", diagnostic.getattr("column")?), + ("message", diagnostic.getattr("message")?), + ], + )? + .extract()?; + diagnostics.push(call( + &diagnostic_class, + &[ + ("phase", phase.clone()), + ( + "severity", + severity_class.call1((diagnostic.getattr("severity")?,))?, + ), + ("code", diagnostic.getattr("code")?), + ("message", diagnostic.getattr("message")?), + ("resource_type", model_type.clone()), + ("resource_name", model.getattr("name")?), + ("path", model.getattr("relative_path")?), + ("line", line), + ("column", column), + ], + )?); + } + } + PyTuple::new(py, diagnostics) +} + +/// The native position mapper from cleaned binding SQL back to the authored model file. +fn positions<'py>( + model: &Object<'py>, + cleaned: &Object<'py>, + expansion: Option<&Object<'py>>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let authored_query: Object<'py> = model.getattr("authored_query_sql")?; + let mut passes: Vec> = Vec::new(); + let expanded: Object<'py> = match expansion { + None => authored_query.clone(), + Some(expansion) => { + for spans in expansion.getattr("passes")?.try_iter()? { + let mut rows: Vec> = Vec::new(); + for span in spans?.try_iter()? { + let span: Object<'py> = span?; + let fields: Vec> = + ["source_start", "source_end", "output_start", "output_end"] + .into_iter() + .map(|name| span.getattr(name)) + .collect::>()?; + rows.push(PyTuple::new(py, fields)?.into_any()); + } + passes.push(PyTuple::new(py, rows)?.into_any()); + } + expansion.getattr("expanded_sql")? + } + }; + let request: Bound<'py, PyDict> = PyDict::new(py); + request.set_item("authored", model.getattr("authored_sql")?)?; + request.set_item("query", authored_query)?; + request.set_item("expanded", expanded)?; + request.set_item("cleaned", cleaned)?; + request.set_item("passes", passes)?; + py.import(NATIVE_MODULE)? + .getattr("BindingPositions")? + .call1((request,)) +} + +/// Whether inspected columns differ from the known shape; untyped names match any type. +fn adds_physical_evidence( + physical: &[(String, Option)], + known: Option<&Object<'_>>, +) -> PyResult { + let Some(known) = known else { + return Ok(true); + }; + let known: Vec<(String, String)> = known + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?; + let physical_names: HashSet<&str> = physical.iter().map(|(name, _)| name.as_str()).collect(); + let known_names: HashSet<&str> = known.iter().map(|(name, _)| name.as_str()).collect(); + if physical_names != known_names { + return Ok(true); + } + for (name, physical_type) in physical { + let Some(physical_type) = physical_type.as_deref() else { + continue; + }; + if physical_type.is_empty() { + continue; + } + let known_type: Option<&String> = known + .iter() + .rev() + .find(|(known_name, _)| known_name == name) + .map(|(_, known_type)| known_type); + if known_type.is_some_and(|known_type| known_type != physical_type) { + return Ok(true); + } + } + Ok(false) +} + +/// `{str(key): str(value)}` of a dict `placeholders` config, else None. +fn placeholders<'py>(model: &Object<'py>) -> PyResult> { + let py: Python<'py> = model.py(); + let values: Object<'py> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("placeholders",))?; + if !values.is_instance_of::() { + return Ok(py.None().into_bound(py)); + } + let placeholders: Bound<'py, PyDict> = PyDict::new(py); + for item in values.call_method0("items")?.try_iter()? { + let (key, value): (Object<'py>, Object<'py>) = item?.extract()?; + placeholders.set_item(key.str()?, value.str()?)?; + } + Ok(placeholders.into_any()) +} + +fn ignores_quoted_case<'py>(project: &Object<'py>, dialect: &Object<'py>) -> PyResult> { + call( + &project + .py() + .import(IDENTIFIER_CASE_MODULE)? + .getattr("ignores_quoted_case")?, + &[ + ("connection", project.getattr("effective_connection")?), + ("dialect", dialect.clone()), + ], + ) +} + +fn replaced<'py>(value: &Object<'py>, pairs: &[(&str, Object<'py>)]) -> PyResult> { + let py: Python<'py> = value.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + for (name, item) in pairs { + arguments.set_item(*name, item)?; + } + py.import(DATACLASSES_MODULE)? + .getattr("replace")? + .call((value,), Some(&arguments)) +} + +fn call<'py>(callable: &Object<'py>, pairs: &[(&str, Object<'py>)]) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + arguments.set_item(*name, value)?; + } + callable.call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(source_binding_diagnostics, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 0a1f0971de..96bf439dfa 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -6,8 +6,9 @@ use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, cli, compile_attachment, compile_driver, compile_outputs, compiled_project, contracts, dag_artifact, discovery, - fingerprints, graph, lineage, manifest, model_config, project_assembly, refactor, render, - rules, scopes, semantic_checks, sql_test_glue, sqltext, type_system, + fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, project_assembly, + refactor, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, + type_system, }; use crate::bindings::models; @@ -51,6 +52,7 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { type_system::registration::register(module)?; analysis_session::registration::register(module)?; semantic_checks::registration::register(module)?; + semantic_shapes::registration::register(module)?; contracts::registration::register(module)?; lineage::registration::register(module)?; fingerprints::registration::register(module)?; @@ -61,6 +63,8 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { compiled_project::registration::register(module)?; compile_attachment::registration::register(module)?; compile_driver::registration::register(module)?; + frontier::registration::register(module)?; + macro_bridge::registration::register(module)?; dag_artifact::registration::register(module)?; manifest::registration::register(module)?; Ok(()) diff --git a/python-boundary.toml b/python-boundary.toml index ac0f277ea9..1d21fac1af 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 124254 +remaining_lines = 120834 [[stays]] path = "executor" @@ -212,14 +212,6 @@ note = "Tolerant scope discovery records a failing declaration category as a fau location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." -[[handled_native_errors]] -location = "compiler/macro_bridge/_helpers/store_keys.py::macro_store_token" -note = "A macro without a stable key is not stored; no Python implementation runs." - -[[handled_native_errors]] -location = "compiler/macro_bridge/_helpers/store_keys.py::context_store_token" -note = "A context without a stable key is not stored; no Python implementation runs." - [[handled_native_errors]] location = "lint/_helpers/native_sql.py::_native_responses" note = "A native lint failure is reported as a lint error result, not recomputed in Python." diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py index c84e192236..3c07d85fe7 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py @@ -9,12 +9,8 @@ import sqlbuild._native as _native from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( - get_authored_binding_location, -) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic from sqlbuild.compiler.sql_analysis.types import NativeDiagnosticRow, NativePositionsModule -from sqlbuild.spec.contracts.models import SourceLocation def measure_diagnostic_compile( @@ -63,16 +59,24 @@ def measure_position_mapping(*, count: int, lines: int = 4000) -> float: diagnostic: SqlBindingDiagnostic = SqlBindingDiagnostic( code="B217", message="Incompatible comparison", start=offset, end=offset + len("amount > 5") ) + mapper: Any = cast(NativePositionsModule, _native).BindingPositions( + { + "authored": original, + "query": original, + "expanded": original, + "cleaned": normalized, + "passes": [], + } + ) started: float = perf_counter() for _ in range(count): - location: SourceLocation | None = get_authored_binding_location( - path=Path("models/orders.sql"), - authored_sql=original, - authored_query_sql=original, - cleaned_sql=normalized, - diagnostic=diagnostic, + line, _column = mapper.position( + start=diagnostic.start, + line=diagnostic.line, + column=diagnostic.column, + message=diagnostic.message, ) - if location is None or location.line != lines + 2: + if line != lines + 2: raise CompileBenchmarkFixtureError( "mapping must preserve the authored comparison location" ) diff --git a/scripts/compiler_differential/_helpers/comparing/normalize.py b/scripts/compiler_differential/_helpers/comparing/normalize.py index ae1ecac43c..c3b9d57f4d 100644 --- a/scripts/compiler_differential/_helpers/comparing/normalize.py +++ b/scripts/compiler_differential/_helpers/comparing/normalize.py @@ -3,7 +3,7 @@ from __future__ import annotations from scripts.compiler_differential.constants import ELAPSED_TIME_MASK, ELAPSED_TIME_PATTERN -from sqlbuild.compiler.frontier.classes.stage_capture_encoder import mask_capture_noise +from sqlbuild._native import mask_capture_noise def normalize_artifact_text(text: str) -> str: diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index c7e4576ccd..0730f7136e 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -26,8 +26,8 @@ ANALYSIS_UPSTREAM_CHAIN_HOPS, COMPILED_PROJECT_CAPTURE_TYPE, ) +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _SET_OPERATION: re.Pattern[str] = re.compile(r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE) _LEADING_CTE: re.Pattern[str] = re.compile(r"^\s*WITH\s+\w+\s+AS\s*\(", re.IGNORECASE) diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index 31ef17aada..b4febeba93 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -35,9 +35,9 @@ STRATEGY_HEADER_KEYS, TAB, ) +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _OMITTED_DISCOVERY_FIELDS: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( f"{DiscoveredProjectInputs.__module__}:{DiscoveredProjectInputs.__qualname__}", frozenset() diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 8ea156ca18..3b20207b54 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -30,9 +30,9 @@ RENDER_NON_COLLECTION_FIELDS, RENDER_STAGE_CAPTURE_SUFFIX, ) +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_NAMES from sqlbuild.compiler.compile.models import CompileProjectInputs -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") _REFERENCE_CALL: re.Pattern[str] = re.compile(r"__(ref|source|seed)\(\s*\"([^\"]+)\"\s*\)") diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 42d2504d8b..9207da6f01 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -544,7 +544,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: DIAGNOSTIC_CODE_PATTERN: re.Pattern[str] = re.compile(r"[A-Z][0-9]{3}") RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012", "P004", "P010", "P011"}) RENDER_CODE_SCAN_EXCLUDED: dict[str, str] = { - "_helpers/assembly": "semantic and metadata validation of the assembled CompiledProject (M4)", "_helpers/analysis": "SQL analysis of rendered SQL, after the CompileProjectInputs frontier", } _SCOPE_QUERY_ONLY: str = "reported only by sqb scope queries, never by compile" @@ -659,17 +658,16 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ), } ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( - "compile/_helpers/assembly", "compile/_helpers/analysis", "compile/_helpers/diagnostics", "contracts", - "pipeline/_helpers/target_validation.py", "sql_analysis", ) ANALYSIS_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( "crates/sqlbuild-analysis/src/semantic_checks/constants.rs", "crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs", "crates/sqlbuild-analysis/src/contracts/constants.rs", + "crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs", ) NATIVE_CODE_TEST_DIRECTORY: str = "tests" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1dfd46f5d4..ba39d0eb51 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,13 +1,13 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Hashable, Mapping, Sequence, Set +from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence, Set from pathlib import Path, PurePath from typing import Any, Literal, TypedDict, overload from _typeshed import SupportsRichComparison from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile, RelationInfo from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import CompileAnalysis @@ -40,6 +40,7 @@ from sqlbuild.compiler.compile.models import ( CompileSqlReference, CompileSqlScenarioInput, CompileSqlTestInput, + DeclarationResolutionContext, DeclarationScopeBuild, DeclarationScopeResolver, DynamicColumnContractProof, @@ -100,6 +101,7 @@ from sqlbuild.compiler.discovery.models import ( TolerantScopeDiscovery, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, DirectSemanticColumnUse, @@ -112,9 +114,17 @@ from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, ProjectGraph, + PythonPlanEntry, + PythonRunPlanOutputs, + StaticCommandContext, +) +from sqlbuild.compiler.planner.models import DeferralInputs, PlanOutput +from sqlbuild.compiler.python_nodes.models import ( + PythonNodeGraph, + PythonNodeIdentity, + PythonSqlRunLifecyclePlan, + PythonSqlRunSelection, ) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -145,6 +155,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import ( DefaultsConfig, @@ -161,6 +172,7 @@ from sqlbuild.spec.contracts.models import ( from sqlbuild.sql_values.models import SqlValue BUILD_IDENTITY: str +STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] class NativeCompilerError(RuntimeError): """An internal native compiler failure: a panic or a broken native invariant.""" @@ -1111,6 +1123,25 @@ def build_compiled_project( resolved_connection: dict[str, object] | None = None, analysis_selection: CompileAnalysisSelection | None = None, ) -> CompiledProjectView: ... +def compile_frontier[T]( + *, + until: CompilerStage, + python_stage: Callable[[], T], + native_stage: Callable[[], T] | None = None, +) -> T: ... +def render_stage_capture(value: object) -> str: ... +def digest_module_files(paths: Iterable[str], /) -> dict[str, str] | None: ... +def module_digests_metadata(digests: dict[str, str], /) -> bytes: ... +def unchanged_module_digests( + *, metadata: bytes, known: dict[str, str] +) -> dict[str, str] | None: ... +def macro_store_token( + *, loaded: LoadedMacro, identity: DeclarationIdentity | None +) -> str | None: ... +def context_store_token( + *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None +) -> str | None: ... +def mask_capture_noise(text: str) -> str: ... def assemble_project( *, inputs: CompileInputsView, @@ -1160,6 +1191,104 @@ def build_discovered_project_inputs( sql_analysis_enabled: bool, extract_output_column_locations: bool = True, ) -> DiscoveredProjectInputs: ... +def validate_named_target_schema_strategy( + *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, selected_target: str | None +) -> None: ... +def validate_managed_loader_target_isolation( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + resolved_connection: dict[str, object] | None = None, +) -> None: ... +def validate_project_targets(*, adapter_name: str, project: CompiledProjectView) -> None: ... +def validate_managed_write_schemas( + *, adapter: BaseAdapter, project: CompiledProjectView +) -> None: ... +def resolve_compile_analysis_selection( + *, options: CompilePipelineOptions, discovered_inputs: DiscoveredProjectInputs +) -> CompileAnalysisSelection | None: ... +def build_project_graph_from_compiled_project(*, project: CompiledProjectView) -> ProjectGraph: ... +def plan_has_executable_work( + *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () +) -> bool: ... +def build_static_pipeline_result( + *, context: StaticCommandContext, plan_output: PlanOutput +) -> CompilePipelineResult: ... +def run_compile_only_pipeline( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions, + on_progress: Callable[[str], None] | None = None, +) -> CompilePipelineResult: ... +def build_deferred_locations( + *, + project: CompiledProjectView, + deferred_target_config: TargetConfig, + effective_vars: dict[str, object], + default_schema: str | None, + default_database: str | None, + render_qualified_name: Callable[..., str | None], +) -> dict[str, CompiledRelationLocation]: ... +def resolve_deferred_target_config( + *, discovered_inputs: DiscoveredProjectInputs, defer_to: str, current_target_name: str | None +) -> TargetConfig: ... +def gather_deferred_relations( + *, + adapter: BaseAdapter, + connection: Any, + deferred_locations: dict[str, CompiledRelationLocation], +) -> dict[str, RelationInfo]: ... +def resolve_model_deferral_inputs( + *, + project: CompiledProjectView, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + connection: Any, + defer_to: str | None, +) -> DeferralInputs: ... +def build_python_relation_targets( + *, + adapter: BaseAdapter, + project: CompiledProjectView, + plan_output: PlanOutput, + required_refs: frozenset[SqlResourceRef] | None = None, +) -> dict[SqlResourceRef, str]: ... +def build_project_relation_targets( + *, adapter: BaseAdapter, plan_output: PlanOutput +) -> dict[SqlResourceRef, str] | None: ... +def resolve_diff_model_names( + *, project: CompiledProjectView, select: tuple[str, ...], exclude: tuple[str, ...] +) -> tuple[str, ...]: ... +def run_diff_pipeline( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + from_target: str, + to_target: str, + resolved_connection: dict[str, object], + no_sql_validation: bool, + select: tuple[str, ...], + exclude: tuple[str, ...], + cli_vars: dict[str, object] | None = None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, +) -> tuple[CompiledProjectView, CompiledProjectView, tuple[str, ...]]: ... +def python_read_source_names_for( + *, discovered_inputs: DiscoveredProjectInputs, python_node_names: frozenset[str] +) -> frozenset[str]: ... +def build_python_run_plan_outputs( + *, + discovered_inputs: DiscoveredProjectInputs, + plan_output: PlanOutput, + run_selection: PythonSqlRunSelection | None, + selected_python_node_names: frozenset[str], +) -> PythonRunPlanOutputs: ... +def build_python_plan_entries( + *, + lifecycle_plan: PythonSqlRunLifecyclePlan, + python_graph: PythonNodeGraph, + previous_identities: dict[tuple[str, str], Fingerprint] | None = None, +) -> tuple[PythonPlanEntry, ...]: ... def compile_project_phase( *, discovered_inputs: DiscoveredProjectInputs, @@ -1426,9 +1555,6 @@ def normalize_typed_sql_value( def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... -def deferred_target_segments( - template: str, logical: str, context_keys: list[str], variables: list[str], / -) -> list[tuple[bool, str]]: ... def unknown_mapping_keys( keys: list[tuple[str, bool]], allowed: list[str], / ) -> tuple[str, str] | None: ... @@ -1783,6 +1909,22 @@ def _oracle_cte_fact_recovery( ) -> tuple[str | None, list[tuple[str, str]], list[tuple[str, str]], list[str], list[str]]: ... # Native analysis: semantic completion. +def semantic_shapes(*, project: Any, profile: Any | None = None) -> dict[str, dict[str, str]]: ... +def published_model_shape( + *, model: Any, columns: dict[str, str], inputs: dict[str, dict[str, str]], profile: Any +) -> dict[str, str]: ... +def model_binding_inputs( + model: Any, shapes: dict[str, dict[str, str]], / +) -> dict[str, dict[str, str]]: ... +def binding_relation_names(references: Sequence[Any], /) -> frozenset[str]: ... +def semantic_coverage(project: Any, /) -> dict[str, tuple[str, ...]]: ... +def source_binding_diagnostics( + *, project: Any, columns: Mapping[str, Any], profile: Any, selected_keys: frozenset[Any] +) -> tuple[Any, ...]: ... +def known_function_names(functions: Sequence[Any], /) -> tuple[str, ...]: ... +def known_declared_types( + *, functions: Sequence[Any], column_types: Mapping[str, Mapping[str, str]] +) -> tuple[str, ...]: ... def complete_semantic_project( *, project: Any, diff --git a/src/sqlbuild/cli/commands/_helpers/audit/planning.py b/src/sqlbuild/cli/commands/_helpers/audit/planning.py index 35ea3ecca0..4d62c167a2 100644 --- a/src/sqlbuild/cli/commands/_helpers/audit/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/audit/planning.py @@ -2,12 +2,12 @@ from __future__ import annotations +from sqlbuild._native import build_static_pipeline_result from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands.models import AuditCommandRequest, AuditInvocation from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py index df113307de..acb0929b8d 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import build_project_relation_targets from sqlbuild.cli.commands._helpers.check.core import ( build_check_relation_targets, load_results_by_loader_name, @@ -18,7 +19,6 @@ BuildRunOutcome, ) from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph from sqlbuild.compiler.python_nodes.models import PythonNodeGraph diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py index b1fde65ad6..55b54ce85d 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py @@ -4,6 +4,7 @@ from typing import Any, TextIO +from sqlbuild._native import build_project_relation_targets from sqlbuild.cli.commands._helpers.build_execution.run_context import ( write_build_run_context, ) @@ -30,7 +31,6 @@ ConnectionProgressReporter, ) from sqlbuild.cli.progress.main._write_execution_header import write_execution_header -from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides import ( typed_cursor_overrides, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py b/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py index 1fa591f413..add32112c4 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py @@ -2,6 +2,7 @@ from datetime import UTC, datetime +from sqlbuild._native import plan_has_executable_work from sqlbuild.cli.commands._helpers.build_execution.outputs import write_no_work_build_json from sqlbuild.cli.commands._helpers.build_execution.run_context import ( write_build_run_context, @@ -13,7 +14,6 @@ BuildInvocation, BuildRunContext, ) -from sqlbuild.compiler.pipeline.main.plan_work import plan_has_executable_work from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.cost.models import CostRunRecord diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py index 76c8389315..35e3b8147a 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py +++ b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py @@ -6,6 +6,10 @@ from datetime import datetime from typing import Any, TextIO +from sqlbuild._native import ( + build_project_relation_targets, + build_python_relation_targets, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.build_python_nodes.python_lifecycle_selection import ( python_node_result_names, @@ -22,10 +26,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project_relation_targets import ( - build_project_relation_targets, -) -from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.main.selection.loader_dag import build_intermediate_source_map from sqlbuild.compiler.planner.models import PlanOutput diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py index 1427b6c497..6e10aafd25 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ b/src/sqlbuild/cli/commands/_helpers/check/core.py @@ -6,6 +6,7 @@ from collections.abc import Callable from typing import Any, TextIO +from sqlbuild._native import build_python_relation_targets from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup @@ -24,7 +25,6 @@ from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector diff --git a/src/sqlbuild/cli/commands/_helpers/check/execution.py b/src/sqlbuild/cli/commands/_helpers/check/execution.py index 94689f7487..16c4ef547d 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/check/execution.py @@ -5,6 +5,7 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import build_project_relation_targets from sqlbuild.cli.commands._helpers.check.core import ( build_check_relation_targets, load_results_by_loader_name, @@ -23,7 +24,6 @@ current_native_progress_projector, ) from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph diff --git a/src/sqlbuild/cli/commands/_helpers/check/planning.py b/src/sqlbuild/cli/commands/_helpers/check/planning.py index b0a712f1db..58771d70c1 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/check/planning.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import build_static_pipeline_result from sqlbuild.cli.commands._helpers.check.core import ( check_dependency_closure, resolve_selected_check_names, @@ -15,7 +16,6 @@ from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py index eb3e0fa90f..577c73c5c3 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/execution.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import run_diff_pipeline from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.contract.types import BuiltinAdapter @@ -37,7 +38,6 @@ from sqlbuild.compiler.compile.main.effective_target_namespace import ( build_effective_target_namespace, ) -from sqlbuild.compiler.pipeline.main.diff import run_diff_pipeline from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.cursor_algebra.models import Duration from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index f014cde75c..9b6c7ff7f1 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_only_pipeline from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -40,7 +40,6 @@ from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index 7f4f6f86e9..eb26c85133 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_only_pipeline from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -57,7 +57,6 @@ from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, diff --git a/src/sqlbuild/cli/commands/_helpers/seed/planning.py b/src/sqlbuild/cli/commands/_helpers/seed/planning.py index 38f0fd6dfa..95632c8c8a 100644 --- a/src/sqlbuild/cli/commands/_helpers/seed/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/seed/planning.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import build_static_pipeline_result from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -16,7 +17,6 @@ from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/test/planning.py b/src/sqlbuild/cli/commands/_helpers/test/planning.py index 62c4562208..7be9b302ba 100644 --- a/src/sqlbuild/cli/commands/_helpers/test/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/test/planning.py @@ -2,12 +2,12 @@ from __future__ import annotations +from sqlbuild._native import build_static_pipeline_result from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py index 545e5315f8..0f5f0171ee 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ b/src/sqlbuild/cli/commands/main/inspection/_contract.py @@ -7,7 +7,11 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + build_project_graph_from_compiled_project, + discover_project_inputs, +) from sqlbuild.cli.commands._helpers.contract.progress import ( inspect_contracts_with_progress, write_contract_updates_with_progress, @@ -29,7 +33,6 @@ from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.contract_adoption.types import ContractAction from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project_graph import build_project_graph_from_compiled_project from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py deleted file mode 100644 index 1388e76984..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Construct authored locations from native normalization and expansion maps.""" - -from functools import lru_cache -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledSqlExpansion, ExpansionSpan -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -from sqlbuild.compiler.sql_analysis.types import NativeBindingPositions, NativePositionsModule -from sqlbuild.spec.contracts.models import SourceLocation - - -@lru_cache(maxsize=32) -def _mapper( - *, - authored_sql: str, - authored_query_sql: str, - expanded_sql: str, - cleaned_sql: str, - passes: tuple[tuple[ExpansionSpan, ...], ...], -) -> NativeBindingPositions: - native_passes: list[tuple[tuple[int, int, int, int], ...]] = [] - for spans in passes: - native_passes.append( - tuple( - (span.source_start, span.source_end, span.output_start, span.output_end) - for span in spans - ) - ) - return cast(NativePositionsModule, _native).BindingPositions( - { - "authored": authored_sql, - "query": authored_query_sql, - "expanded": expanded_sql, - "cleaned": cleaned_sql, - "passes": native_passes, - } - ) - - -def get_authored_binding_location( - *, - path: Path, - authored_sql: str, - authored_query_sql: str, - cleaned_sql: str, - diagnostic: SqlBindingDiagnostic, - expansion: CompiledSqlExpansion | None = None, -) -> SourceLocation | None: - mapper: NativeBindingPositions = _mapper( - authored_sql=authored_sql, - authored_query_sql=authored_query_sql, - expanded_sql=expansion.expanded_sql if expansion is not None else authored_query_sql, - cleaned_sql=cleaned_sql, - passes=expansion.passes if expansion is not None else (), - ) - line, column = mapper.position( - start=diagnostic.start, - line=diagnostic.line, - column=diagnostic.column, - message=diagnostic.message, - ) - if line is None or column is None: - return None - end_line: int | None = None - end_column: int | None = None - if diagnostic.end is not None: - end_line, end_column = mapper.position( - start=diagnostic.end, line=None, column=None, message="" - ) - return SourceLocation( - path=path, line=line, column=column, end_line=end_line, end_column=end_column - ) - - -def get_authored_binding_position( - *, - authored_sql: str, - authored_query_sql: str, - cleaned_sql: str, - diagnostic: SqlBindingDiagnostic, - expansion: CompiledSqlExpansion | None = None, -) -> tuple[int | None, int | None]: - mapper: NativeBindingPositions = _mapper( - authored_sql=authored_sql, - authored_query_sql=authored_query_sql, - expanded_sql=expansion.expanded_sql if expansion is not None else authored_query_sql, - cleaned_sql=cleaned_sql, - passes=expansion.passes if expansion is not None else (), - ) - return mapper.position( - start=diagnostic.start, - line=diagnostic.line, - column=diagnostic.column, - message=diagnostic.message, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py deleted file mode 100644 index 7c80f25bcf..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Explicit boundary conversion of a hand-built model input's native analysis.""" - -from __future__ import annotations - -from collections.abc import Sequence -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import dynamic_contract_proof, session_outcome_analysis -from sqlbuild.compiler.compile._helpers.assembly.targets import build_model_relation_target -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompiledModel, - CompiledObjectKey, - CompiledSqlExpansion, - CompilerDiagnostic, - DynamicColumnContractProof, - ExpansionSpan, - HandBuiltAnalysis, - InferredColumn, - ModelSqlAnalysis, - PolyglotAnalysisResult, -) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - ModelInput, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -from sqlbuild.spec.contracts.models import SourceLocation - - -def hand_built_compiled_model( - *, - model_input: ModelInput, - deps: list[tuple[str, str]], - analysis: HandBuiltAnalysis, - macro_deps: tuple[str, ...], -) -> CompiledModel: - """A `CompiledModel` for a model input built outside the native compile-input stage.""" - - model_name: str = model_input.model_file.file_path.stem - inferred: tuple[InferredColumn, ...] | None = None - lineage: Sequence[CompiledLineageColumnFact] | None = None - has_star: bool = False - star_resolved: bool = False - diagnostics: tuple[CompilerDiagnostic, ...] = () - validated: bool = False - proof: DynamicColumnContractProof | None = dynamic_contract_proof(analysis.standalone_proof) - if analysis.session is not None: - session, position = analysis.session - outcome: ModelSqlAnalysis = session_outcome_analysis( - session=session, position=position, placeholders=None - ) - result: PolyglotAnalysisResult = outcome.polyglot_analysis - if result.analysis_succeeded: - inferred = result.columns - lineage = result.lineage_columns - has_star, star_resolved = result.has_star, result.star_resolved - else: - inferred = dynamic_fallback_columns(analysis.fallback) - proof = outcome.dynamic_column_contract or proof - validated = result.binding_validated - diagnostics = _located( - model_input=model_input, - diagnostics=result.binding_diagnostics, - cleaned=outcome.cleaned_sql or "", - ) - if proof is not None and proof.output_proven: - inferred = proof.fixed_columns - has_star = True - return CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=model_name), - deps=tuple( - CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) - for kind, name in deps - ), - name=model_name, - relative_path=model_input.model_file.relative_path, - query_sql=model_input.query_sql, - config=model_input.config, - destination=build_model_relation_target(model_input=model_input, model_name=model_name), - references=model_input.references, - schema_entry=model_input.schema_entry, - inferred_columns=inferred, - fast_lineage_columns=lineage, - fast_lineage_has_star=has_star, - fast_lineage_star_resolved=star_resolved, - authored_sql=model_input.model_file.contents, - authored_query_sql=model_input.model_file.query_sql, - output_column_locations=model_input.model_file.output_column_locations, - extract_implicit_alias_columns=model_input.model_file.extract_implicit_alias_columns, - macro_deps=macro_deps, - enum_declarations=model_input.enum_declarations, - constant_declarations=model_input.constant_declarations, - enum_columns=model_input.enum_columns, - binding_diagnostics=diagnostics, - binding_validated=validated, - dynamic_column_contract=proof, - rejected_sql_analysis_opt_out=model_input.rejected_sql_analysis_opt_out, - ) - - -def dynamic_fallback_columns( - rows: list[tuple[str, str | None, str]] | None, -) -> tuple[InferredColumn, ...] | None: - """A failed analysis's parse-mode column rows as inferred columns.""" - - if rows is None: - return None - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in rows - ) - - -def _located( - *, model_input: ModelInput, diagnostics: tuple[SqlBindingDiagnostic, ...], cleaned: str -) -> tuple[CompilerDiagnostic, ...]: - if not diagnostics: - return () - expansion: CompiledSqlExpansion | None = model_input.sql_expansion - path: Path = model_input.model_file.relative_path - located: list[ - tuple[str, str, str, int | None, int | None, tuple[int, int, int | None, int | None] | None] - ] = _native.locate_model_binding_diagnostics( - [ - (item.code, item.message, item.line, item.column, item.start, item.end, item.severity) - for item in diagnostics - ], - (model_input.model_file.contents, model_input.model_file.query_sql, cleaned), - None if expansion is None else (expansion.expanded_sql, _pass_rows(expansion)), - ) - return tuple( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=severity, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model_input.model_file.file_path.stem, - path=path, - line=line, - column=column, - location=None - if span is None - else SourceLocation( - path=path, line=span[0], column=span[1], end_line=span[2], end_column=span[3] - ), - ) - for code, message, severity, line, column, span in located - ) - - -def _pass_rows(expansion: CompiledSqlExpansion) -> list[list[tuple[int, int, int, int]]]: - return [list(map(_span_row, spans)) for spans in expansion.passes] - - -def _span_row(span: ExpansionSpan) -> tuple[int, int, int, int]: - return (span.source_start, span.source_end, span.output_start, span.output_end) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py deleted file mode 100644 index 96987854ef..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Compile-owned inputs of the native semantic metadata checks.""" - -from __future__ import annotations - -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile.models import CompiledModelSqlTestPayload -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def sql_test_columns( - *, project: CompiledProjectView, profile: ExpressionInferenceProfile -) -> dict[str, tuple[str, ...]]: - """Infer fixture interfaces in one batch instead of repeating Python AST walks.""" - bodies: dict[str, None] = {} - for test in project.sql_tests: - if isinstance(test.payload, CompiledModelSqlTestPayload): - for cte in (*test.payload.authored_ctes, *test.payload.expected_ctes): - bodies[cte.sql_body] = None - queries: tuple[str, ...] = tuple(bodies) - if not queries: - return {} - analyses: tuple[NativeQueryColumns, ...] = infer_native_query_columns( - queries=tuple( - NativeColumnQuery(sql=sql, mode="batch", recover_cte_facts=False) for sql in queries - ), - profile=profile, - ) - return {sql: _column_names(analysis) for sql, analysis in zip(queries, analyses, strict=True)} - - -def _column_names(analysis: NativeQueryColumns) -> tuple[str, ...]: - return tuple(column.name for column in analysis.columns or ()) - - -def names(value: object) -> tuple[str, ...]: - """The column names a metadata config value lists.""" - - if isinstance(value, str): - return (value,) - if isinstance(value, (tuple, list)): - return tuple(item for item in value if isinstance(item, str)) - return () diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/native_declarations.py b/src/sqlbuild/compiler/compile/_helpers/assembly/native_declarations.py deleted file mode 100644 index fd9450effb..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/native_declarations.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Names and types registered with native validation from project declarations.""" - -from sqlbuild.compiler.compile.types import CompiledFunctionView, FunctionInput - - -def known_function_names( - functions: tuple[CompiledFunctionView, ...] | tuple[FunctionInput, ...], -) -> tuple[str, ...]: - """Register authored names and the analysis-only scalar/table function names.""" - names: set[str] = set() - for function in functions: - names.update( - ( - function.name, - f"__sqlbuild_udf_{function.name}", - f"__sqlbuild_table_function_{function.name}", - ) - ) - return tuple(sorted(names)) - - -def known_declared_types( - *, - functions: tuple[CompiledFunctionView, ...] | tuple[FunctionInput, ...], - column_types: dict[str, dict[str, str]], -) -> tuple[str, ...]: - """Register types named by relation contracts and scalar/table function signatures.""" - types: set[str] = set() - for columns in column_types.values(): - types.update(columns.values()) - for function in functions: - types.update(argument.type for argument in function.arguments) - types.update(column.type for column in function.return_columns) - if not function.return_columns: - types.add(function.returns) - types.discard("UNKNOWN") - return tuple(sorted(types)) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py deleted file mode 100644 index 5282094dee..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Explain the limits of offline semantic checks without per-model warnings.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.constants import TYPE_CHECKED_DIALECTS - - -def semantic_coverage(project: CompiledProjectView) -> dict[str, tuple[str, ...]]: - """Return partial-check reasons for machine reports.""" - - if not project.settings.sql_analysis: - return {} - shapes: dict[str, dict[str, str]] = semantic_shapes(project=project) - invalid: set[str] = set() - for model in project.models: - if any(item.is_error for item in model.binding_diagnostics): - invalid.add(model.name) - blocked_by: dict[str, set[str]] = {name: {name} for name in invalid} - if invalid: - for _ in project.models: - changed: bool = False - for model in project.models: - causes: set[str] = set(blocked_by.get(model.name, ())) - for _, name, _, _ in model.reference_rows: - causes.update(blocked_by.get(name, ())) - if causes and causes != blocked_by.get(model.name): - blocked_by[model.name] = causes - changed = True - if not changed: - break - reasons: dict[str, tuple[str, ...]] = {} - for model in project.models: - if model.config.values.get("sql_analysis") is False: - reasons[model.name] = ("SQL analysis disabled",) - continue - partial: list[str] = [ - f"open source: {name}" if kind == SqlReferenceKind.SOURCE else f"open input: {name}" - for kind, name, _, _ in model.reference_rows - if kind in {SqlReferenceKind.SOURCE, SqlReferenceKind.REF, SqlReferenceKind.DBT_REF} - and name not in shapes - ] - partial.extend( - f"depends on invalid model {name}" - for name in sorted(blocked_by.get(model.name, set()) - {model.name}) - ) - inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows - if not inferred: - partial.append("output shape could not be inferred") - elif model.fast_lineage_has_star and model.name not in shapes: - partial.append("unresolved star over open inputs") - if any(column_type is None for _, column_type, _ in inferred or ()): - partial.append("some output types are unknown") - if (project.sql_analysis_dialect or "generic") not in TYPE_CHECKED_DIALECTS: - partial.append("type checking awaits dialect coercion support") - if partial: - reasons[model.name] = tuple(sorted(set(partial))) - return reasons diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py deleted file mode 100644 index 31f71d945a..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Authoritative relation shapes shared by offline and warehouse validation.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -from sqlbuild._native import infer_native_expression_source_shapes -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.reference_names import table_function_analysis_name -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - ModelInput, -) -from sqlbuild.compiler.planner.constants import ( - SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, - SNAPSHOT_DEFAULT_VALID_TO_COLUMN, -) -from sqlbuild.compiler.planner.types import ContractPolicy, MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.constants import ( - CASE_SENSITIVE_BINDING_DIALECTS, -) -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.spec.contracts.models import ( - SourceEntry, -) - - -def binding_relation_names(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: - return binding_relation_row_names( - [(str(reference.ref_kind), reference.ref_name, None, None) for reference in references] - ) - - -def binding_relation_row_names( - rows: list[tuple[str, str, str | None, int | None]], -) -> frozenset[str]: - """The analysis names `(kind, name, package, count)` reference rows bind, UDFs excluded.""" - - return frozenset( - table_function_analysis_name(name) if kind == SqlReferenceKind.TABLE_FUNCTION else name - for kind, name, _, _ in rows - if kind != SqlReferenceKind.UDF - ) - - -def binding_required_names(model_input: ModelInput) -> frozenset[str] | None: - if not model_input.sql_validation_enabled: - return None - return binding_relation_row_names(model_input.reference_rows) - - -def published_model_shape( - *, - sql: str, - columns: dict[str, str], - inputs: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, - config_values: dict[str, object], -) -> dict[str, str]: - """Return a model relation's shape, including snapshot validity columns.""" - shape: dict[str, str] = inferred_binding_shape( - sql=sql, columns=columns, inputs=inputs, profile=profile - ) - if config_values.get("materialized") == MaterializationType.SNAPSHOT: - for name in ( - config_values.get("valid_from_column") or SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, - config_values.get("valid_to_column") or SNAPSHOT_DEFAULT_VALID_TO_COLUMN, - ): - shape[str(name)] = "UNKNOWN" - return shape - - -def binding_schema_for_model( - *, model_input: ModelInput, complete_binding_schemas: dict[str, dict[str, str]] -) -> dict[str, dict[str, str]] | None: - required_names: frozenset[str] | None = binding_required_names(model_input) - if required_names is None: - return None - return {name: complete_binding_schemas.get(name) or {} for name in required_names} - - -def inferred_binding_shape( - *, - sql: str, - columns: dict[str, str], - inputs: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, -) -> dict[str, str]: - if ( - profile.quoted_identifiers_ignore_case - or profile.sql_analysis_dialect not in CASE_SENSITIVE_BINDING_DIALECTS - ): - return columns - catalog: Any = profile.binding_catalog or create_binding_catalog( - dialect=profile.sql_analysis_dialect or "generic", - quoted_ignore_case=False, - known_functions=profile.semantic_known_functions, - known_types=profile.semantic_known_types, - relations=inputs, - ) - return catalog.inferred_schema( - sql=normalize_analysis_sql(sql=sql, dialect=profile.sql_analysis_dialect), - columns=columns, - inputs=inputs, - ) - - -def semantic_shapes( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile | None = None, -) -> dict[str, dict[str, str]]: - """Return closed shapes; absence means open, never an empty physical table.""" - - profile = profile or ExpressionInferenceProfile( - sql_analysis_dialect=project.sql_analysis_dialect - ) - profile = replace(profile, binding_catalog=project.binding_catalog) - shapes: dict[str, dict[str, str]] = {} - for seed in project.seeds: - shapes[seed.name] = { - column.name: column.type or "UNKNOWN" for column in seed.schema_entry.columns - } - for function in project.functions: - if function.return_columns: - shapes[table_function_analysis_name(function.name)] = { - column.name: column.type for column in function.return_columns - } - expression_sources: list[tuple[str, str]] = [] - for source in project.sources: - entry: SourceEntry = source.source_entry - if entry.contract == ContractPolicy.ENFORCED: - shapes[source.name] = { - column.name: column.type or "UNKNOWN" for column in entry.columns - } - elif entry.expression: - expression_sources.append((source.name, entry.expression)) - source_shapes: tuple[dict[str, str] | None, ...] = infer_native_expression_source_shapes( - expressions=tuple(expression for _, expression in expression_sources), profile=profile - ) - for (source_name, _), source_shape in zip(expression_sources, source_shapes, strict=True): - if source_shape is not None: - shapes[source_name] = source_shape - pending: list[CompiledModelView] = list(project.models) - while pending: - remaining: list[CompiledModelView] = [] - for model in pending: - declared: list[tuple[str, str | None]] | None = model.schema_column_rows - inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows - rows: list[tuple[str, str, str | None, int | None]] = model.reference_rows - relations: frozenset[str] = binding_relation_row_names(rows) - if ( - model.config.values.get("contract") == ContractPolicy.ENFORCED - and declared is not None - and not model.has_dynamic_columns - ): - unchecked: frozenset[str] = model.unchecked_output_columns - shapes[model.name] = { - name: "UNKNOWN" if name in unchecked else column_type or "UNKNOWN" - for name, column_type in declared - } - elif inferred and ( - not model.fast_lineage_has_star - or (model.fast_lineage_star_resolved and rows and relations <= shapes.keys()) - ): - shapes[model.name] = published_model_shape( - sql=model.query_sql, - profile=replace(profile, binding_catalog=project.binding_catalog), - columns={name: column_type or "UNKNOWN" for name, column_type, _ in inferred}, - inputs={name: shapes.get(name, {}) for name in relations}, - config_values=model.config.values, - ) - else: - remaining.append(model) - if len(remaining) == len(pending): - break - pending = remaining - return shapes diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py deleted file mode 100644 index 495f234713..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Rebind models to source columns already inspected by the planner.""" - -from __future__ import annotations - -from dataclasses import replace -from graphlib import TopologicalSorter -from typing import Any - -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.analysis.binding_requests import ( - get_complete_schema_binding_request, -) -from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( - get_authored_binding_position, -) -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - binding_relation_row_names, - published_model_shape, - semantic_shapes, -) -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlExpansion, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.constants import ( - BINDING_UNKNOWN_TABLE_INTERNAL_CODE, - CASE_SENSITIVE_BINDING_DIALECTS, - NATIVE_DIALECT_ALIASES, -) -from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest - - -def _model_shapes( - *, model: CompiledModelView, shapes: dict[str, dict[str, str]] -) -> dict[str, dict[str, str]]: - return {name: shapes.get(name, {}) for name in binding_relation_row_names(model.reference_rows)} - - -def get_source_binding_diagnostics( - *, - project: CompiledProjectView, - columns: dict[str, tuple[ColumnInfo, ...]], - profile: ExpressionInferenceProfile, - selected_keys: frozenset[CompiledObjectKey], -) -> tuple[CompilerDiagnostic, ...]: - """Validate selected source readers without executing or inspecting warehouse SQL.""" - - if not project.settings.sql_analysis or not columns: - return () - profile = replace( - profile, - sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( - profile.sql_analysis_dialect or "generic", profile.sql_analysis_dialect - ), - quoted_identifiers_ignore_case=ignores_quoted_case( - connection=project.effective_connection, dialect=profile.sql_analysis_dialect - ), - ) - shapes: dict[str, dict[str, str]] = semantic_shapes(project=project, profile=profile) - changed_sources: set[str] = set() - for name, value in columns.items(): - if value: - raw_shape: dict[str, str] = {column.name: column.type for column in value} - if _adds_physical_evidence(physical=raw_shape, known=shapes.get(name)) or ( - profile.sql_analysis_dialect in CASE_SENSITIVE_BINDING_DIALECTS - and not profile.quoted_identifiers_ignore_case - ): - changed_sources.add(name) - shapes[name] = { - '"' + column.name.replace('"', '""') + '"': column.type for column in value - } - physical_catalog: Any | None = ( - project.binding_catalog.with_relations(shapes) - if project.binding_catalog is not None - else None - ) - profile = replace(profile, binding_catalog=physical_catalog) - by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - graph: dict[str, tuple[str, ...]] = {} - for model in project.models: - graph[model.name] = tuple( - reference.ref_name - for reference in model.references - if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name in by_name - ) - affected: set[str] = set() - models: list[CompiledModelView] = [] - for name in TopologicalSorter(graph).static_order(): - model: CompiledModelView = by_name[name] - if model.key not in selected_keys or model.config.values.get("sql_analysis") is False: - continue - if any( - ( - reference.ref_kind == SqlReferenceKind.SOURCE - and reference.ref_name in changed_sources - ) - or (reference.ref_kind == SqlReferenceKind.REF and reference.ref_name in affected) - for reference in model.references - ): - models.append(model) - affected.add(model.name) - inputs: dict[str, dict[str, str]] = _model_shapes(model=model, shapes=shapes) - nullability: dict[str, dict[str, InferredNullability]] = { - table: dict.fromkeys(shape, InferredNullability.UNKNOWN) - for table, shape in inputs.items() - } - analysis: NativeQueryColumns = infer_native_query_columns( - queries=( - NativeColumnQuery( - sql=model.query_sql, - mode="reanalysis", - placeholders=_placeholders(model), - references=model.references, - ), - ), - profile=profile, - column_types_by_table=inputs, - column_nullability_by_table=nullability, - )[0] - if analysis.columns and (not analysis.has_star or all(inputs.values())): - shapes[model.name] = published_model_shape( - sql=model.query_sql, - profile=profile, - inputs=inputs, - columns={column.name: column.type or "UNKNOWN" for column in analysis.columns}, - config_values=model.config.values, - ) - else: - shapes.pop(model.name, None) - if physical_catalog is not None: - physical_catalog = physical_catalog.with_relations(shapes) - functions: tuple[str, ...] = known_function_names(project.functions) - types: tuple[str, ...] = known_declared_types(functions=project.functions, column_types=shapes) - requests: tuple[SqlSchemaValidationRequest, ...] = tuple( - replace( - get_complete_schema_binding_request( - known_functions=functions, - known_types=types, - query_sql=cursor_intrinsics_analysis_sql( - sql=model.query_sql, cursor_type=model.config.values.get("cursor_type") - ), - placeholders=_placeholders(model), - dialect=profile.sql_analysis_dialect, - binding_schema=_model_shapes(model=model, shapes=shapes), - ), - catalog=physical_catalog, - quoted_identifiers_ignore_case=ignores_quoted_case( - connection=project.effective_connection, dialect=profile.sql_analysis_dialect - ), - ) - for model in models - ) - results: tuple[SqlBindingResult, ...] = get_schema_validations(requests=requests) - expansions: dict[str, CompiledSqlExpansion] = { - path.stem: expansion for path, expansion in project.sql_expansions.items() - } - diagnostics: list[CompilerDiagnostic] = [] - for model, request, result in zip(models, requests, results, strict=True): - for diagnostic in result.diagnostics: - if diagnostic.code == BINDING_UNKNOWN_TABLE_INTERNAL_CODE: - continue - line: int | None - column: int | None - line, column = get_authored_binding_position( - authored_sql=model.authored_sql, - authored_query_sql=model.authored_query_sql, - cleaned_sql=request.sql, - diagnostic=diagnostic, - expansion=expansions.get(model.name), - ) - diagnostics.append( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity(diagnostic.severity), - code=diagnostic.code, - message=diagnostic.message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - ) - return tuple(diagnostics) - - -def _adds_physical_evidence(*, physical: dict[str, str], known: dict[str, str] | None) -> bool: - """Return whether inspected columns differ from the shape; untyped names match any type.""" - - if known is None or physical.keys() != known.keys(): - return True - return any( - physical_type and physical_type != known[name] for name, physical_type in physical.items() - ) - - -def _placeholders(model: CompiledModelView) -> dict[str, str] | None: - values: object = model.config.values.get("placeholders") - if isinstance(values, dict): - return {str(key): str(value) for key, value in values.items()} - return None diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py b/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py deleted file mode 100644 index bf4775ec49..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Compiled model and seed relation target resolution.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import ModelInput - - -def build_model_relation_target( - *, model_input: ModelInput, model_name: str -) -> CompiledRelationLocation: - """Resolve one model's configured physical and logical destination.""" - - raw_database: object | None = model_input.config.values.get("database") - raw_schema: object | None = model_input.config.values.get("schema") - raw_alias: object | None = model_input.config.values.get("alias") - database: str | None = raw_database if isinstance(raw_database, str) else None - schema: str | None = raw_schema if isinstance(raw_schema, str) else None - name: str = raw_alias if isinstance(raw_alias, str) else model_name - return CompiledRelationLocation( - database=database, - schema=schema, - name=name, - qualified_name=None, - logical_schema=model_input.config.logical_schema, - logical_database=model_input.config.logical_database, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index e342978177..4febf59601 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -51,7 +51,6 @@ from sqlbuild.compiler.macro_bridge.constants import ( GENERATED_REFERENCE_MARKERS, ) -from sqlbuild.compiler.macro_bridge.main._macro_store_tokens import macro_store_tokens from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, @@ -530,7 +529,10 @@ def macro_expansion_classes() -> dict[str, object]: "ref_functions": { kind.value: function for function, kind in typed_reference_kinds().items() }, - "store_tokens": macro_store_tokens(), + "store_tokens": { + "macro_token": _native.macro_store_token, + "context_token": _native.context_store_token, + }, "relation_renderer": RELATION_RENDERER, "invocation_context": macro_invocation_context, "generated": { diff --git a/src/sqlbuild/compiler/compile/classes/python_model_analysis.py b/src/sqlbuild/compiler/compile/classes/python_model_analysis.py deleted file mode 100644 index 3b5315e234..0000000000 --- a/src/sqlbuild/compiler/compile/classes/python_model_analysis.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Python's half of native model analysis: per-model request rows and cache metrics.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.cache import record_analysis_cache_metrics -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - binding_schema_for_model, -) -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, -) -from sqlbuild.compiler.compile.models import ( - ModelSqlAnalysisRequest, -) -from sqlbuild.compiler.compile.types import ModelInput -from sqlbuild.compiler.planner.constants import ( - SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, - SNAPSHOT_DEFAULT_VALID_TO_COLUMN, -) -from sqlbuild.compiler.planner.types import ContractPolicy, MaterializationType -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -class PythonModelAnalysis: - """The analysed models' request facts.""" - - def __init__( - self, - *, - model_inputs: tuple[ModelInput, ...], - inference_profile: ExpressionInferenceProfile, - complete_binding_schemas: dict[str, dict[str, str]], - ) -> None: - self._profile: ExpressionInferenceProfile = inference_profile - self.requests: tuple[ModelSqlAnalysisRequest, ...] = tuple( - ModelSqlAnalysisRequest( - model_input=model_input, - query_sql=cursor_intrinsics_analysis_sql( - sql=model_input.query_sql, - cursor_type=model_input.config.values.get("cursor_type"), - ), - placeholders=_placeholders(model_input), - cache_key=None, - binding_schema=binding_schema_for_model( - model_input=model_input, complete_binding_schemas=complete_binding_schemas - ), - ) - for model_input in model_inputs - ) - - def model_rows(self) -> list[tuple[object, ...]]: - """Each model's native request row, in analysis order.""" - - return [_model_row(request) for request in self.requests] - - @staticmethod - def family_row(family: SchemaDynamicColumnFamily) -> tuple[str, str, str, str, str, str | None]: - """One declared dynamic column family as the native request row.""" - - return ( - family.name, - family.pivot_column, - family.value_column, - family.aggregate, - family.type, - family.name_pattern, - ) - - def record_cached(self, *, hits: int, misses: int) -> None: - """Record the analysis cache entries a native session read and missed.""" - - record_analysis_cache_metrics(batch_hits=0, entry_hits=hits, misses=misses, bypasses=0) - - def record_uncached(self) -> None: - """Record that every analysed model bypassed the analysis cache.""" - - record_analysis_cache_metrics( - batch_hits=0, entry_hits=0, misses=0, bypasses=len(self.requests) - ) - - -def _model_row(request: ModelSqlAnalysisRequest) -> tuple[object, ...]: - model_input: ModelInput = request.model_input - rows: list[tuple[str, str, str | None, int | None]] = model_input.reference_rows - values: dict[str, object] = model_input.config.values - return ( - model_input.model_file.file_path.stem, - request.query_sql, - list((request.placeholders or {}).items()), - [(str(kind), name) for kind, name, _, _ in rows], - _recovers_cte_facts(model_input), - ( - ( - str(values.get("valid_from_column") or SNAPSHOT_DEFAULT_VALID_FROM_COLUMN), - str(values.get("valid_to_column") or SNAPSHOT_DEFAULT_VALID_TO_COLUMN), - ) - if values.get("materialized") == MaterializationType.SNAPSHOT - else None - ), - [ - PythonModelAnalysis.family_row(family) - for family in ( - model_input.schema_entry.dynamic_columns - if model_input.schema_entry is not None - else () - ) - ], - ) - - -def _placeholders(model_input: ModelInput) -> dict[str, str] | None: - raw_placeholders: object | None = model_input.config.values.get("placeholders") - return ( - {str(key): str(value) for key, value in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - - -def _recovers_cte_facts(model_input: ModelInput) -> bool: - return model_input.config.values.get("contract") == ContractPolicy.ENFORCED or ( - model_input.schema_entry is not None and bool(model_input.schema_entry.type_enforcement) - ) diff --git a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py deleted file mode 100644 index 6ab8504125..0000000000 --- a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Compile-owned inputs of the native semantic completion stage.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.binding_requests import ( - get_complete_schema_binding_request, -) -from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( - names, - sql_test_columns, -) -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql import get_resource_sql_diagnostics -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case -from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest - - -class SemanticCompletionInputs: - """Expose the shapes, revalidation requests and metadata checks semantic completion reads.""" - - @staticmethod - def shapes(project: CompiledProjectView) -> dict[str, dict[str, str]]: - """Closed relation shapes, as type recovery, recovery and explanation read them.""" - - return semantic_shapes(project=project) - - @staticmethod - def revalidation_request( - *, project: CompiledProjectView, model: CompiledModelView, shapes: dict[str, dict[str, str]] - ) -> SqlSchemaValidationRequest: - """Type recovery's request revalidating one failed model against `shapes`.""" - - return replace( - get_complete_schema_binding_request( - query_sql=model.query_sql, - placeholders=None, - dialect=project.sql_analysis_dialect, - known_functions=known_function_names(project.functions), - known_types=known_declared_types(functions=project.functions, column_types=shapes), - binding_schema={ - reference.ref_name: shapes.get(reference.ref_name, {}) - for reference in model.references - }, - ), - catalog=project.binding_catalog, - quoted_identifiers_ignore_case=ignores_quoted_case( - connection=project.effective_connection, dialect=project.sql_analysis_dialect - ), - ) - - @staticmethod - def metadata_shapes( - *, project: CompiledProjectView, profile: ExpressionInferenceProfile - ) -> dict[str, dict[str, str]]: - """Closed relation shapes, as the metadata checks read them under `profile`.""" - - return semantic_shapes(project=project, profile=profile) - - @staticmethod - def column_names(value: object) -> tuple[str, ...]: - """The column names a metadata config value lists.""" - - return names(value) - - @staticmethod - def sql_test_columns( - *, project: CompiledProjectView, profile: ExpressionInferenceProfile - ) -> dict[str, tuple[str, ...]]: - """The inferred output columns of every model SQL test CTE body.""" - - return sql_test_columns(project=project, profile=profile) - - @staticmethod - def resource_sql_diagnostics( - *, - project: CompiledProjectView, - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, - ) -> tuple[CompilerDiagnostic, ...]: - """The compile-time SQL checks of audits, SQL tests and SQL hooks.""" - - return tuple(get_resource_sql_diagnostics(project=project, shapes=shapes, profile=profile)) diff --git a/src/sqlbuild/compiler/compile/main/_source_bindings.py b/src/sqlbuild/compiler/compile/main/_source_bindings.py index 1f9bb0da27..083f3e1c5a 100644 --- a/src/sqlbuild/compiler/compile/main/_source_bindings.py +++ b/src/sqlbuild/compiler/compile/main/_source_bindings.py @@ -1,9 +1,7 @@ """Planner entrypoint for warehouse-backed semantic binding.""" +from sqlbuild._native import source_binding_diagnostics from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.source_bindings import ( - get_source_binding_diagnostics as _get_source_binding_diagnostics, -) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView @@ -16,6 +14,6 @@ def get_source_binding_diagnostics( selected_keys: frozenset[CompiledObjectKey], ) -> tuple[CompilerDiagnostic, ...]: """Bind selected source readers to already-inspected warehouse columns.""" - return _get_source_binding_diagnostics( + return source_binding_diagnostics( project=project, columns=columns, profile=profile, selected_keys=selected_keys ) diff --git a/src/sqlbuild/compiler/compile/main/semantic_coverage.py b/src/sqlbuild/compiler/compile/main/semantic_coverage.py index a7c2ac9483..e960e20a53 100644 --- a/src/sqlbuild/compiler/compile/main/semantic_coverage.py +++ b/src/sqlbuild/compiler/compile/main/semantic_coverage.py @@ -1,6 +1,6 @@ """Public semantic-check coverage report.""" -from sqlbuild.compiler.compile._helpers.assembly.semantic_coverage import semantic_coverage +from sqlbuild._native import semantic_coverage from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py deleted file mode 100644 index 1505be9c60..0000000000 --- a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py +++ /dev/null @@ -1,21 +0,0 @@ -"""The native side of the compiler frontier.""" - -from collections.abc import Callable - -from sqlbuild.compiler.frontier.types import CompilerStage -from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge - - -def native_frontier[T]( - *, - until: CompilerStage, - python_stage: Callable[[], T], - native_stage: Callable[[], T] | None, -) -> T: - """Produce the frontier object for `until` natively; stages not yet native run in Python.""" - - if native_stage is not None: - return native_stage() - if until is CompilerStage.COMPILE_PROJECT_INPUTS: - return run_with_macro_bridge(stage=python_stage) - return python_stage() diff --git a/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py b/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py deleted file mode 100644 index 9613adc832..0000000000 --- a/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Dump frontier objects as canonical JSON so compiles can be compared stage by stage.""" - -import io -import os -from pathlib import Path -from typing import TextIO - -from sqlbuild.compiler.frontier.classes.stage_capture_encoder import StageCaptureEncoder -from sqlbuild.compiler.frontier.classes.stage_capture_sequence import StageCaptureSequence -from sqlbuild.compiler.frontier.classes.stage_capture_writer import StageCaptureWriter -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR, STAGE_CAPTURE_SUFFIX -from sqlbuild.compiler.frontier.types import CompilerStage - - -def write_stage_capture(*, stage: CompilerStage, value: object) -> None: - """Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory.""" - - directory: str | None = os.environ.get(STAGE_CAPTURE_DIR_ENV_VAR) - if not directory: - return - path: Path = Path(directory) / ( - f"{StageCaptureSequence.next_value():03d}-{stage.value}{STAGE_CAPTURE_SUFFIX}" - ) - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8", newline="\n") as stream: - _write_capture(value=value, stream=stream) - - -def render_stage_capture(value: object) -> str: - """Render a value as the JSON document `write_stage_capture` writes.""" - - stream: io.StringIO = io.StringIO() - _write_capture(value=value, stream=stream) - return stream.getvalue() - - -def _write_capture(*, value: object, stream: TextIO) -> None: - writer: StageCaptureWriter = StageCaptureWriter(stream) - writer.finish(StageCaptureEncoder(on_shared=writer.write_shared).encode(value)) diff --git a/src/sqlbuild/compiler/frontier/classes/__init__.py b/src/sqlbuild/compiler/frontier/classes/__init__.py deleted file mode 100644 index 27eb23ee2d..0000000000 --- a/src/sqlbuild/compiler/frontier/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler frontier runtime classes.""" diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py deleted file mode 100644 index ad5a2de920..0000000000 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Convert frontier objects into canonical JSON-compatible values.""" - -from __future__ import annotations - -import dataclasses -import datetime -import decimal -import enum -import functools -import hashlib -import json -import math -import re -import types -from collections.abc import Callable, Mapping, Sequence -from collections.abc import Set as AbstractSet -from pathlib import PurePath -from typing import Any - -from sqlbuild.compiler.frontier.constants import ( - STAGE_CAPTURE_CANONICAL_TYPES, - STAGE_CAPTURE_DECODED_SEQUENCES, - STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN, - STAGE_CAPTURE_INVOCATION_ID_MASK, - STAGE_CAPTURE_INVOCATION_ID_PATTERN, - STAGE_CAPTURE_OMITTED_ATTRIBUTES, - STAGE_CAPTURE_PROJECTED_ATTRIBUTES, - STAGE_CAPTURE_REFACTOR_STAGING_MASK, - STAGE_CAPTURE_REFACTOR_STAGING_PATTERN, - STAGE_CAPTURE_RESERVED_KEYS, - STAGE_CAPTURE_SHARED_MARKER, - STAGE_CAPTURE_SHARED_MIN_BYTES, - STAGE_CAPTURE_SKIPPED_SLOTS, - STAGE_CAPTURE_UNORDERED_ATTRIBUTES, -) - -_CALLABLE_TYPES: tuple[type, ...] = ( - types.FunctionType, - types.BuiltinFunctionType, - types.MethodType, - types.BuiltinMethodType, - type, -) - - -class StageCaptureEncoder: - """Encode an object graph deterministically, sharing large subtrees by content digest.""" - - def __init__( - self, - *, - on_shared: Callable[..., None] | None = None, - share_min_bytes: int = STAGE_CAPTURE_SHARED_MIN_BYTES, - ) -> None: - self._active: set[int] = set() - self._cycles: int = 0 - self._encoded: dict[int, tuple[object, object]] = {} - self._on_shared: Callable[..., None] | None = on_shared - self._share_min_bytes: int = share_min_bytes - self._digests: set[str] = set() - - def encode(self, value: object) -> object: - """Return a JSON-compatible value that keeps insertion order and sorts only sets.""" - - if isinstance(value, enum.Enum): - return {"__enum__": f"{qualified_name(type(value))}.{value.name}"} - if isinstance(value, str): - return value if len(value) < self._share_min_bytes else self._share(value) - if value is None or isinstance(value, (bool, int)): - return value - if isinstance(value, float): - return value if math.isfinite(value) else {"__float__": repr(value)} - scalar: object | None = _encode_scalar(value) - if scalar is not None: - return scalar - identity: int = id(value) - if identity in self._active: - self._cycles += 1 - return {"__cycle__": qualified_name(type(value))} - known: tuple[object, object] | None = self._encoded.get(identity) - if known is not None: - return known[1] - cycles: int = self._cycles - self._active.add(identity) - try: - encoded: object = self._share(self._encode_container(value)) - finally: - self._active.discard(identity) - if self._cycles == cycles: - self._encoded[identity] = (value, encoded) - return encoded - - def _share(self, encoded: object) -> object: - if self._on_shared is None or not isinstance(encoded, (dict, list, str)): - return encoded - text: str = json.dumps(encoded, ensure_ascii=False, allow_nan=False, separators=(",", ":")) - if len(text) < self._share_min_bytes: - return encoded - digest: str = hashlib.sha256(mask_capture_noise(text).encode("utf-8")).hexdigest() - if digest not in self._digests: - self._digests.add(digest) - self._on_shared(digest=digest, text=text) - return {STAGE_CAPTURE_SHARED_MARKER: digest} - - def _encode_container(self, value: object) -> object: - type_name: str = qualified_name(type(value)) - if type_name in STAGE_CAPTURE_DECODED_SEQUENCES and isinstance(value, Sequence): - return [self.encode(item) for item in value] - if dataclasses.is_dataclass(value) and not isinstance(value, type): - omitted_fields: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( - type_name, frozenset() - ) - return { - "__type__": type_name, - **{ - field.name: self.encode(getattr(value, field.name, None)) - for field in dataclasses.fields(value) - if field.name not in omitted_fields - }, - } - if isinstance(value, Mapping): - return self._encode_mapping(value) - if isinstance(value, AbstractSet): - return {"__set__": sorted((self.encode(item) for item in value), key=_sort_key)} - if isinstance(value, (list, tuple)): - return [self.encode(item) for item in value] - if isinstance(value, functools.partial): - return { - "__partial__": self.encode(value.func), - "args": self.encode(value.args), - "keywords": self.encode(value.keywords), - } - projected: tuple[str, ...] | None = STAGE_CAPTURE_PROJECTED_ATTRIBUTES.get(type_name) - attributes: dict[str, object] | None = ( - _instance_attributes(value) - if projected is None - else {name: getattr(value, name) for name in projected} - ) - if attributes is None: - return {"__opaque__": type_name} - unordered: frozenset[str] = STAGE_CAPTURE_UNORDERED_ATTRIBUTES.get(type_name, frozenset()) - omitted: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(type_name, frozenset()) - return { - "__type__": STAGE_CAPTURE_CANONICAL_TYPES.get(type_name, type_name), - **{ - name: self._encode_attribute(value=item, unordered=name in unordered) - for name, item in attributes.items() - if name not in omitted - }, - } - - def _encode_attribute(self, *, value: object, unordered: bool) -> object: - if not unordered or not isinstance(value, Mapping): - return self.encode(value) - return {"__unordered_mapping__": sorted(self._mapping_pairs(value), key=_sort_key)} - - def _encode_mapping(self, value: Mapping[Any, object]) -> object: - if all(isinstance(key, str) for key in value) and STAGE_CAPTURE_RESERVED_KEYS.isdisjoint( - value - ): - return {str(key): self.encode(item) for key, item in value.items()} - return {"__mapping__": self._mapping_pairs(value)} - - def _mapping_pairs(self, value: Mapping[Any, object]) -> list[list[object]]: - return [[self.encode(key), self.encode(item)] for key, item in value.items()] - - -def qualified_name(value: object) -> str: - """Return `module:qualname` for a callable or class.""" - - module: object = getattr(value, "__module__", None) - name: object = getattr(value, "__qualname__", None) or getattr(value, "__name__", None) - if name is None: - return qualified_name(type(value)) - return f"{module}:{name}" if isinstance(module, str) else str(name) - - -def _encode_scalar(value: object) -> object | None: - if isinstance(value, PurePath): - return {"__path__": value.as_posix()} - if isinstance(value, decimal.Decimal): - return {"__decimal__": str(value)} - if isinstance(value, (datetime.date, datetime.time)): - return {"__datetime__": value.isoformat()} - if isinstance(value, (bytes, bytearray, memoryview)): - data: bytes = bytes(value) - return {"__bytes__": len(data), "sha256": hashlib.sha256(data).hexdigest()} - if isinstance(value, re.Pattern): - return {"__pattern__": str(value.pattern)} - if isinstance(value, types.ModuleType): - return {"__module__": value.__name__} - if isinstance(value, _CALLABLE_TYPES): - return {"__callable__": qualified_name(value)} - return None - - -def mask_capture_noise(text: str) -> str: - """Mask invocation ids, native store suffixes and refactor staging folders of a compile.""" - - masked: str = STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN.sub( - "", - STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub(STAGE_CAPTURE_INVOCATION_ID_MASK, text), - ) - return STAGE_CAPTURE_REFACTOR_STAGING_PATTERN.sub(STAGE_CAPTURE_REFACTOR_STAGING_MASK, masked) - - -def _sort_key(item: object) -> str: - return mask_capture_noise(json.dumps(item, sort_keys=True, ensure_ascii=False)) - - -def _instance_attributes(value: object) -> dict[str, object] | None: - instance_dict: object = getattr(value, "__dict__", None) - attributes: dict[str, object] = ( - {str(name): item for name, item in instance_dict.items()} - if isinstance(instance_dict, dict) - else {} - ) - for cls in type(value).__mro__: - slots: object = cls.__dict__.get("__slots__", ()) - for name in (slots,) if isinstance(slots, str) else slots: - if ( - isinstance(name, str) - and name not in STAGE_CAPTURE_SKIPPED_SLOTS - and hasattr(value, name) - ): - attributes[name] = getattr(value, name) - if not attributes and not isinstance(instance_dict, dict): - return None - return attributes diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py deleted file mode 100644 index 786ba5da74..0000000000 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Process-wide ordering of stage captures.""" - -from __future__ import annotations - -import itertools -import threading -from typing import ClassVar - - -class StageCaptureSequence: - """Number stage captures in the order this process produced them.""" - - _counter: ClassVar[itertools.count[int]] = itertools.count(1) - _lock: ClassVar[threading.Lock] = threading.Lock() - - @classmethod - def next_value(cls) -> int: - """Return the next capture number.""" - - with cls._lock: - return next(cls._counter) diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_writer.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_writer.py deleted file mode 100644 index 1dd17b9ad1..0000000000 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_writer.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Stream one stage capture so shared subtrees reach the file as they are encoded.""" - -from __future__ import annotations - -import json -from typing import TextIO - -from sqlbuild.compiler.frontier.constants import ( - STAGE_CAPTURE_ROOT_KEY, - STAGE_CAPTURE_SHARED_NODES_KEY, -) - - -class StageCaptureWriter: - """Write a capture as one JSON document with each shared node and the root on its own line.""" - - def __init__(self, stream: TextIO) -> None: - self._stream: TextIO = stream - self._shared: bool = False - - def write_shared(self, *, digest: str, text: str) -> None: - """Append one shared node, given as compact JSON.""" - - prefix: str = ( - ",\n" if self._shared else f"{{{json.dumps(STAGE_CAPTURE_SHARED_NODES_KEY)}: {{\n" - ) - self._shared = True - _ = self._stream.write(f"{prefix}{json.dumps(digest)}: {text}") - - def finish(self, root: object) -> None: - """Write the root and close the document.""" - - if not self._shared: - _ = self._stream.write( - json.dumps(root, indent=1, ensure_ascii=False, allow_nan=False) + "\n" - ) - return - compact: str = json.dumps(root, ensure_ascii=False, allow_nan=False, separators=(",", ":")) - _ = self._stream.write(f"\n}},\n{json.dumps(STAGE_CAPTURE_ROOT_KEY)}: {compact}\n}}\n") diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 9f28dee1d6..cd8407ff47 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -1,6 +1,8 @@ """Compiler cache namespace and stage capture constants.""" -import re +from __future__ import annotations + +import itertools NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" RETIRED_CACHE_NAMESPACE_SUFFIXES: tuple[str, ...] = ( @@ -17,284 +19,4 @@ STAGE_CAPTURE_SHARED_MARKER: str = "__shared__" STAGE_CAPTURE_ROOT_KEY: str = "__capture__" STAGE_CAPTURE_SHARED_NODES_KEY: str = "__shared_nodes__" -STAGE_CAPTURE_SHARED_MIN_BYTES: int = 1024 -STAGE_CAPTURE_RESERVED_KEYS: frozenset[str] = frozenset( - {STAGE_CAPTURE_SHARED_MARKER, STAGE_CAPTURE_ROOT_KEY, STAGE_CAPTURE_SHARED_NODES_KEY} -) -STAGE_CAPTURE_INVOCATION_ID_PATTERN: re.Pattern[str] = re.compile(r"\b\d{8}T\d{6}Z_[0-9a-f]{12}\b") -STAGE_CAPTURE_INVOCATION_ID_MASK: str = "" -STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native-v\d+\b") -STAGE_CAPTURE_REFACTOR_STAGING_PATTERN: re.Pattern[str] = re.compile(r"\bsqb-refactor-[0-9a-z_]+") -STAGE_CAPTURE_REFACTOR_STAGING_MASK: str = "sqb-refactor-" -STAGE_CAPTURE_SKIPPED_SLOTS: frozenset[str] = frozenset({"__dict__", "__weakref__"}) -STAGE_CAPTURE_UNORDERED_ATTRIBUTES: dict[str, frozenset[str]] = { - "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( - {"schemas", "analysis_shapes", "expression_shapes"} - ) -} -STAGE_CAPTURE_PROJECTED_ATTRIBUTES: dict[str, tuple[str, ...]] = { - "sqlbuild._native:DiscoveredSqlModelFile": ( - "file_path", - "relative_path", - "contents", - "header_values", - "header_column_locations", - "output_column_locations", - "query_sql", - "enum_declarations", - "constant_declarations", - "extract_implicit_alias_columns", - ), - "sqlbuild._native:NativeAuditInput": ( - "audit_file", - "audit_block", - "sql_body", - "evaluation_mode", - "measurement_contract", - "thresholds", - "minimum_samples", - "measure_sql", - "evidence_sql", - "evidence_limit", - "references", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "severity", - "run_scope", - "always_run", - "description", - "declaration_usages", - "name", - ), - "sqlbuild._native:NativeCompiledAudit": ( - "key", - "scope_deps", - "name", - "definition_name", - "audit_file", - "audit_block", - "sql_body", - "evaluation_mode", - "measurement_contract", - "thresholds", - "minimum_samples", - "measure_sql", - "evidence_sql", - "evidence_limit", - "references", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "severity", - "run_scope", - "always_run", - "description", - ), - "sqlbuild._native:NativeFunctionInput": ( - "function_file", - "name", - "arguments", - "returns", - "body_sql", - "return_columns", - "references", - "database", - "schema", - "logical_database", - "logical_schema", - "fingerprint_database", - "fingerprint_schema", - "fingerprint_logical_database", - "fingerprint_logical_schema", - "language", - "runtime_version", - "entry_point", - "packages", - "declaration_usages", - "tags", - "description", - ), - "sqlbuild._native:NativeCompiledFunction": ( - "key", - "deps", - "name", - "relative_path", - "arguments", - "returns", - "body_sql", - "destination", - "fingerprint_destination", - "return_columns", - "references", - "language", - "source_file_path", - "runtime_version", - "entry_point", - "packages", - "tags", - "description", - ), - "sqlbuild._native:NativeSourceInput": ( - "source_entry", - "source_file", - "declaration_usages", - ), - "sqlbuild._native:NativeCompiledSource": ( - "key", - "deps", - "name", - "source_entry", - "source_file", - ), - "sqlbuild._native:NativeSeedInput": ("seed_file", "schema_entry", "schema_file"), - "sqlbuild._native:NativeCompiledSeed": ( - "key", - "deps", - "name", - "seed_file", - "schema_entry", - "schema_file", - "destination", - "external", - ), - "sqlbuild._native:NativeInputsView": ( - "project_config", - "local_config", - "discovered_inputs", - "sql_lexical_syntax", - "run_id", - "effective_target_name", - "effective_target", - "compile_cache_dir", - "effective_connection", - "effective_settings", - "no_sql_validation", - "effective_vars", - "macro_context", - "loaded_macros", - "public_enums", - "public_constants", - "model_inputs", - "seed_inputs", - "source_inputs", - "sql_function_inputs", - "test_inputs", - "scenario_inputs", - "audit_inputs", - "diagnostics", - "external_sql_reference_resolver", - "scope_index", - "declaration_scope", - ), - "sqlbuild._native:NativeProjectView": ( - "run_id", - "effective_target_name", - "effective_connection", - "effective_vars", - "binding_catalog", - "effective_target_database", - "effective_target_schema", - "sql_analysis_dialect", - "sql_lexical_syntax", - "compile_cache_dir", - "settings", - "scenario", - "models", - "sources", - "seeds", - "functions", - "audits", - "sql_tests", - "sql_scenarios", - "loader_functions", - "hook_functions", - "enforce_explicit_references", - "sql_hook_files", - "materialization_files", - "public_enums", - "public_constants", - "loaded_macros", - "diagnostics", - "unmatched_literal_sql_relations", - "external_sql_reference_resolver", - "scope_index", - "sql_expansions", - ), - "sqlbuild._native:NativeCompiledModel": ( - "key", - "deps", - "name", - "relative_path", - "query_sql", - "config", - "destination", - "references", - "schema_entry", - "inferred_columns", - "fast_lineage_columns", - "fast_lineage_has_star", - "fast_lineage_star_resolved", - "authored_sql", - "authored_query_sql", - "output_column_locations", - "extract_implicit_alias_columns", - "macro_deps", - "enum_declarations", - "constant_declarations", - "enum_columns", - "binding_diagnostics", - "binding_validated", - "dynamic_column_contract", - "unchecked_output_columns", - "rejected_sql_analysis_opt_out", - ), - "sqlbuild._native:NativeModelInput": ( - "model_file", - "config", - "query_sql", - "macro_source_sql", - "references", - "schema_entry", - "schema_file", - "sql_validation_enabled", - "rejected_sql_analysis_opt_out", - "enum_declarations", - "constant_declarations", - "enum_columns", - "macro_deps", - "macro_usages", - "declaration_usages", - "sql_expansion", - "lint_expansion", - ), -} -STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] = { - "sqlbuild.compiler.scopes.models:ScopeLookup": frozenset({"native"}), - "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( - {"shared_analyses"} - ), - "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), - "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), - "sqlbuild.compiler.compile.models:DeclarationScopeBuild": frozenset({"sql_test_scans"}), - "sqlbuild.compiler.compile.models:CompileProjectInputs": frozenset({"native_project"}), - "sqlbuild.compiler.compile.models:CompiledProject": frozenset( - {"native_session", "native_project", "lineage_graph"} - ), -} -STAGE_CAPTURE_CANONICAL_TYPES: dict[str, str] = { - "sqlbuild._native:DiscoveredSqlModelFile": ( - "sqlbuild.compiler.discovery.models:DiscoveredSqlModelFile" - ), - "sqlbuild._native:NativeInputsView": "sqlbuild.compiler.compile.models:CompileProjectInputs", - "sqlbuild._native:NativeProjectView": "sqlbuild.compiler.compile.models:CompiledProject", - "sqlbuild._native:NativeModelInput": "sqlbuild.compiler.compile.models:CompileModelInput", - "sqlbuild._native:NativeCompiledModel": "sqlbuild.compiler.compile.models:CompiledModel", - "sqlbuild._native:NativeCompiledSource": "sqlbuild.compiler.compile.models:CompiledSource", - "sqlbuild._native:NativeCompiledSeed": "sqlbuild.compiler.compile.models:CompiledSeed", - "sqlbuild._native:NativeCompiledFunction": "sqlbuild.compiler.compile.models:CompiledFunction", - "sqlbuild._native:NativeCompiledAudit": "sqlbuild.compiler.compile.models:CompiledAudit", -} -STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( - {"sqlbuild.compiler.compile.models:CompactLineageFacts"} -) +STAGE_CAPTURE_SEQUENCE: itertools.count[int] = itertools.count(1) diff --git a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py deleted file mode 100644 index c815c36c8d..0000000000 --- a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py +++ /dev/null @@ -1,20 +0,0 @@ -"""The single seam between the compiler stages and the native frontier.""" - -from collections.abc import Callable - -from sqlbuild.compiler.frontier._helpers.native_frontier import native_frontier -from sqlbuild.compiler.frontier._helpers.stage_capture import write_stage_capture -from sqlbuild.compiler.frontier.types import CompilerStage - - -def compile_frontier[T]( - *, - until: CompilerStage, - python_stage: Callable[[], T], - native_stage: Callable[[], T] | None = None, -) -> T: - """Produce the frontier object for `until`, capturing it on request.""" - - result: T = native_frontier(until=until, python_stage=python_stage, native_stage=native_stage) - write_stage_capture(stage=until, value=result) - return result diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py index 600237decb..a22b1804b0 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -21,7 +21,6 @@ MACRO_CALL_STORE_ENVIRONMENT_VERSION, MAIN_MODULE_NAME, MISSING_SEARCH_PATH_STAMP, - MODULE_DIGEST_ROW_FIELDS, PROJECT_ROOT_SEARCH_PATH_STAMP, STORE_TRACKED_ENVIRONMENT_PREFIXES, STORE_UNTRACKED_ENVIRONMENT_NAMES, @@ -69,48 +68,6 @@ def module_sources(modules: Iterable[object]) -> ModuleSources: return ModuleSources(paths=tuple(paths), complete=True) -def digest_module_files(paths: Iterable[str]) -> dict[str, str] | None: - """Content digest of each module file, or None when any file cannot be read.""" - - ordered: list[str] = list(dict.fromkeys(paths)) - digests: list[str | None] = _native.digest_files(ordered) - if any(digest is None for digest in digests): - return None - return {path: digest for path, digest in zip(ordered, digests, strict=True) if digest} - - -def module_digests_metadata(digests: dict[str, str]) -> bytes: - """Encode module file digests for the store's metadata.""" - - return json.dumps(sorted([path, digest] for path, digest in digests.items())).encode() - - -def unchanged_module_digests(*, metadata: bytes, known: dict[str, str]) -> dict[str, str] | None: - """Return the stored module digests when every module file still has them, else None.""" - - try: - rows: object = json.loads(metadata) - except ValueError: - return None - if not isinstance(rows, list) or not all(map(_is_digest_row, rows)): - return None - stored: dict[str, str] = {row[0]: row[1] for row in rows} - unknown: list[str] = [path for path in stored if path not in known] - current: dict[str, str] | None = digest_module_files(unknown) - if current is None: - return None - current.update((path, known[path]) for path in stored if path in known) - return stored if current == stored else None - - -def _is_digest_row(row: object) -> bool: - return ( - isinstance(row, list) - and len(row) == MODULE_DIGEST_ROW_FIELDS - and all(isinstance(field, str) for field in row) - ) - - def _backing_file(module: object) -> tuple[str | None, bool]: spec: object = getattr(module, "__spec__", None) file: object = getattr(module, "__file__", None) diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py deleted file mode 100644 index 565800f529..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Value-based text of everything a stored macro call result depends on besides its call text.""" - -from __future__ import annotations - -import dataclasses -import datetime -import decimal -import enum -import json -import types -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - LoadedMacro, - MacroContext, -) -from sqlbuild.compiler.macro_bridge.constants import VALUE_RENDERER_FIELD -from sqlbuild.compiler.macro_bridge.exceptions import UncacheableValueError -from sqlbuild.compiler.scopes.models import DeclarationIdentity - -_UNSTABLE_KEY_ERRORS: tuple[type[Exception], ...] = ( - UncacheableValueError, - UnicodeEncodeError, - RecursionError, -) - - -def macro_store_token(*, loaded: LoadedMacro, identity: DeclarationIdentity | None) -> str | None: - """Name a macro by its name, file, identity and exact source, or None when it has no key.""" - - try: - return _encodable( - _key_text( - [ - loaded.name, - loaded.relative_path.as_posix(), - _canonical(identity), - _native.content_digest([loaded.raw_source]), - ] - ) - ) - except _UNSTABLE_KEY_ERRORS: - return None - - -def context_store_token( - *, macro_context: MacroContext, declarations: DeclarationResolutionContext | None -) -> str | None: - """Digest the whole context a macro may read, or None when a value has no stable text.""" - - try: - return _native.content_digest( - [ - _key_text( - [ - _canonical_context(macro_context), - None - if declarations is None - else [ - _canonical(declarations.constants), - _canonical(declarations.enums), - sorted(declarations.inaccessible_constants), - sorted(declarations.inaccessible_enums), - ], - ] - ) - ] - ) - except _UNSTABLE_KEY_ERRORS: - return None - - -def _canonical_context(macro_context: MacroContext) -> object: - fields: list[object] = [] - for field in dataclasses.fields(macro_context): - value: object = getattr(macro_context, field.name) - fields.append( - [ - field.name, - _type_name(type(value)) - if field.name == VALUE_RENDERER_FIELD - else _canonical(value), - ] - ) - return [_type_name(type(macro_context)), fields] - - -def _canonical(value: object) -> object: # noqa: PLR0911 - if isinstance(value, enum.Enum): - return ["enum", _type_name(type(value)), _canonical(value.value)] - if value is None or isinstance(value, bool | str): - return value - if isinstance(value, int): - return ["int", str(value)] - if isinstance(value, float): - return ["float", value.hex()] - if isinstance(value, decimal.Decimal): - return ["decimal", str(value)] - if isinstance(value, datetime.date | datetime.time): - return [_type_name(type(value)), value.isoformat()] - if isinstance(value, datetime.timedelta): - return ["timedelta", value.days, value.seconds, value.microseconds] - if isinstance(value, bytes): - return ["bytes", value.hex()] - if isinstance(value, PurePath): - return ["path", value.as_posix()] - if isinstance(value, list | tuple): - return [_type_name(type(value)), [_canonical(item) for item in value]] - if isinstance(value, set | frozenset): - return [ - _type_name(type(value)), - sorted(json.dumps(_canonical(item), sort_keys=True) for item in value), - ] - if isinstance(value, dict | types.MappingProxyType): - return [ - "mapping", - [[_canonical(key), _canonical(item)] for key, item in value.items()], - ] - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return [ - _type_name(type(value)), - [ - [field.name, _canonical(getattr(value, field.name))] - for field in dataclasses.fields(value) - ], - ] - raise UncacheableValueError(_type_name(type(value))) - - -def _type_name(kind: type) -> str: - return f"{kind.__module__}.{kind.__qualname__}" - - -def _key_text(value: object) -> str: - return json.dumps(value, ensure_ascii=False, separators=(",", ":")) - - -def _encodable(text: str) -> str: - """Return `text`, raising `UnicodeEncodeError` when the native store cannot hold it.""" - - _ = text.encode("utf-8") - return text diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index ebbc32ba4e..c46880de37 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -7,13 +7,15 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( +from sqlbuild._native import ( digest_module_files, module_digests_metadata, + unchanged_module_digests, +) +from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( module_sources, project_fingerprint, store_environment, - unchanged_module_digests, ) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.compiler.macro_bridge.models import ( @@ -72,19 +74,21 @@ def save_store(self) -> None: return project_dir, model_paths, fingerprint = self._project_inputs try: - if ( - not self.observe_modules() - or digest_module_files(self._module_digests) != self._module_digests - or project_fingerprint(project_dir=project_dir, model_paths=model_paths) - != fingerprint - ): + observed: bool = self.observe_modules() + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("Macro call store not saved: %s", error) + return + if not observed or digest_module_files(self._module_digests) != self._module_digests: + logging.getLogger(__name__).debug("Macro call store not saved: code or files changed") + return + metadata: bytes = module_digests_metadata(self._module_digests) + try: + if project_fingerprint(project_dir=project_dir, model_paths=model_paths) != fingerprint: logging.getLogger(__name__).debug( "Macro call store not saved: code or files changed" ) return - _ = self.memo.save_store( - str(self._store_path), module_digests_metadata(self._module_digests) - ) + _ = self.memo.save_store(str(self._store_path), metadata) except (OSError, RuntimeError) as error: logging.getLogger(__name__).debug("Macro call store not saved: %s", error) diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 860f685bbb..3dcfd15726 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -24,7 +24,6 @@ VALUE_RENDERER_FIELD: str = "_value_renderer" PROJECT_ROOT_SEARCH_PATH_STAMP: int = -2 MISSING_SEARCH_PATH_STAMP: int = -1 -MODULE_DIGEST_ROW_FIELDS: int = 2 INSTALLED_PACKAGE_DIRECTORY_NAMES: frozenset[str] = frozenset({"site-packages", "dist-packages"}) INSTALLED_DISTRIBUTION_SUFFIX: str = ".dist-info" INSTALLED_RECORD_FILE_NAME: str = "RECORD" diff --git a/src/sqlbuild/compiler/macro_bridge/exceptions.py b/src/sqlbuild/compiler/macro_bridge/exceptions.py deleted file mode 100644 index e3e4181d26..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Macro bridge exceptions.""" - -from __future__ import annotations - - -class UncacheableValueError(Exception): - """A value a macro may read has no stable text, so calls that may read it are not stored.""" diff --git a/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py b/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py deleted file mode 100644 index a8e9e0379b..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py +++ /dev/null @@ -1,14 +0,0 @@ -"""The value text of a stored macro call's inputs, as the native memo keys persistent classes.""" - -from __future__ import annotations - -from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( - context_store_token, - macro_store_token, -) - - -def macro_store_tokens() -> dict[str, object]: - """`macro_store_token` and `context_store_token`, by the name the native memo reads them.""" - - return {"macro_token": macro_store_token, "context_token": context_store_token} diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 75b8a930ef..596615324a 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -49,31 +49,6 @@ def apply_unit_test_selectors( ) -def resolve_compile_analysis_selection( - *, - options: CompilePipelineOptions, - discovered_inputs: DiscoveredProjectInputs, -) -> CompileAnalysisSelection | None: - """Limit analysis when selectors cannot resolve through Python graph nodes.""" - - has_python_nodes: bool = bool( - discovered_inputs.loader_functions - or discovered_inputs.task_functions - or discovered_inputs.asset_functions - or discovered_inputs.check_functions - ) - if ( - options.resolve_python_run_selectors and has_python_nodes - ) or options.sql_test_selection.names: - return CompileAnalysisSelection(no_cache=options.no_cache) - return CompileAnalysisSelection( - select=options.select, - exclude=options.exclude, - auto_load_sources=options.auto_load_sources, - no_cache=options.no_cache, - ) - - def resolve_configured_rule_selection( *, options: CompilePipelineOptions, @@ -84,9 +59,11 @@ def resolve_configured_rule_selection( if not options.sql_test_selection.models_selected: return frozenset() - analysis_selection: CompileAnalysisSelection | None = resolve_compile_analysis_selection( - options=options, - discovered_inputs=discovered_inputs, + analysis_selection: CompileAnalysisSelection | None = ( + _native.resolve_compile_analysis_selection( + options=options, + discovered_inputs=discovered_inputs, + ) ) if analysis_selection is None or ( not analysis_selection.select and not analysis_selection.exclude diff --git a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py b/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py deleted file mode 100644 index af64844bfe..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Compute deferred relation locations and gather deferred relations.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.constants import DEFERRED_TARGET_CONTEXT_KEYS -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_deferred_locations( - *, - project: CompiledProjectView, - deferred_target_config: TargetConfig, - effective_vars: dict[str, object], - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], -) -> dict[str, CompiledRelationLocation]: - """Build physical locations for all models and seeds under a deferred target.""" - - locations: dict[str, CompiledRelationLocation] = {} - model: CompiledModelView - for model in project.models: - locations[model.name] = _resolve_deferred_location( - target=model.destination, - deferred_target_config=deferred_target_config, - effective_vars=effective_vars, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ) - seed: CompiledSeedView - for seed in project.seeds: - locations[seed.name] = _resolve_deferred_location( - target=seed.destination, - deferred_target_config=deferred_target_config, - effective_vars=effective_vars, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ) - return locations - - -def resolve_deferred_target_config( - *, - discovered_inputs: DiscoveredProjectInputs, - defer_to: str, - current_target_name: str | None, -) -> TargetConfig: - """Validate and resolve the deferred target config.""" - - targets: dict[str, TargetConfig] = discovered_inputs.project_config.targets - if defer_to not in targets: - raise PlannerInputError(f"Unknown deferred target '{defer_to}'") - if defer_to == current_target_name: - raise PlannerInputError(f"Cannot defer to the current target '{defer_to}'") - return targets[defer_to] - - -def gather_deferred_relations( - *, - adapter: BaseAdapter, - connection: Any, - deferred_locations: dict[str, CompiledRelationLocation], -) -> dict[str, RelationInfo]: - """Gather existing relations from the deferred target's relation-location schemas.""" - - schemas: set[str] = set() - database: str | None = None - location: CompiledRelationLocation - for location in deferred_locations.values(): - if location.schema is not None: - schemas.add(location.schema) - if location.database is not None and database is None: - database = location.database - - if not schemas: - return {} - - relations: tuple[RelationInfo, ...] = adapter.list_relations( - connection=connection, database=database, schemas=tuple(sorted(schemas)) - ) - return {rel.name: rel for rel in relations} - - -def _resolve_deferred_location( - *, - target: CompiledRelationLocation, - deferred_target_config: TargetConfig, - effective_vars: dict[str, object], - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], -) -> CompiledRelationLocation: - """Resolve one relation location under the deferred target's naming rules.""" - - schema: str | None = _resolve_target_field( - target_value=deferred_target_config.schema, - logical_value=target.logical_schema, - effective_vars=effective_vars, - ) - database: str | None = _resolve_target_field( - target_value=deferred_target_config.database, - logical_value=target.logical_database, - effective_vars=effective_vars, - ) - - if schema is None: - schema = default_schema - if database is None: - database = default_database - - qualified_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=target.name, - ) - return CompiledRelationLocation( - database=database, - schema=schema, - name=target.name, - qualified_name=qualified_name, - logical_schema=target.logical_schema, - logical_database=target.logical_database, - ) - - -def _resolve_target_field( - *, - target_value: str | None, - logical_value: str | None, - effective_vars: dict[str, object], -) -> str | None: - """Resolve one target schema or database field against the logical value.""" - - if target_value is None or target_value == PRESERVE_TARGET_VALUE: - return logical_value - - segments: list[tuple[bool, str]] = _native.deferred_target_segments( - target_value, - "" if logical_value is None else logical_value, - sorted(DEFERRED_TARGET_CONTEXT_KEYS), - [name for name in effective_vars if isinstance(name, str)], - ) - return "".join( - render_project_var_text( - value=effective_vars[text], label=f"deferred target variable '${{{text}}}'" - ) - if is_variable - else text - for is_variable, text in segments - ) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/diff.py b/src/sqlbuild/compiler/pipeline/_helpers/diff.py deleted file mode 100644 index 2b2884a36d..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/diff.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Diff pipeline assembly helpers.""" - -from __future__ import annotations - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views -from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.graph.models import LineageGraphViews -from sqlbuild.compiler.pipeline.constants import ( - MODEL_PATH_ROOT, - PATH_SELECTOR_PREFIX, - PATH_SEPARATORS, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def compile_project_for_diff_environment( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - target_name: str, - resolved_connection: dict[str, object], - no_sql_validation: bool, - cli_vars: dict[str, object] | None = None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> CompiledProjectView: - """Compile a project for one diff environment.""" - - return build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=target_name, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def resolve_diff_model_names( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> tuple[str, ...]: - """Resolve diff selectors to model names only.""" - - lineage: LineageGraphViews = project_lineage_views(project) - all_keys: dict[str, CompiledObjectKey] = { - **{model.name: model.key for model in project.models}, - **{source.name: source.key for source in project.sources}, - **{seed.name: seed.key for seed in project.seeds}, - } - selected_keys: frozenset[CompiledObjectKey] = _resolve_model_selectors( - select=select, - exclude=exclude, - all_keys=all_keys, - upstream=lineage.upstream_deps, - downstream=lineage.downstream_deps, - tag_index=lineage.tag_index, - path_index=lineage.path_index, - ) - return tuple( - model.name - for model in project.models - if model.key in selected_keys and model.key.resource_type == CompiledResourceType.MODEL - ) - - -def _resolve_model_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - selected: set[CompiledObjectKey] = set() - raw_select: str - for raw_select in select: - token: str - for token in raw_select.split(): - selected.update( - _resolve_selector_token( - token=token, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - ) - excluded: set[CompiledObjectKey] = set() - raw_exclude: str - for raw_exclude in exclude: - for token in raw_exclude.split(): - excluded.update( - _resolve_selector_token( - token=token, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - ) - return frozenset(selected - excluded) - - -def _resolve_selector_token( - *, - token: str, - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - parts: list[str] = token.split(",") - if len(parts) > 1: - resolved_parts: list[frozenset[CompiledObjectKey]] = [ - _resolve_selector_token( - token=part, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - for part in parts - ] - result: frozenset[CompiledObjectKey] = resolved_parts[0] - part_result: frozenset[CompiledObjectKey] - for part_result in resolved_parts[1:]: - result = result & part_result - return result - - upstream_requested: bool = token.startswith("+") - downstream_requested: bool = token.endswith("+") - core: str = token.lstrip("+").rstrip("+") - if not core: - raise PlannerInputError("empty selector", code="S001") - if core.startswith("tag:"): - keys: frozenset[CompiledObjectKey] = tag_index.get(core.removeprefix("tag:"), frozenset()) - if not keys: - raise PlannerInputError( - f"no models found with tag '{core.removeprefix('tag:')}'", - code="S008", - ) - return _expand_keys( - keys=keys, - upstream_requested=upstream_requested, - downstream_requested=downstream_requested, - upstream=upstream, - downstream=downstream, - ) - if core.startswith(PATH_SELECTOR_PREFIX) or any( - separator in core for separator in PATH_SEPARATORS - ): - path_value: str = core.removeprefix(PATH_SELECTOR_PREFIX).replace("\\", "/").strip("/") - if path_value == MODEL_PATH_ROOT: - path_value = "" - elif path_value.startswith("models/"): - path_value = path_value.removeprefix("models/") - keys = frozenset( - key - for key, folder in path_index.items() - if folder == path_value or folder.startswith(f"{path_value}/") - ) - if not keys: - raise PlannerInputError(f"no models found under path '{path_value}'", code="S009") - return _expand_keys( - keys=keys, - upstream_requested=upstream_requested, - downstream_requested=downstream_requested, - upstream=upstream, - downstream=downstream, - ) - - key: CompiledObjectKey | None = all_keys.get(core) - if key is None: - raise PlannerInputError(f"unknown selector name '{core}'", code="S007") - return _expand_keys( - keys=frozenset((key,)), - upstream_requested=upstream_requested, - downstream_requested=downstream_requested, - upstream=upstream, - downstream=downstream, - ) - - -def _expand_keys( - *, - keys: frozenset[CompiledObjectKey], - upstream_requested: bool, - downstream_requested: bool, - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - expanded: set[CompiledObjectKey] = set(keys) - if upstream_requested: - expanded.update(transitive_closure_many(starts=keys, edges=upstream, include_starts=False)) - if downstream_requested: - expanded.update( - transitive_closure_many(starts=keys, edges=downstream, include_starts=False) - ) - return frozenset(expanded) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/graph.py b/src/sqlbuild/compiler/pipeline/_helpers/graph.py deleted file mode 100644 index 21ce3d6f05..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/graph.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Static compiled project graph helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection._graph_selection import resolve_graph_selection - - -def build_project_graph_impl(project: CompiledProjectView) -> ProjectGraph: - """The project's natively held graph; dict views are built only when a boundary reads them.""" - - return ProjectGraph(project=project, native=project.lineage_graph) - - -def select_project_graph_impl( - *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> frozenset[CompiledObjectKey]: - """Resolve selectors on the graph's native handle.""" - - return resolve_graph_selection(graph=graph.native, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py b/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py deleted file mode 100644 index 8d7f564267..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Build display-ready Python entries for plan output.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure -from sqlbuild.compiler.pipeline.models import PythonPlanEntry, PythonRunPlanOutputs -from sqlbuild.compiler.planner.models import PlanOutput, PlanWarning -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.compiler.python_nodes.types import ( - PythonIdentityStatus, - PythonNodeKind, - PythonRunPhase, -) -from sqlbuild.python_nodes.types import SqlResourceRefKind - - -def python_read_source_names_for( - *, discovered_inputs: DiscoveredProjectInputs, python_node_names: frozenset[str] -) -> frozenset[str]: - """Return sources declared by the selected Python nodes and their Python upstreams.""" - - if not python_node_names: - return frozenset() - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - return frozenset( - ref.name - for ref in python_graph.selected_sql_refs(selected_names=python_node_names) - if ref.kind == SqlResourceRefKind.SOURCE - ) - - -def build_python_run_plan_outputs( - *, - discovered_inputs: DiscoveredProjectInputs, - plan_output: PlanOutput, - run_selection: PythonSqlRunSelection | None, - selected_python_node_names: frozenset[str], -) -> PythonRunPlanOutputs: - """Attach python-aware selection warnings and plan entries to the plan output.""" - - if run_selection is None: - return PythonRunPlanOutputs( - plan_output=plan_output, - python_plan_entries=(), - selected_python_node_names=selected_python_node_names, - ) - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - plan_output = replace( - plan_output, - warnings=( - *plan_output.warnings, - *build_skipped_task_asset_ingress_warnings( - plan_output=plan_output, - run_selection=run_selection, - python_graph=python_graph, - ), - ), - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=PythonSqlRunSelection( - sql_keys=plan_output.selected_keys, - python_node_names=selected_python_node_names, - ), - python_graph=python_graph, - ) - return PythonRunPlanOutputs( - plan_output=plan_output, - python_plan_entries=build_python_plan_entries( - lifecycle_plan=lifecycle_plan, - python_graph=python_graph, - previous_identities=plan_output.python_identity_fingerprints, - ), - selected_python_node_names=selected_python_node_names, - ) - - -def build_python_plan_entries( - *, - lifecycle_plan: PythonSqlRunLifecyclePlan, - python_graph: PythonNodeGraph, - previous_identities: dict[tuple[str, str], Fingerprint] | None = None, -) -> tuple[PythonPlanEntry, ...]: - """Return task/asset plan entries ordered by lifecycle dependency readiness.""" - - entries: list[PythonPlanEntry] = [] - resolved_previous_identities: dict[tuple[str, str], Fingerprint] = ( - {} if previous_identities is None else previous_identities - ) - node_name: str - for node_name in _ordered_python_names( - selected_names=lifecycle_plan.ingress_python_node_names, - python_graph=python_graph, - ): - node: DiscoveredPythonNode = python_graph.nodes_by_name[node_name] - if node.kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - previous_identity: Fingerprint | None = _previous_identity( - node=node, - previous_identities=resolved_previous_identities, - ) - entries.append( - PythonPlanEntry( - name=node.name, - kind=node.kind, - phase=PythonRunPhase.PRE_SQL_INGRESS, - identity_status=_identity_status( - node=node, - previous_identities=resolved_previous_identities, - ), - current_definition_json=( - node.identity.definition_json if node.identity is not None else None - ), - previous_definition_json=( - previous_identity.definition if previous_identity is not None else None - ), - current_metadata_json=( - node.identity.metadata_json if node.identity is not None else None - ), - previous_metadata_json=( - previous_identity.metadata_json if previous_identity is not None else None - ), - provider_usages=node.provider_usages, - ) - ) - for node_name in _ordered_python_names( - selected_names=lifecycle_plan.read_side_python_node_names, - python_graph=python_graph, - ): - node = python_graph.nodes_by_name[node_name] - if node.kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - previous_identity = _previous_identity( - node=node, - previous_identities=resolved_previous_identities, - ) - entries.append( - PythonPlanEntry( - name=node.name, - kind=node.kind, - phase=PythonRunPhase.READ_SIDE, - identity_status=_identity_status( - node=node, - previous_identities=resolved_previous_identities, - ), - current_definition_json=( - node.identity.definition_json if node.identity is not None else None - ), - previous_definition_json=( - previous_identity.definition if previous_identity is not None else None - ), - current_metadata_json=( - node.identity.metadata_json if node.identity is not None else None - ), - previous_metadata_json=( - previous_identity.metadata_json if previous_identity is not None else None - ), - provider_usages=node.provider_usages, - ) - ) - return tuple(entries) - - -def _ordered_python_names( - *, selected_names: frozenset[str], python_graph: PythonNodeGraph -) -> tuple[str, ...]: - upstream_names: dict[str, tuple[str, ...]] = {} - for name in selected_names: - selected_upstreams: list[str] = [] - for upstream_name in python_graph.upstream_deps.get(name, ()): - if upstream_name in selected_names: - selected_upstreams.append(upstream_name) - upstream_names[name] = tuple(selected_upstreams) - downstream_names: dict[str, list[str]] = {name: [] for name in selected_names} - node_name: str - upstream_name: str - for node_name, upstreams in upstream_names.items(): - for upstream_name in upstreams: - downstream_names[upstream_name].append(node_name) - in_degree: dict[str, int] = {name: len(upstreams) for name, upstreams in upstream_names.items()} - ready: list[str] = sorted(name for name, degree in in_degree.items() if degree == 0) - ordered: list[str] = [] - while ready: - node_name = ready.pop(0) - ordered.append(node_name) - for downstream_name in sorted(downstream_names[node_name]): - in_degree[downstream_name] -= 1 - if in_degree[downstream_name] == 0: - ready.append(downstream_name) - ready.sort() - return tuple(ordered) - - -def _identity_status( - *, - node: DiscoveredPythonNode, - previous_identities: dict[tuple[str, str], Fingerprint], -) -> PythonIdentityStatus: - if node.identity is None: - return PythonIdentityStatus.UNKNOWN - previous: Fingerprint | None = _previous_identity( - node=node, - previous_identities=previous_identities, - ) - if previous is None: - return PythonIdentityStatus.NEW - if previous.version_hash == node.identity.version_hash: - return PythonIdentityStatus.UNCHANGED - return PythonIdentityStatus.CHANGED - - -def _previous_identity( - *, - node: DiscoveredPythonNode, - previous_identities: dict[tuple[str, str], Fingerprint], -) -> Fingerprint | None: - if node.identity is None: - return None - return previous_identities.get((node.identity.node_type, node.identity.node_name)) - - -def build_skipped_task_asset_ingress_warnings( - *, - plan_output: PlanOutput, - run_selection: PythonSqlRunSelection, - python_graph: PythonNodeGraph, -) -> tuple[PlanWarning, ...]: - """Return warnings for skipped task/asset deps of planned source loaders.""" - - warnings: list[PlanWarning] = [] - entry_loader_names: frozenset[str] = frozenset( - entry.loader for entry in plan_output.source_load_entries - ) - loader_name: str - for loader_name in sorted(entry_loader_names): - if loader_name not in python_graph.nodes_by_name: - continue - upstream_name: str - for upstream_name in sorted( - transitive_closure(start=loader_name, edges=python_graph.upstream_deps) - ): - if upstream_name in run_selection.python_node_names: - continue - upstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[upstream_name] - if upstream_node.kind not in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - continue - warnings.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.WARNING, - message=( - f"Source loader '{loader_name}' has unselected upstream " - f"{upstream_node.kind.value} '{upstream_name}'. SQLBuild will not run " - "it and cannot verify side effects or payload availability; use " - f"+source:{loader_name} to refresh upstream ingress dependencies." - ), - code="P501", - ) - ) - return tuple(warnings) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py deleted file mode 100644 index e33a69dee9..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Runtime SQL relation maps for Python node contexts and hard-coded name guards.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.references.main.render_source_relation import render_source_relation -from sqlbuild.errors.contracts.exceptions import SharedInputError -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_python_relation_targets_impl( - *, - adapter: BaseAdapter, - project: CompiledProjectView, - plan_output: PlanOutput, - required_refs: frozenset[SqlResourceRef] | None = None, -) -> dict[SqlResourceRef, str]: - """Return adapter-qualified runtime relations required by selected Python nodes.""" - - targets: dict[SqlResourceRef, str] = {} - refs: frozenset[SqlResourceRef] = ( - required_refs - if required_refs is not None - else _planned_relation_refs(plan_output=plan_output) - ) - ref: SqlResourceRef - for ref in refs: - targets[ref] = _resolve_relation( - adapter=adapter, - project=project, - plan_output=plan_output, - ref=ref, - ) - return targets - - -def build_project_relation_targets_impl( - *, adapter: BaseAdapter, plan_output: PlanOutput -) -> dict[SqlResourceRef, str] | None: - """Return every planned project relation for hard-coded name warnings, or None when off.""" - - if not plan_output.enforce_explicit_references: - return None - return { - ref: relation - for ref in _planned_relation_refs(plan_output=plan_output) - if (relation := _planned_relation(adapter=adapter, plan_output=plan_output, ref=ref)) - is not None - } - - -def _planned_relation_refs(*, plan_output: PlanOutput) -> frozenset[SqlResourceRef]: - refs: set[SqlResourceRef] = { - SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name) - for name in plan_output.model_locations - } - refs.update( - SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name) - for name in plan_output.seed_locations - ) - source_name: str - for source_name in plan_output.python_source_entries: - refs.add(SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name)) - return frozenset(refs) - - -def _resolve_relation( - *, - adapter: BaseAdapter, - project: CompiledProjectView, - plan_output: PlanOutput, - ref: SqlResourceRef, -) -> str: - planned: str | None = _planned_relation(adapter=adapter, plan_output=plan_output, ref=ref) - if planned is not None: - return planned - if ref.kind == SqlResourceRefKind.SOURCE: - raise SharedInputError( - f"Python node source '{ref.name}' is missing from the resolved plan source map" - ) - location: CompiledRelationLocation | None = next( - ( - resource.destination - for resource in ( - project.models if ref.kind == SqlResourceRefKind.MODEL else project.seeds - ) - if resource.name == ref.name - ), - None, - ) - if location is None: - raise SharedInputError(f"Python node references unknown {ref.kind.value} '{ref.name}'") - return resolve_relation_location_qualified_name(adapter=adapter, location=location) - - -def _planned_relation( - *, adapter: BaseAdapter, plan_output: PlanOutput, ref: SqlResourceRef -) -> str | None: - if ref.kind == SqlResourceRefKind.SOURCE: - planned_sources: dict[str, SourceEntry] = plan_output.python_source_entries - planned_source: SourceEntry | None = planned_sources.get(ref.name) - if planned_source is None: - return None - return render_source_relation(entry=planned_source, adapter=adapter) - locations: dict[str, CompiledRelationLocation] = ( - plan_output.model_locations - if ref.kind == SqlResourceRefKind.MODEL - else plan_output.seed_locations - ) - location: CompiledRelationLocation | None = locations.get(ref.name) - if location is None: - return None - return resolve_relation_location_qualified_name(adapter=adapter, location=location) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py deleted file mode 100644 index 1c3f4505a5..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Apply adapter default schema/database to compiled relation locations.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import replace -from typing import cast - -import sqlbuild._native as _native -from sqlbuild._native import ( - project_with, -) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledRelationLocation, - CompiledSeed, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSeedView, -) - - -def apply_target_defaults( - *, - project: CompiledProjectView, - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], - python_functions_inherit_default_namespace: bool, -) -> CompiledProjectView: - """Apply adapter default schema/database to compiled project targets.""" - - models: tuple[CompiledModelView, ...] = tuple( - _model_with_defaults( - model=model, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ) - for model in project.models - ) - seeds: tuple[CompiledSeedView, ...] = tuple( - _seed_with_defaults( - seed=seed, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ) - for seed in project.seeds - ) - functions: tuple[CompiledFunctionView, ...] = tuple( - cast(_native.NativeCompiledFunction, function).with_target_defaults( - (default_schema, default_database), - render_qualified_name, - python_functions_inherit_default_namespace, - ) - for function in project.functions - ) - return project_with( - project=project, - effective_target_database=project.effective_target_database or default_database, - effective_target_schema=project.effective_target_schema or default_schema, - models=models, - seeds=seeds, - functions=functions, - ) - - -def _model_with_defaults( - *, - model: CompiledModelView, - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], -) -> CompiledModelView: - if not isinstance(model, CompiledModel): - return cast(_native.NativeCompiledModel, model).with_target_defaults( - (default_schema, default_database), render_qualified_name - ) - return replace( - model, - destination=_resolve_target( - target=model.destination, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ), - ) - - -def _seed_with_defaults( - *, - seed: CompiledSeedView, - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], -) -> CompiledSeedView: - if isinstance(seed, _native.NativeCompiledSeed): - return seed.with_target_defaults((default_schema, default_database), render_qualified_name) - return replace( - cast(CompiledSeed, seed), - destination=_resolve_target( - target=seed.destination, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ), - ) - - -def _resolve_target( - *, - target: CompiledRelationLocation, - default_schema: str | None, - default_database: str | None, - render_qualified_name: Callable[..., str | None], -) -> CompiledRelationLocation: - """Fill in adapter defaults for None schema/database on a target.""" - - schema: str | None = target.schema if target.schema is not None else default_schema - database: str | None = target.database if target.database is not None else default_database - qualified_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=target.name, - ) - if ( - schema == target.schema - and database == target.database - and qualified_name == target.qualified_name - ): - return target - return CompiledRelationLocation( - database=database, - schema=schema, - name=target.name, - qualified_name=qualified_name, - logical_schema=target.logical_schema, - logical_database=target.logical_database, - ) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py b/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py deleted file mode 100644 index 10cb92d949..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Adapter-aware target namespace validation.""" - -from __future__ import annotations - -from typing import cast - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.constants import DEFAULTS_SECTION, PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import SourceEntry, TargetConfig - -_CONNECTION_IDENTITY_KEYS: frozenset[str] = frozenset( - {"account", "host", "port", "project", "server", "server_hostname", "workspace_url"} -) - - -def validate_managed_loader_target_isolation( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - resolved_connection: dict[str, object] | None = None, -) -> None: - """Reject managed loader write namespaces shared by multiple targets.""" - managed_sources: tuple[SourceEntry, ...] = _managed_sources(discovered_inputs) - if not managed_sources: - return - target_names: tuple[str, ...] = tuple( - sorted( - set(discovered_inputs.project_config.targets) - | set(discovered_inputs.local_config.targets) - ) - ) - namespaces_by_target: dict[str, frozenset[tuple[str | None, str | None]]] = {} - connections_by_target: dict[str, tuple[tuple[str, str], ...]] = {} - for target_name in target_names: - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - _, effective_vars, _ = build_effective_runtime_config( - discovered_inputs=discovered_inputs, - selected_target=target_name, - ) - effective_connection: dict[str, object] = ( - resolved_connection - if resolved_connection is not None - else build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=target_name, - ) - ) - connections_by_target[target_name] = _connection_identity(effective_connection) - namespaces_by_target[target_name] = frozenset( - _managed_source_namespace( - source_entry=source_entry, - target_config=target_config, - effective_connection=effective_connection, - effective_vars=effective_vars, - adapter=adapter, - ) - for source_entry in managed_sources - ) - for index, left_target in enumerate(target_names): - for right_target in target_names[index + 1 :]: - if connections_by_target[left_target] != connections_by_target[right_target]: - continue - shared_namespaces: frozenset[tuple[str | None, str | None]] = ( - namespaces_by_target[left_target] & namespaces_by_target[right_target] - ) - if not shared_namespaces: - continue - database, schema = sorted(shared_namespaces, key=repr)[0] - raise PlannerInputError( - "Managed loader write collision: targets " - f"'{left_target}' and '{right_target}' both resolve to " - f"database={database!r} schema={schema!r}. Set distinct " - f"targets.{left_target}.loader_schema and " - f"targets.{right_target}.loader_schema values.", - code="S102", - ) - - -def validate_named_target_schema_strategy( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None, -) -> None: - """Require shared-warehouse named targets to choose a schema strategy.""" - - if adapter.allows_implicit_managed_write_schema: - return - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - return - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - if isinstance(target_config.schema, str) and target_config.schema.strip(): - return - raise PlannerInputError( - f"Named target '{target_name}' must explicitly set schema to a nonblank literal or " - f"'{PRESERVE_TARGET_VALUE}'. Resource schemas, defaults, and connection schemas do not " - "select a named target schema strategy.", - code="S104", - ) - - -def _connection_identity(connection: dict[str, object]) -> tuple[tuple[str, str], ...]: - return tuple( - sorted( - (key, repr(value)) - for key, value in connection.items() - if key in _CONNECTION_IDENTITY_KEYS - ) - ) - - -def _managed_sources(discovered_inputs: DiscoveredProjectInputs) -> tuple[SourceEntry, ...]: - managed_sources: list[SourceEntry] = [] - for source_file in discovered_inputs.source_files: - for source_entry in source_file.source_entries: - if source_entry.managed: - managed_sources.append(source_entry) - return tuple(managed_sources) - - -def _managed_source_namespace( - *, - source_entry: SourceEntry, - target_config: TargetConfig, - effective_connection: dict[str, object], - effective_vars: dict[str, object], - adapter: BaseAdapter, -) -> tuple[str | None, str | None]: - database_value: object = ( - source_entry.database - or target_config.database - or effective_connection.get("database") - or adapter.default_database() - ) - schema_value: object = ( - source_entry.schema - or target_config.loader_schema - or target_config.schema - or effective_connection.get("schema") - or adapter.default_schema() - ) - expanded: object = expand_template_data( - value={"database": database_value, "schema": schema_value}, - variables=effective_vars, - context_values={}, - context_label="managed loader target namespace", - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) - values: dict[str, object] = cast(dict[str, object], expanded) - return _optional_string(values["database"]), _optional_string(values["schema"]) - - -def _optional_string(value: object) -> str | None: - return None if value is None else str(value) - - -def validate_project_targets(*, adapter_name: str, project: CompiledProjectView) -> None: - """Validate compiled model and seed locations for the effective adapter.""" - - if adapter_name not in { - BuiltinAdapter.SNOWFLAKE, - BuiltinAdapter.BIGQUERY, - BuiltinAdapter.DATABRICKS, - }: - return - _validate_required_target_parts( - adapter_name=adapter_name, - resource_kind="model", - targets={model.name: model.destination for model in project.models}, - ) - _validate_required_target_parts( - adapter_name=adapter_name, - resource_kind="seed", - targets={seed.name: seed.destination for seed in project.seeds}, - ) - - -def validate_managed_write_schemas(*, adapter: BaseAdapter, project: CompiledProjectView) -> None: - """Require an explicitly resolved physical schema for every managed warehouse write.""" - - if adapter.allows_implicit_managed_write_schema: - return - connection_schema: str | None = _optional_explicit_schema( - project.effective_connection.get("schema") - ) - for model in project.models: - _validate_managed_write_schema( - resource_label=f"Model '{model.name}'", - schema=model.destination.schema, - connection_schema=connection_schema, - ) - for seed in project.seeds: - _validate_managed_write_schema( - resource_label=f"Seed '{seed.name}'", - schema=seed.destination.schema, - connection_schema=connection_schema, - ) - for function in project.functions: - _validate_managed_write_schema( - resource_label=f"Function '{function.name}'", - schema=function.destination.schema, - connection_schema=connection_schema, - ) - for source in project.sources: - if not source.source_entry.managed: - continue - _validate_managed_write_schema( - resource_label=f"Managed source '{source.name}'", - schema=source.source_entry.schema, - connection_schema=connection_schema, - ) - - -def _validate_managed_write_schema( - *, resource_label: str, schema: str | None, connection_schema: str | None -) -> None: - if _optional_explicit_schema(schema) is not None or connection_schema is not None: - return - raise PlannerInputError( - f"{resource_label} has no explicitly resolved physical write schema. Set schema on the " - "resource or its defaults, selected target, loader_schema, or connection.", - code="S103", - ) - - -def _optional_explicit_schema(value: object) -> str | None: - if not isinstance(value, str): - return None - schema: str = value.strip() - if not schema or schema == PRESERVE_TARGET_VALUE: - return None - return schema - - -def _validate_required_target_parts( - *, - adapter_name: str, - resource_kind: str, - targets: dict[str, CompiledRelationLocation], -) -> None: - resource_name: str - target: CompiledRelationLocation - for resource_name, target in targets.items(): - missing_parts: list[str] = [] - if target.database is None: - missing_parts.append("database") - if target.schema is None: - missing_parts.append("schema") - if not missing_parts: - continue - missing_text: str = ", ".join(missing_parts) - raise PlannerInputError( - f"{adapter_name} execution requires explicit target {missing_text}. " - f"{resource_kind} '{resource_name}' resolved to " - f"database={target.database!r} schema={target.schema!r}. " - "Set them in the model config, the environment config, or the project defaults.", - code="S101", - help=setting_help( - purpose="to set project-wide defaults", - file_name=PROJECT_CONFIG_FILENAME, - section=DEFAULTS_SECTION, - key=missing_parts[0], - value=f"my_{missing_parts[0]}", - ), - ) diff --git a/src/sqlbuild/compiler/pipeline/constants.py b/src/sqlbuild/compiler/pipeline/constants.py deleted file mode 100644 index ca3b73ed98..0000000000 --- a/src/sqlbuild/compiler/pipeline/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Compiler pipeline decision constants.""" - -from __future__ import annotations - -DEFERRED_TARGET_CONTEXT_KEYS: frozenset[str] = frozenset({"schema", "database"}) -MODEL_PATH_ROOT: str = "models" -PATH_SELECTOR_PREFIX: str = "path:" -PATH_SEPARATORS: frozenset[str] = frozenset({"/", "\\"}) diff --git a/src/sqlbuild/compiler/pipeline/main/_plan_result.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py index 31a26dfcf8..3da6c17930 100644 --- a/src/sqlbuild/compiler/pipeline/main/_plan_result.py +++ b/src/sqlbuild/compiler/pipeline/main/_plan_result.py @@ -5,6 +5,7 @@ from collections.abc import Callable from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( @@ -13,17 +14,7 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( - build_deferred_locations, - gather_deferred_relations, - resolve_deferred_target_config, -) -from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl from sqlbuild.compiler.pipeline._helpers.materializations import load_custom_materializations -from sqlbuild.compiler.pipeline._helpers.python_plan_entries import ( - build_python_run_plan_outputs, - python_read_source_names_for, -) from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, @@ -60,12 +51,12 @@ def build_pipeline_result( deferred_locations: dict[str, CompiledRelationLocation] | None = None deferred_relations: dict[str, RelationInfo] | None = None if options.defer_to is not None: - deferred_target_config: TargetConfig = resolve_deferred_target_config( + deferred_target_config: TargetConfig = _native.resolve_deferred_target_config( discovered_inputs=discovered_inputs, defer_to=options.defer_to, current_target_name=project.effective_target_name, ) - deferred_locations = build_deferred_locations( + deferred_locations = _native.build_deferred_locations( project=project, deferred_target_config=deferred_target_config, effective_vars=project.effective_vars, @@ -73,7 +64,7 @@ def build_pipeline_result( default_database=adapter.default_database(), render_qualified_name=adapter.render_qualified_name, ) - deferred_relations = gather_deferred_relations( + deferred_relations = _native.gather_deferred_relations( adapter=adapter, connection=connection, deferred_locations=deferred_locations, @@ -89,12 +80,12 @@ def build_pipeline_result( run_selection = resolve_python_sql_run_selection_from_inputs( select=select, exclude=exclude, - project_graph=build_project_graph_impl(project), + project_graph=_native.build_project_graph_from_compiled_project(project=project), discovered_inputs=discovered_inputs, ) selected_sql_keys = run_selection.sql_keys selected_python_node_names = run_selection.python_node_names - python_read_source_names = python_read_source_names_for( + python_read_source_names = _native.python_read_source_names_for( discovered_inputs=discovered_inputs, python_node_names=selected_python_node_names ) @@ -134,7 +125,7 @@ def build_pipeline_result( custom_materializations: dict[str, Any] = load_custom_materializations( discovered_inputs.materialization_files ) - python_outputs: PythonRunPlanOutputs = build_python_run_plan_outputs( + python_outputs: PythonRunPlanOutputs = _native.build_python_run_plan_outputs( discovered_inputs=discovered_inputs, plan_output=plan_output, run_selection=run_selection, diff --git a/src/sqlbuild/compiler/pipeline/main/compile_only.py b/src/sqlbuild/compiler/pipeline/main/compile_only.py deleted file mode 100644 index ea0416fe8f..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/compile_only.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Public compile-only pipeline entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import compile_project_phase -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ( - CompiledProjectPhaseResult, - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.compiler.planner.models import PlanOutput - - -def run_compile_only_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions, - on_progress: Callable[[str], None] | None = None, -) -> CompilePipelineResult: - """Compile a project for consumers that construct their own focused plans.""" - - compiled: CompiledProjectPhaseResult = compile_project_phase( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=options, - on_progress=on_progress, - ) - return CompilePipelineResult( - project=compiled.project, - plan_output=PlanOutput(), - compile_seconds=compiled.compile_seconds, - planning_seconds=0.0, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/diff.py b/src/sqlbuild/compiler/pipeline/main/diff.py deleted file mode 100644 index dbdf62d710..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/diff.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Public diff pipeline entrypoints.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.diff import ( - compile_project_for_diff_environment, - resolve_diff_model_names, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def run_diff_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - from_target: str, - to_target: str, - resolved_connection: dict[str, object], - no_sql_validation: bool, - select: tuple[str, ...], - exclude: tuple[str, ...], - cli_vars: dict[str, object] | None = None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> tuple[CompiledProjectView, CompiledProjectView, tuple[str, ...]]: - """Compile both environments and resolve selected diffable model names.""" - - left_project: CompiledProjectView = compile_project_for_diff_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=from_target, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - right_project: CompiledProjectView = compile_project_for_diff_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=to_target, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - selected_names: tuple[str, ...] = resolve_diff_model_names( - project=right_project, - select=select, - exclude=exclude, - ) - return left_project, right_project, selected_names diff --git a/src/sqlbuild/compiler/pipeline/main/plan_work.py b/src/sqlbuild/compiler/pipeline/main/plan_work.py deleted file mode 100644 index 79d865eae6..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/plan_work.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Public entrypoint for planned work detection.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput - - -def plan_has_executable_work( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () -) -> bool: - """Return whether a planned build has runtime work to execute.""" - - return bool( - plan.model_entries - or plan.seed_entries - or plan.source_load_entries - or plan.function_entries - or plan.audit_entries - or plan.test_entries - or python_plan_entries - ) diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph.py b/src/sqlbuild/compiler/pipeline/main/project_graph.py deleted file mode 100644 index f90b53f4f4..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/project_graph.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Build a project graph from an already-compiled project.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -def build_project_graph_from_compiled_project(*, project: CompiledProjectView) -> ProjectGraph: - """Build a static dependency graph for an already-compiled project.""" - - return build_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py b/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py index 586feb3fa6..8a5017538f 100644 --- a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py +++ b/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py @@ -3,8 +3,8 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.pipeline._helpers.graph import select_project_graph_impl from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.planner.main.selection._graph_selection import resolve_graph_selection def select_project_graph( @@ -12,4 +12,4 @@ def select_project_graph( ) -> frozenset[CompiledObjectKey]: """Resolve `--select`/`--exclude` on the graph, adding the functions they need to build.""" - return select_project_graph_impl(graph=graph, select=select, exclude=exclude) + return resolve_graph_selection(graph=graph.native, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py b/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py deleted file mode 100644 index d517110bfe..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Planned project relations for hard-coded relation name guards.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.pipeline._helpers.relation_targets import ( - build_project_relation_targets_impl, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.python_nodes.models import SqlResourceRef - - -def build_project_relation_targets( - *, adapter: BaseAdapter, plan_output: PlanOutput -) -> dict[SqlResourceRef, str] | None: - """Return every planned project relation for hard-coded name warnings, or None when off.""" - - return build_project_relation_targets_impl(adapter=adapter, plan_output=plan_output) diff --git a/src/sqlbuild/compiler/pipeline/main/relation_targets.py b/src/sqlbuild/compiler/pipeline/main/relation_targets.py deleted file mode 100644 index d5d2b1a38f..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/relation_targets.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Runtime SQL relation maps for Python node contexts.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers.relation_targets import ( - build_python_relation_targets_impl, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.python_nodes.models import SqlResourceRef - - -def build_python_relation_targets( - *, - adapter: BaseAdapter, - project: CompiledProjectView, - plan_output: PlanOutput, - required_refs: frozenset[SqlResourceRef] | None = None, -) -> dict[SqlResourceRef, str]: - """Return adapter-qualified runtime relations required by selected Python nodes.""" - - return build_python_relation_targets_impl( - adapter=adapter, project=project, plan_output=plan_output, required_refs=required_refs - ) diff --git a/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py b/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py deleted file mode 100644 index 3f735d2777..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Resolve native SQLBuild model deferral inputs.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( - build_deferred_locations, - gather_deferred_relations, - resolve_deferred_target_config, -) -from sqlbuild.compiler.planner.models import DeferralInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def resolve_model_deferral_inputs( - *, - project: CompiledProjectView, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection: Any, - defer_to: str | None, -) -> DeferralInputs: - """Resolve native target locations and warehouse relations for model deferral.""" - - if defer_to is None: - return DeferralInputs() - deferred_target_config: TargetConfig = resolve_deferred_target_config( - discovered_inputs=discovered_inputs, - defer_to=defer_to, - current_target_name=project.effective_target_name, - ) - deferred_locations: dict[str, CompiledRelationLocation] = build_deferred_locations( - project=project, - deferred_target_config=deferred_target_config, - effective_vars=project.effective_vars, - default_schema=adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - ) - deferred_relations: dict[str, RelationInfo] = gather_deferred_relations( - adapter=adapter, - connection=connection, - deferred_locations=deferred_locations, - ) - return DeferralInputs( - deferred_locations=deferred_locations, - deferred_relations=deferred_relations, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/static_command.py b/src/sqlbuild/compiler/pipeline/main/static_command.py index df9c0e7689..b6bea2c6d4 100644 --- a/src/sqlbuild/compiler/pipeline/main/static_command.py +++ b/src/sqlbuild/compiler/pipeline/main/static_command.py @@ -4,15 +4,15 @@ from collections.abc import Callable -from sqlbuild._native import compile_project_phase +from sqlbuild._native import ( + build_deferred_locations, + compile_project_phase, + resolve_deferred_target_config, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.analysis_selection import apply_unit_test_selectors -from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( - build_deferred_locations, - resolve_deferred_target_config, -) from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, diff --git a/src/sqlbuild/compiler/pipeline/main/static_result.py b/src/sqlbuild/compiler/pipeline/main/static_result.py deleted file mode 100644 index bd396353bf..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/static_result.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public focused-command result assembly entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import CompilePipelineResult, StaticCommandContext -from sqlbuild.compiler.planner.models import PlanOutput - - -def build_static_pipeline_result( - *, context: StaticCommandContext, plan_output: PlanOutput -) -> CompilePipelineResult: - """Wrap a focused command projection in the established pipeline envelope.""" - - return CompilePipelineResult( - project=context.project, - plan_output=plan_output, - compile_seconds=context.compile_seconds, - planning_seconds=0.0, - ) diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index aa90f13b44..38e2d058ad 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -5,7 +5,7 @@ from functools import partial from pathlib import Path -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import build_compile_inputs, compile_frontier, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( @@ -14,7 +14,6 @@ from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.scopes._helpers.builder import build_tolerant_scope_index from sqlbuild.compiler.scopes._helpers.cache import ( diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index 3153120509..f392acd97c 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -8,14 +8,14 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + resolve_model_deferral_inputs, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.main.execution.command_warehouse import apply_command_warehouse from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.resolve_model_deferral_inputs import ( - resolve_model_deferral_inputs, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.display_plan import build_display_only_sqlbuild_plan from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py b/src/sqlbuild/integrations/dbt/main/pipeline/execute.py index 7ffc04f6f3..fd63739f65 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/execute.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.pipeline.main.plan_work import plan_has_executable_work +from sqlbuild._native import plan_has_executable_work from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt._helpers.pipeline.execute import ( build_failed_sqlbuild_model_names, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 678f20d622..f2774c5e89 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -13,6 +13,7 @@ import duckdb import pytest +import sqlbuild._native as native_module from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -27,7 +28,6 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.analysis_session.models import NativeQueryColumns -from sqlbuild.compiler.compile._helpers.assembly import source_bindings as source_bindings_module from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError @@ -636,17 +636,21 @@ def record_source_rebinding_analyses(monkeypatch: pytest.MonkeyPatch) -> list[st analysed: list[str] = [] original: Callable[..., tuple[NativeQueryColumns, ...]] = ( - source_bindings_module.infer_native_query_columns + native_module.infer_native_query_columns ) def recording(**kwargs: Any) -> tuple[NativeQueryColumns, ...]: - analysed.extend(query.sql for query in kwargs["queries"]) + analysed.extend(query.sql for query in filter(_reanalysis, kwargs["queries"])) return original(**kwargs) - monkeypatch.setattr(source_bindings_module, "infer_native_query_columns", recording) + monkeypatch.setattr(native_module, "infer_native_query_columns", recording) return analysed +def _reanalysis(query: Any) -> bool: + return query.mode == "reanalysis" + + def eager_output_column_scans(*, capture_dir: Path) -> tuple[int, int]: """Count the captured model files, and those discovery located output columns for eagerly.""" diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 10ecbee4ec..a62d963c88 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -44,7 +44,6 @@ from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, ) -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, @@ -663,15 +662,14 @@ def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> Compile def duplicate_analysed_model_names(*, monkeypatch: pytest.MonkeyPatch) -> None: """Name every analysed model after the first, an invariant discovery guarantees (D007).""" - original: Callable[[PythonModelAnalysis], list[tuple[object, ...]]] = ( - PythonModelAnalysis.model_rows - ) + original: Callable[..., Any] = native_module.start_model_analysis_session - def renamed(analysis: PythonModelAnalysis) -> list[tuple[object, ...]]: - rows: list[tuple[object, ...]] = original(analysis) - return [(rows[0][0], *row[1:]) for row in rows] + def renamed(catalog: object, request: tuple[object, ...], *options: Any) -> Any: + rows: list[tuple[object, ...]] = cast(list[tuple[object, ...]], request[-1]) + named: list[tuple[object, ...]] = [(rows[0][0], *row[1:]) for row in rows] + return original(catalog, (*request[:-1], named), *options) - monkeypatch.setattr(PythonModelAnalysis, "model_rows", renamed) + monkeypatch.setattr(native_module, "start_model_analysis_session", renamed) def custom_nullability_rule(arguments: tuple[InferredNullability, ...]) -> InferredNullability: diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 0895685c2b..57c19d9ea2 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -14,7 +14,13 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + build_compile_inputs, + compile_frontier, + discover_project_inputs, + mask_capture_noise, + render_stage_capture, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -26,12 +32,6 @@ ) from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture -from sqlbuild.compiler.frontier.constants import ( - STAGE_CAPTURE_INVOCATION_ID_MASK, - STAGE_CAPTURE_INVOCATION_ID_PATTERN, -) -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE @@ -255,9 +255,8 @@ def render_compile_inputs(*, project_dir: Path) -> CompileInputsView: def comparable_capture(inputs: CompileInputsView) -> str: """Render compile inputs as canonical JSON without the per-run identifier.""" - return STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub( - STAGE_CAPTURE_INVOCATION_ID_MASK, - _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)), + return mask_capture_noise( + _RUN_ID_PATTERN.sub('"run_id": ""', render_stage_capture(inputs)) ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 19f32584fb..e011ac3a2d 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -28,6 +28,7 @@ discover_project_inputs, discover_sql_function_files, discover_sql_hook_files, + render_stage_capture, ) from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( build_tolerant_scope_discovery, @@ -52,7 +53,6 @@ SqlHookEntry, TolerantScopeDiscovery, ) -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.sql_values.models import AuthoredSqlSet, AuthoredSqlValueCall from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 90d2179bbd..f3fd696ec9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,19 +8,19 @@ import pytest -import sqlbuild.compiler.compile._helpers.assembly.semantic_shapes as semantic_shapes_module +import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( build_compile_inputs, build_compiled_project, discover_project_inputs, run_compile_pipeline, + semantic_shapes, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledLineageColumnFact, @@ -218,7 +218,7 @@ def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ... """Record each shape request's expressions the binding catalog's shape cache misses.""" batches: list[tuple[str, ...]] = [] - for module in (native_stage_assembly, semantic_shapes_module): + for module in (native_stage_assembly, native_module): original: Callable[..., Any] = module.infer_native_expression_source_shapes monkeypatch.setattr( module, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index b8323417ee..259e866ba7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -5,14 +5,14 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + resolve_diff_model_names, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.diff import ( - compile_project_for_diff_environment, - resolve_diff_model_names, -) from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( DiffSelectorIntegrationTestCase, ) @@ -102,10 +102,10 @@ def test_given_compiled_project_when_resolving_diff_selectors_then_returns_expec ) -> None: write_repo_files(tmp_path, _PROJECT_FILES) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project_for_diff_environment( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=DuckDbAdapter(), - target_name="dev", + selected_target="dev", resolved_connection={"database": ":memory:"}, no_sql_validation=True, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index 9775f7f77e..90bc429c68 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -8,9 +8,9 @@ from sqlbuild._native import ( project_with, + semantic_shapes, ) from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.lineage.helpers import compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 14926d931c..58f78450d9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -10,8 +10,7 @@ from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.assembly import source_bindings -from sqlbuild.compiler.compile.classes import semantic_completion_inputs +from sqlbuild.compiler.compile._helpers.analysis import binding_requests from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView @@ -178,10 +177,7 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( def repeated_binding(**kwargs: Any) -> None: raise AssertionError("already validated SQL was normalized and bound again") - monkeypatch.setattr(source_bindings, "get_complete_schema_binding_request", repeated_binding) - monkeypatch.setattr( - semantic_completion_inputs, "get_complete_schema_binding_request", repeated_binding - ) + monkeypatch.setattr(binding_requests, "get_complete_schema_binding_request", repeated_binding) assert ( main( [ diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 0d2a8caf65..aa8be654ca 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -6,9 +6,9 @@ from itertools import filterfalse from typing import cast +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py index 81d0289c37..ba728bf872 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -15,8 +15,8 @@ required_discovery_kinds, ) from scripts.compiler_differential.constants import DISCOVERY_NON_COLLECTION_FIELDS +from sqlbuild._native import render_stage_capture from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( AuthoredByteKindsTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py index 4aebac3039..daca70f1c0 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -15,9 +15,9 @@ required_render_kinds, ) from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS +from sqlbuild._native import render_stage_capture from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py index 336df812e4..3106cbbf5e 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -27,24 +27,12 @@ class StageCaptureOrderTestCase: expected_identical: bool -@dataclass(frozen=True) -class UnorderedAttributeTestCase: - """Two fill orders for a memo map and an ordered map, and whether captures match.""" - - description: str - first_memo_keys: tuple[str, ...] - second_memo_keys: tuple[str, ...] - first_ordered_keys: tuple[str, ...] - second_ordered_keys: tuple[str, ...] - expected_identical: bool - - @dataclass(frozen=True) class OmittedFieldTestCase: - """Dataclass fields left out of a capture, and the capture that remains.""" + """A dataclass value, and its capture without the fields declared omitted.""" description: str - omitted: frozenset[str] + value: Callable[[], object] expected_capture: object diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index e46543557f..730b82e719 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -5,8 +5,9 @@ from dataclasses import dataclass from enum import StrEnum from pathlib import PurePosixPath +from typing import Any, cast -from sqlbuild.compiler.compile.models import CompactLineageFacts +from sqlbuild.compiler.compile.models import CompactLineageFacts, DeclarationScopeBuild from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" @@ -119,3 +120,14 @@ def order_line() -> OrderLine: """Return one neutral order line.""" return OrderLine(order_id=7, status=OrderStatus.PLACED, path=PurePosixPath("models/a.sql")) + + +def declaration_scope_build() -> DeclarationScopeBuild: + """Return a declaration scope build whose process-local scan cache is not captured.""" + + return DeclarationScopeBuild( + loaded_macros={}, + index=cast(Any, "orders_index"), + resolver=cast(Any, "orders_resolver"), + sql_test_scans=cast(Any, object()), + ) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 63846c16d3..9c2ca715cf 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -9,15 +9,11 @@ import pytest from scripts.compiler_differential.classes.capture_file import expand_capture_text -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture -from sqlbuild.compiler.frontier.classes.stage_capture_encoder import StageCaptureEncoder +from sqlbuild._native import compile_frontier, render_stage_capture from sqlbuild.compiler.frontier.constants import ( STAGE_CAPTURE_DIR_ENV_VAR, - STAGE_CAPTURE_OMITTED_ATTRIBUTES, STAGE_CAPTURE_SHARED_NODES_KEY, - STAGE_CAPTURE_UNORDERED_ATTRIBUTES, ) -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, @@ -25,10 +21,10 @@ SharedCaptureTestCase, StageCaptureOrderTestCase, StageCaptureTestCase, - UnorderedAttributeTestCase, ) from tests.unit.src.sqlbuild.compiler.frontier.helpers import ( HELPERS_MODULE, + declaration_scope_build, linked_customer, memo_catalog, order_line, @@ -192,6 +188,12 @@ def test_given_frontier_value_when_rendering_capture_then_json_is_canonical( ), expected_identical=True, ), + StageCaptureOrderTestCase( + description="undeclared_attribute_map_order_stays_visible", + first=lambda: memo_catalog(memo_keys=("orders", "customers"), ordered_keys=()), + second=lambda: memo_catalog(memo_keys=("customers", "orders"), ordered_keys=()), + expected_identical=False, + ), StageCaptureOrderTestCase( description="binding_catalog_relations_stay_visible", first=lambda: orders_binding_catalog( @@ -211,61 +213,22 @@ def test_given_reordered_value_when_rendering_capture_then_only_unordered_collec ) is test_case.expected_identical -@pytest.mark.parametrize( - "test_case", - [ - UnorderedAttributeTestCase( - description="memo_fill_order_is_not_visible", - first_memo_keys=("orders", "customers"), - second_memo_keys=("customers", "orders"), - first_ordered_keys=("orders", "customers"), - second_ordered_keys=("orders", "customers"), - expected_identical=True, - ), - UnorderedAttributeTestCase( - description="other_attribute_order_stays_visible", - first_memo_keys=("orders", "customers"), - second_memo_keys=("orders", "customers"), - first_ordered_keys=("orders", "customers"), - second_ordered_keys=("customers", "orders"), - expected_identical=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_declared_memo_attribute_when_rendering_capture_then_only_it_is_sorted( - test_case: UnorderedAttributeTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setitem( - STAGE_CAPTURE_UNORDERED_ATTRIBUTES, f"{HELPERS_MODULE}:MemoCatalog", frozenset({"memo"}) - ) - first: str = render_stage_capture( - memo_catalog(memo_keys=test_case.first_memo_keys, ordered_keys=test_case.first_ordered_keys) - ) - second: str = render_stage_capture( - memo_catalog( - memo_keys=test_case.second_memo_keys, ordered_keys=test_case.second_ordered_keys - ) - ) - - assert (first == second) is test_case.expected_identical - - @pytest.mark.parametrize( "test_case", [ OmittedFieldTestCase( description="declared_process_handle_field_is_left_out", - omitted=frozenset({"status"}), + value=declaration_scope_build, expected_capture={ - "__type__": f"{HELPERS_MODULE}:OrderLine", - "order_id": 7, - "path": {"__path__": "models/a.sql"}, + "__type__": f"{_COMPILE_MODELS}:DeclarationScopeBuild", + "loaded_macros": {}, + "index": "orders_index", + "resolver": "orders_resolver", }, ), OmittedFieldTestCase( description="undeclared_dataclass_keeps_every_field", - omitted=frozenset(), + value=order_line, expected_capture={ "__type__": f"{HELPERS_MODULE}:OrderLine", "order_id": 7, @@ -277,13 +240,9 @@ def test_given_declared_memo_attribute_when_rendering_capture_then_only_it_is_so ids=lambda case: case.description, ) def test_given_declared_omitted_field_when_rendering_dataclass_capture_then_it_is_left_out( - test_case: OmittedFieldTestCase, monkeypatch: pytest.MonkeyPatch + test_case: OmittedFieldTestCase, ) -> None: - monkeypatch.setitem( - STAGE_CAPTURE_OMITTED_ATTRIBUTES, f"{HELPERS_MODULE}:OrderLine", test_case.omitted - ) - - capture: object = json.loads(render_stage_capture(order_line())) + capture: object = json.loads(render_stage_capture(test_case.value())) assert capture == test_case.expected_capture @@ -306,7 +265,7 @@ def test_given_repeated_large_subtrees_when_rendering_capture_then_each_is_store shared: dict[str, object] = json.loads(text)[STAGE_CAPTURE_SHARED_NODES_KEY] assert len(shared) == test_case.expected_shared_nodes - assert expand_capture_text(text) == StageCaptureEncoder().encode(test_case.value()) + assert expand_capture_text(text) == test_case.value() assert { hashlib.sha256( json.dumps(node, ensure_ascii=False, separators=(",", ":")).encode() diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py index 0ac53a7378..b9d4945b5f 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_environment.py @@ -4,12 +4,12 @@ import pytest -from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( +from sqlbuild._native import ( digest_module_files, module_digests_metadata, - module_sources, unchanged_module_digests, ) +from sqlbuild.compiler.macro_bridge._helpers.store_environment import module_sources from sqlbuild.compiler.macro_bridge.models import ModuleSources from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( ModuleDigestsRoundTripTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py index 42cca1fbde..12f5c389b3 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( +from sqlbuild._native import ( context_store_token, macro_store_token, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index 14ffe833c9..4620e2c515 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -4,6 +4,12 @@ import pytest +from sqlbuild._native import ( + validate_managed_loader_target_isolation, + validate_managed_write_schemas, + validate_named_target_schema_strategy, + validate_project_targets, +) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter @@ -17,12 +23,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.pipeline._helpers.target_validation import ( - validate_managed_loader_target_isolation, - validate_managed_write_schemas, - validate_named_target_schema_strategy, - validate_project_targets, -) from sqlbuild.spec.contracts.models import ( LocalConfig, LocalTargetConfig, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index 6fe6d27209..2bead4698d 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -4,13 +4,13 @@ import pytest +from sqlbuild._native import build_deferred_locations from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers.deferred_locations import build_deferred_locations from sqlbuild.spec.contracts.models import TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( DeferredTargetTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index f012c38d8c..25de77272c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import build_python_plan_entries from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.pipeline._helpers.python_plan_entries import build_python_plan_entries from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( build_python_sql_run_lifecycle_plan, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 68bdc913c3..a3594a3e41 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -65,9 +65,6 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di adapter: Mock = Mock() monkeypatch.setattr(time, "monotonic", monotonic) monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr( - analysis_selection, "resolve_compile_analysis_selection", Mock(return_value=None) - ) monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py index 93aaa93bb9..eca1fbe918 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.pipeline.main.plan_work import plan_has_executable_work +from sqlbuild._native import plan_has_executable_work from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import ( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index d4e855c383..5779d71d6c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import build_python_relation_targets from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind