diff --git a/AGENTS.md b/AGENTS.md index 6d5c006c03..7776917bb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -215,8 +215,9 @@ - An incremental rebuild after a focused change takes minutes. A build that is unexpectedly cold or much slower than that is a signal to stop and investigate, not to wait or start over. - Get fast feedback first: `cargo check -p ` while editing, `cargo test -p ` for - the crate you touched, then one maturin build and the focused Python tests. Run clippy with the - CI toolchain for the touched crates only. + the crate you touched, then one maturin build and the focused Python tests. CI lints with the + latest stable toolchain, so keep the local `stable` toolchain current (`rustup update stable`); + `make check-ci` includes `make rust-lint` (workspace clippy with `-D warnings`) before a push. - Fix compiler errors by reading them and making the smallest correct change. Never delete and rewrite a working module, or restart a port from scratch, because a build failed. - Do not run two native builds against the same target directory at once, and do not run diff --git a/Makefile b/Makefile index 1b9ccf6c88..52be76b8d4 100644 --- a/Makefile +++ b/Makefile @@ -33,12 +33,15 @@ type: dev-native: uv run --no-sync maturin develop --profile dev --skip-install -rust-check: - cargo fmt --all --check - cargo clippy --workspace --all-targets --all-features -- -D warnings +rust-check: rust-lint uv run fensu check cargo test --workspace --all-features +.PHONY: rust-lint +rust-lint: + cargo fmt --all --check + cargo clippy --workspace --all-targets --all-features -- -D warnings + test: @mkdir -p /tmp/opencode @@ -604,7 +607,7 @@ verify-quick: exit $$status -check-ci: check-e2e-shards check-native-layering check-python-boundary +check-ci: check-e2e-shards check-native-layering check-python-boundary rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index 6fb1e98127..d4e0b4a64d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -5,6 +5,7 @@ pub mod finished_fact_models; pub mod finished_model_facts; pub mod model_reference_facts; pub mod names_set_operation; +pub mod native_dialect; pub mod projection_facts; pub mod prove_dynamic_contracts; pub mod prove_finished_dynamic_contracts; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs new file mode 100644 index 0000000000..29f8ac170e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs @@ -0,0 +1,8 @@ +//! The polyglot dialect name a project's SQL analysis dialect selects. + +use crate::assembly::analysis_session::_helpers::mappings; + +/// Python's `NATIVE_DIALECT_ALIASES.get(dialect or "", dialect)`. +pub fn native_dialect(dialect: Option<&str>) -> Option<&str> { + dialect.map(mappings::native_dialect) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index 0cc551662e..d3db69bdad 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod mappings; #[path = "tests/test_model_columns.rs"] mod model_columns; +#[path = "tests/test_native_dialect.rs"] +mod native_dialect; #[path = "tests/test_reference_facts.rs"] mod reference_facts; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs new file mode 100644 index 0000000000..aae3097136 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs @@ -0,0 +1,38 @@ +use crate::assembly::analysis_session::main::native_dialect::native_dialect; +use crate::assembly::analysis_session::tests::test_types::NativeDialectTestCase; + +#[test] +fn given_analysis_dialect_when_resolving_native_dialect_then_matches_python_aliases() { + let test_cases = [ + NativeDialectTestCase { + description: "an aliased dialect maps to its polyglot name", + dialect: Some("postgres"), + expected_dialect: Some("postgresql"), + }, + NativeDialectTestCase { + description: "an unaliased dialect is kept", + dialect: Some("snowflake"), + expected_dialect: Some("snowflake"), + }, + NativeDialectTestCase { + description: "an empty dialect is kept", + dialect: Some(""), + expected_dialect: Some(""), + }, + NativeDialectTestCase { + description: "no dialect stays absent", + dialect: None, + expected_dialect: None, + }, + ]; + + for test_case in test_cases { + let dialect = native_dialect(test_case.dialect); + + assert_eq!( + dialect, test_case.expected_dialect, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 8a81d47677..a25f7ee25e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -177,3 +177,9 @@ pub(crate) struct ReferenceFactsTestCase { pub(crate) expected_lineage: &'static [(&'static str, &'static str, &'static str)], pub(crate) expected_required: &'static [&'static str], } + +pub(crate) struct NativeDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) expected_dialect: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs b/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs new file mode 100644 index 0000000000..ef7ad6a4b2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs @@ -0,0 +1,9 @@ +//! `_str_or_none`: a configured database or schema name, stripped, when it names one. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Python's `value.strip() if value.strip() else None` for a string `value`. +pub fn configured_namespace(value: &str) -> Option { + let stripped: &str = python_strip(value); + (!stripped.is_empty()).then(|| stripped.to_owned()) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index 9cbfd9734e..9ac75e428e 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -1,5 +1,6 @@ pub mod assemble_project_resources; pub mod check_sql_syntax; +pub mod configured_namespace; pub mod environment_template_names; pub mod preserved_logical_namespace; pub mod sql_syntax_error; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs index 48e3358156..3ac1cfe81c 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_configured_namespace.rs"] +mod configured_namespace; #[path = "tests/test_environment_names.rs"] mod environment_names; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs new file mode 100644 index 0000000000..b417746180 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs @@ -0,0 +1,39 @@ +use crate::assembly::project::main::configured_namespace::configured_namespace; +use crate::assembly::project::tests::test_types::ConfiguredNamespaceTestCase; + +#[test] +fn given_configured_value_when_reading_namespace_then_matches_python_strip() { + let test_cases = [ + ConfiguredNamespaceTestCase { + description: "surrounding whitespace is stripped", + value: " \tanalytics\n", + expected_namespace: Some("analytics"), + }, + ConfiguredNamespaceTestCase { + description: "a blank value names no namespace", + value: " \u{3000}\u{1c} ", + expected_namespace: None, + }, + ConfiguredNamespaceTestCase { + description: "an empty value names no namespace", + value: "", + expected_namespace: None, + }, + ConfiguredNamespaceTestCase { + description: "inner whitespace is kept", + value: "\u{a0}sales data\u{2028}", + expected_namespace: Some("sales data"), + }, + ]; + + for test_case in test_cases { + let namespace = configured_namespace(test_case.value); + + assert_eq!( + namespace.as_deref(), + test_case.expected_namespace, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index 88b3291f57..5d6acb6b71 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -86,3 +86,9 @@ pub(crate) struct EnvironmentNamesTestCase { pub(crate) values: &'static [Option<&'static str>], pub(crate) expected_names: &'static [&'static str], } + +pub(crate) struct ConfiguredNamespaceTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_namespace: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs new file mode 100644 index 0000000000..5f2e72c092 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs @@ -0,0 +1,279 @@ +//! `DagCheck` values of SQL tests, audits, scenarios and Python checks, sorted. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::edges::python_kinds; +use crate::dag_artifact::_helpers::nodes::{node_id, python_node_id}; +use crate::dag_artifact::_helpers::values::{integer, object, optional_text, text, texts}; +use crate::dag_artifact::constants::{ + AUDIT_KIND, CHECK_KIND, MODEL_KIND, PYTHON_CHECK_KIND, SCENARIO_KIND, SQL_TEST_KIND, +}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{ + DagAuditInput, DagProjectInput, DagPythonGraphInput, DagPythonNodeInput, DagScenarioInput, + DagSqlTestInput, +}; +use crate::dag_artifact::types::DagValue; + +/// Every `DagCheck` field, `None` or empty where a check kind leaves it out. +#[derive(Default)] +struct CheckFields { + id: String, + kind: String, + name: String, + checked_asset_ids: Vec, + path: Option, + description: Option, + severity: Option, + tags: Vec, + group: Option, + meta: Option, + attachment_kind: Option, + attached_target_name: Option, + attached_column_name: Option, + mode: Option, + assertion_names: Vec, + expected_model_names: Vec, + graph_asset_ids: Vec, + fixture_refs: Vec, +} + +fn strings(values: &[String]) -> DagValue { + texts(values.iter().map(String::as_str)) +} + +impl CheckFields { + fn value(self) -> DagValue { + object(vec![ + ("id", text(&self.id)), + ("kind", text(&self.kind)), + ("name", text(&self.name)), + ("checked_asset_ids", strings(&self.checked_asset_ids)), + ("path", optional_text(self.path.as_deref())), + ("description", optional_text(self.description.as_deref())), + ("severity", optional_text(self.severity.as_deref())), + ("tags", strings(&self.tags)), + ("group", optional_text(self.group.as_deref())), + ("meta", self.meta.unwrap_or(DagValue::Object(Vec::new()))), + ( + "attachment_kind", + optional_text(self.attachment_kind.as_deref()), + ), + ( + "attached_target_name", + optional_text(self.attached_target_name.as_deref()), + ), + ( + "attached_column_name", + optional_text(self.attached_column_name.as_deref()), + ), + ("mode", optional_text(self.mode.as_deref())), + ("assertion_names", strings(&self.assertion_names)), + ("expected_model_names", strings(&self.expected_model_names)), + ("graph_asset_ids", strings(&self.graph_asset_ids)), + ("fixture_refs", strings(&self.fixture_refs)), + ]) + } +} + +fn optional_integer(value: Option) -> DagValue { + value.map_or(DagValue::Null, integer) +} + +fn sql_test_meta(test: &DagSqlTestInput) -> Result { + let Some(case) = &test.case else { + return Ok(DagValue::Object(Vec::new())); + }; + let types: HashMap<&str, &str> = case + .parameter_schema + .iter() + .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) + .collect(); + let parameters: Vec = case + .parameters + .iter() + .map(|(name, value)| { + let value_type: &str = types + .get(name.as_str()) + .ok_or_else(|| DagArtifactError::MissingParameterType(name.clone()))?; + Ok(object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("value", value.clone()), + ])) + }) + .collect::>()?; + Ok(object(vec![ + ("source_path", text(&test.source_path)), + ("block_index", optional_integer(case.block_index)), + ("parent_name", optional_text(case.parent_name.as_deref())), + ("case_name", text(&case.case_name)), + ("case_index", optional_integer(case.case_index)), + ( + "case_fingerprint", + optional_text(case.case_fingerprint.as_deref()), + ), + ( + "parameter_schema", + DagValue::Array( + case.parameter_schema + .iter() + .map(|(name, value_type, nullable)| { + object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("nullable", DagValue::Bool(*nullable)), + ]) + }) + .collect(), + ), + ), + ("parameters", DagValue::Array(parameters)), + ])) +} + +fn sql_test_check(test: &DagSqlTestInput) -> Result<(String, String, DagValue), DagArtifactError> { + let id: String = match &test.case { + None => node_id(&test.key), + Some(case) => { + let block_index: String = case + .block_index + .map_or_else(|| String::from("None"), |index| index.to_string()); + format!( + "{SQL_TEST_KIND}:{}:{block_index}:{}", + test.source_path, case.case_name + ) + } + }; + let value: DagValue = CheckFields { + id, + kind: SQL_TEST_KIND.to_owned(), + name: test.name.clone(), + checked_asset_ids: test.scope_deps.iter().map(node_id).collect(), + path: Some(test.path.clone()), + mode: Some(test.mode.clone()), + meta: Some(sql_test_meta(test)?), + ..CheckFields::default() + } + .value(); + Ok((SQL_TEST_KIND.to_owned(), test.name.clone(), value)) +} + +fn audit_check(audit: &DagAuditInput) -> (String, String, DagValue) { + let checked_asset_ids: Vec = + match (&audit.attached_target_kind, &audit.attached_target_name) { + (Some(kind), Some(name)) => vec![node_id(&(kind.clone(), name.clone()))], + _ => audit.scope_deps.iter().map(node_id).collect(), + }; + let id: String = [ + Some(node_id(&audit.key)), + audit.attached_target_kind.clone(), + audit.attached_target_name.clone(), + audit.attached_column_name.clone(), + ] + .into_iter() + .flatten() + .filter(|part| !part.is_empty()) + .collect::>() + .join(":"); + let value: DagValue = CheckFields { + id, + kind: AUDIT_KIND.to_owned(), + name: audit.name.clone(), + checked_asset_ids, + path: Some(audit.path.clone()), + severity: audit.severity.clone(), + attachment_kind: audit.attached_target_kind.clone(), + attached_target_name: audit.attached_target_name.clone(), + attached_column_name: audit.attached_column_name.clone(), + ..CheckFields::default() + } + .value(); + (AUDIT_KIND.to_owned(), audit.name.clone(), value) +} + +fn model_id(name: &str) -> String { + node_id(&(MODEL_KIND.to_owned(), name.to_owned())) +} + +fn scenario_check(scenario: &DagScenarioInput) -> (String, String, DagValue) { + let value: DagValue = CheckFields { + id: node_id(&scenario.key), + kind: SCENARIO_KIND.to_owned(), + name: scenario.name.clone(), + checked_asset_ids: scenario + .expected_model_names + .iter() + .map(|name| model_id(name)) + .collect(), + path: Some(scenario.path.clone()), + assertion_names: scenario.assertion_names.clone(), + expected_model_names: scenario.expected_model_names.clone(), + graph_asset_ids: scenario + .authored_cte_names + .iter() + .map(|name| model_id(name)) + .collect(), + fixture_refs: scenario.fixture_names.concat(), + ..CheckFields::default() + } + .value(); + (SCENARIO_KIND.to_owned(), scenario.name.clone(), value) +} + +fn python_check( + node: &DagPythonNodeInput, + python: &DagPythonGraphInput, + kinds: &HashMap<&str, &str>, +) -> (String, String, DagValue) { + let mut checked_asset_ids: Vec = python + .dependency_edges + .iter() + .filter(|(_, downstream)| *downstream == node.name) + .map(|(upstream, _)| python_node_id(kinds[upstream.as_str()], upstream)) + .collect(); + checked_asset_ids.extend(node.sql_deps.iter().map(node_id)); + let value: DagValue = CheckFields { + id: python_node_id(&node.kind, &node.name), + kind: PYTHON_CHECK_KIND.to_owned(), + name: node.name.clone(), + checked_asset_ids, + path: Some(node.path.clone()), + description: node.description.clone(), + severity: node.check_severity.clone(), + tags: node.tags.clone(), + group: node.group.clone(), + meta: Some(node.meta.clone()), + ..CheckFields::default() + } + .value(); + (PYTHON_CHECK_KIND.to_owned(), node.name.clone(), value) +} + +/// Every check, sorted by `(kind, name)`. +pub(crate) fn checks( + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> Result { + let mut checks: Vec<(String, String, DagValue)> = project + .sql_tests + .iter() + .map(sql_test_check) + .collect::>()?; + checks.extend(project.audits.iter().map(audit_check)); + checks.extend(project.scenarios.iter().map(scenario_check)); + if let Some(python) = python { + let kinds: HashMap<&str, &str> = python_kinds(python); + checks.extend( + python + .nodes + .iter() + .filter(|node| node.kind == CHECK_KIND) + .map(|node| python_check(node, python, &kinds)), + ); + } + checks.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + Ok(DagValue::Array( + checks.into_iter().map(|(_, _, value)| value).collect(), + )) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs new file mode 100644 index 0000000000..092e82a5c7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs @@ -0,0 +1,73 @@ +//! `DagEdge` values: lineage, loader dependency and Python node edges, sorted. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::nodes::{ + loader_or_source_id, node_id, python_node_id, sources_by_loader, +}; +use crate::dag_artifact::_helpers::values::{object, text}; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use crate::dag_artifact::types::DagValue; +use crate::graph::models::ProjectGraph; + +/// Each Python node name's kind, as `nodes_by_name` maps it. +pub(crate) fn python_kinds(python: &DagPythonGraphInput) -> HashMap<&str, &str> { + python + .kinds_by_name + .iter() + .map(|(name, kind)| (name.as_str(), kind.as_str())) + .collect() +} + +fn python_edges(python: &DagPythonGraphInput) -> Vec<(String, String)> { + let kinds: HashMap<&str, &str> = python_kinds(python); + let mut edges: Vec<(String, String)> = python + .dependency_edges + .iter() + .map(|(upstream, downstream)| { + ( + python_node_id(kinds[upstream.as_str()], upstream), + python_node_id(kinds[downstream.as_str()], downstream), + ) + }) + .collect(); + for node in &python.nodes { + for dependency in &node.sql_deps { + edges.push((node_id(dependency), python_node_id(&node.kind, &node.name))); + } + } + edges +} + +/// Every edge, sorted by `(from_id, to_id)`. +pub(crate) fn edges( + graph: &ProjectGraph, + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> DagValue { + let mut edges: Vec<(String, String)> = Vec::new(); + for (key, deps) in &graph.upstream { + for dep in deps { + edges.push((node_id(dep), node_id(key))); + } + } + let sources: HashMap<&str, &str> = sources_by_loader(project); + for loader in &project.loaders { + for dependency in &loader.dependency_names { + edges.push(( + loader_or_source_id(dependency, &sources), + loader_or_source_id(&loader.name, &sources), + )); + } + } + if let Some(python) = python { + edges.extend(python_edges(python)); + } + edges.sort(); + DagValue::Array( + edges + .iter() + .map(|(from, to)| object(vec![("from_id", text(from)), ("to_id", text(to))])) + .collect(), + ) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs new file mode 100644 index 0000000000..7dec7aa968 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod checks; +pub(crate) mod edges; +pub(crate) mod nodes; +pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs new file mode 100644 index 0000000000..fbbd296122 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs @@ -0,0 +1,436 @@ +//! `DagNode` values of sources, loaders, seeds, functions, models and Python nodes. + +use sqlbuild_core::text::main::python_strip::python_strip; +use std::collections::HashMap; + +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::_helpers::values::{ + column, object, optional_text, text, texts, typed_column, +}; +use crate::dag_artifact::constants::{ + ASSET_KIND, DEFAULT_MATERIALIZATION_PYTHON_ASSET, LOADER_DESTINATION_PREFIX, LOADER_KIND, + LOADER_MAX_PARTS, LOADER_NODE_META_KEY, MODEL_KIND, SCALAR_RETURN_KIND, SEED_KIND, SOURCE_KIND, + TABLE_FUNCTION_KIND, TABLE_RETURN_KIND, UDF_KIND, +}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{ + DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, DagProjectInput, + DagPythonNodeInput, DagRelationInput, DagSeedInput, DagSourceEntryInput, DagSourceInput, +}; +use crate::dag_artifact::types::DagColumnLineage; +use crate::dag_artifact::types::DagValue; +use crate::python_values::main::text_key::text_key; + +/// Every `DagNode` field, `None` or empty where a node kind leaves it out. +#[derive(Default)] +pub(crate) struct NodeFields { + pub(crate) id: String, + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) asset_key: Vec, + pub(crate) target: Option, + pub(crate) path: Option, + pub(crate) description: Option, + pub(crate) sql: Option, + pub(crate) tags: Vec, + pub(crate) group: Option, + pub(crate) meta: Option, + pub(crate) columns: Vec, + pub(crate) column_lineage: DagColumnLineage, + pub(crate) materialization_type: Option, + pub(crate) language: Option, + pub(crate) return_kind: Option, + pub(crate) returns: Option, + pub(crate) loader: Option, + pub(crate) arguments: Vec<(String, String)>, +} + +impl NodeFields { + /// The node as `dataclasses.asdict` writes it. + pub(crate) fn value(self) -> DagValue { + object(vec![ + ("id", text(&self.id)), + ("kind", text(&self.kind)), + ("name", text(&self.name)), + ( + "asset_key", + texts(self.asset_key.iter().map(String::as_str)), + ), + ("target", self.target.unwrap_or(DagValue::Null)), + ("path", optional_text(self.path.as_deref())), + ("description", optional_text(self.description.as_deref())), + ("sql", optional_text(self.sql.as_deref())), + ("tags", texts(self.tags.iter().map(String::as_str))), + ("group", optional_text(self.group.as_deref())), + ("meta", self.meta.unwrap_or(DagValue::Object(Vec::new()))), + ("columns", DagValue::Array(self.columns)), + ( + "column_lineage", + DagValue::Object( + self.column_lineage + .iter() + .map(|(name, refs)| (text_key(name), lineage_refs(refs))) + .collect(), + ), + ), + ( + "materialization_type", + optional_text(self.materialization_type.as_deref()), + ), + ("language", optional_text(self.language.as_deref())), + ("return_kind", optional_text(self.return_kind.as_deref())), + ("returns", optional_text(self.returns.as_deref())), + ("loader", optional_text(self.loader.as_deref())), + ( + "arguments", + DagValue::Array( + self.arguments + .iter() + .map(|(name, data_type)| { + object(vec![("name", text(name)), ("type", text(data_type))]) + }) + .collect(), + ), + ), + ]) + } +} + +fn lineage_refs(refs: &[(String, String)]) -> DagValue { + DagValue::Array( + refs.iter() + .map(|(node, column)| object(vec![("node", text(node)), ("column", text(column))])) + .collect(), + ) +} + +/// `{resource type}:{name}`. +pub(crate) fn node_id((kind, name): &ObjectKey) -> String { + format!("{kind}:{name}") +} + +/// `loader:{name}`. +pub(crate) fn loader_node_id(name: &str) -> String { + format!("{LOADER_KIND}:{name}") +} + +/// A Python node's id: loaders share the loader namespace. +pub(crate) fn python_node_id(kind: &str, name: &str) -> String { + if kind == LOADER_KIND { + return loader_node_id(name); + } + format!("{kind}:{name}") +} + +fn present(parts: &[Option<&str>]) -> Vec { + parts + .iter() + .flatten() + .filter(|part| !part.is_empty()) + .map(|part| (*part).to_owned()) + .collect() +} + +fn entry_target(entry: (Option<&str>, Option<&str>, &str)) -> DagValue { + let (database, schema, name) = entry; + object(vec![ + ("database", optional_text(database)), + ("schema", optional_text(schema)), + ("name", text(name)), + ( + "qualified_name", + text(&present(&[database, schema, Some(name)]).join(".")), + ), + ("logical_database", DagValue::Null), + ("logical_schema", DagValue::Null), + ]) +} + +fn relation_target(target: &DagRelationInput) -> DagValue { + object(vec![ + ("database", optional_text(target.database.as_deref())), + ("schema", optional_text(target.schema.as_deref())), + ("name", optional_text(target.name.as_deref())), + ( + "qualified_name", + optional_text(target.qualified_name.as_deref()), + ), + ( + "logical_database", + optional_text(target.logical_database.as_deref()), + ), + ( + "logical_schema", + optional_text(target.logical_schema.as_deref()), + ), + ]) +} + +fn relation_asset_key(target: &DagRelationInput) -> Vec { + let database: Option<&str> = target + .logical_database + .as_deref() + .filter(|part| !part.is_empty()) + .or(target.database.as_deref()); + let schema: Option<&str> = target + .logical_schema + .as_deref() + .filter(|part| !part.is_empty()) + .or(target.schema.as_deref()); + present(&[database, schema, target.name.as_deref()]) +} + +fn entry_table(entry: &DagSourceEntryInput) -> &str { + entry + .table + .as_deref() + .filter(|table| !table.is_empty()) + .unwrap_or(&entry.name) +} + +fn columns(columns: &[DagColumnInput]) -> Vec { + columns.iter().map(column).collect() +} + +/// The source node. +pub(crate) fn source_node(source: &DagSourceInput) -> DagValue { + let entry: &DagSourceEntryInput = &source.entry; + let table: &str = entry_table(entry); + let mut asset_key: Vec = present(&[ + entry.database.as_deref(), + entry.schema.as_deref(), + Some(table), + ]); + if asset_key.is_empty() { + asset_key.push(entry.name.clone()); + } + NodeFields { + id: node_id(&source.key), + kind: SOURCE_KIND.to_owned(), + name: source.name.clone(), + asset_key, + target: Some(entry_target(( + entry.database.as_deref(), + entry.schema.as_deref(), + table, + ))), + path: Some(source.path.clone()), + description: entry.description.clone(), + meta: Some(entry.meta.clone()), + columns: columns(&entry.columns), + ..NodeFields::default() + } + .value() +} + +/// `loader_destination_parts`: `(database, schema, table)` of a one- to three-part destination. +fn destination_parts( + destination: &str, + defaults: (Option<&str>, Option<&str>), +) -> Result<(Option, Option, String), DagArtifactError> { + let parts: Vec<&str> = destination.split('.').collect(); + if parts.len() > LOADER_MAX_PARTS || parts.iter().any(|part| python_strip(part).is_empty()) { + return Err(DagArtifactError::LoaderDestination(format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ))); + } + let owned = |part: Option<&str>| part.map(str::to_owned); + Ok(match parts.as_slice() { + [table] => (owned(defaults.0), owned(defaults.1), (*table).to_owned()), + [schema, table] => ( + owned(defaults.0), + Some((*schema).to_owned()), + (*table).to_owned(), + ), + _ => ( + Some(parts[0].to_owned()), + Some(parts[1].to_owned()), + parts[2].to_owned(), + ), + }) +} + +/// The node of a loader no source declares. +pub(crate) fn loader_node( + loader: &DagLoaderInput, + project: &DagProjectInput, +) -> Result { + let destination: String = loader + .destination + .clone() + .unwrap_or_else(|| format!("{LOADER_DESTINATION_PREFIX}{}", loader.name)); + let (database, schema, table) = destination_parts( + &destination, + ( + project.target_database.as_deref(), + project.target_schema.as_deref(), + ), + )?; + let table: &str = if table.is_empty() { + &loader.name + } else { + &table + }; + Ok(NodeFields { + id: loader_node_id(&loader.name), + kind: LOADER_KIND.to_owned(), + name: loader.name.clone(), + asset_key: vec![loader.name.clone()], + target: Some(entry_target(( + database.as_deref(), + schema.as_deref(), + table, + ))), + path: Some(loader.path.clone()), + meta: Some(DagValue::Object(vec![( + text_key(LOADER_NODE_META_KEY), + DagValue::Bool(true), + )])), + columns: columns(&loader.columns), + loader: Some(loader.name.clone()), + ..NodeFields::default() + } + .value()) +} + +/// The seed node. +pub(crate) fn seed_node(seed: &DagSeedInput) -> DagValue { + NodeFields { + id: node_id(&seed.key), + kind: SEED_KIND.to_owned(), + name: seed.name.clone(), + asset_key: relation_asset_key(&seed.destination), + target: Some(relation_target(&seed.destination)), + path: Some(seed.path.clone()), + description: seed.description.clone(), + tags: seed.tags.clone(), + meta: Some(seed.meta.clone()), + columns: columns(&seed.columns), + ..NodeFields::default() + } + .value() +} + +/// The UDF or table function node. +pub(crate) fn function_node(function: &DagFunctionInput) -> DagValue { + let (kind, return_kind) = if function.return_columns.is_empty() { + (UDF_KIND, SCALAR_RETURN_KIND) + } else { + (TABLE_FUNCTION_KIND, TABLE_RETURN_KIND) + }; + NodeFields { + id: format!("{kind}:{}", function.name), + kind: kind.to_owned(), + name: function.name.clone(), + asset_key: relation_asset_key(&function.destination), + target: Some(relation_target(&function.destination)), + path: Some(function.path.clone()), + sql: Some(function.body_sql.clone()), + language: Some(function.language.clone()), + tags: function.tags.clone(), + return_kind: Some(return_kind.to_owned()), + returns: Some(function.returns.clone()), + columns: function.return_columns.iter().map(typed_column).collect(), + arguments: function.arguments.clone(), + ..NodeFields::default() + } + .value() +} + +fn model_tags(model: &DagModelInput) -> Vec { + let schema_tags: Vec = model + .schema + .as_ref() + .map_or_else(Vec::new, |schema| schema.tags.clone()); + let Some(config_tags) = &model.config_tags else { + return schema_tags; + }; + let mut tags: Vec = Vec::new(); + for tag in schema_tags.iter().chain(config_tags) { + if !tags.contains(tag) { + tags.push(tag.clone()); + } + } + tags +} + +fn model_columns(model: &DagModelInput) -> Vec { + if let Some(schema) = model + .schema + .as_ref() + .filter(|schema| !schema.columns.is_empty()) + { + return columns(&schema.columns); + } + model + .inferred_columns + .as_ref() + .map_or_else(Vec::new, |inferred| { + inferred.iter().map(typed_column).collect() + }) +} + +/// The model node. +pub(crate) fn model_node(model: &DagModelInput) -> DagValue { + NodeFields { + id: node_id(&model.key), + kind: MODEL_KIND.to_owned(), + name: model.name.clone(), + asset_key: relation_asset_key(&model.destination), + target: Some(relation_target(&model.destination)), + path: Some(model.path.clone()), + description: model + .schema + .as_ref() + .and_then(|schema| schema.description.clone()), + sql: Some(model.authored_sql.clone()), + tags: model_tags(model), + meta: model.schema.as_ref().map(|schema| schema.meta.clone()), + columns: model_columns(model), + materialization_type: Some(model.materialization.clone()), + ..NodeFields::default() + } + .value() +} + +/// The node of a Python task, asset or check. +pub(crate) fn python_node(node: &DagPythonNodeInput) -> DagValue { + let mut fields: NodeFields = NodeFields { + id: python_node_id(&node.kind, &node.name), + kind: node.kind.clone(), + name: node.name.clone(), + asset_key: vec![node.kind.clone(), node.name.clone()], + path: Some(node.path.clone()), + description: node.description.clone(), + tags: node.tags.clone(), + group: node.group.clone(), + meta: Some(node.meta.clone()), + ..NodeFields::default() + }; + if let Some(asset) = node.asset.as_ref().filter(|_| node.kind == ASSET_KIND) { + fields.columns = columns(&asset.columns); + fields.column_lineage = asset.column_lineage.clone().unwrap_or_default(); + fields.materialization_type = Some(DEFAULT_MATERIALIZATION_PYTHON_ASSET.to_owned()); + } + fields.value() +} + +/// Each loader's declaring source entry name, the last source naming a loader winning. +pub(crate) fn sources_by_loader(project: &DagProjectInput) -> HashMap<&str, &str> { + let mut sources: HashMap<&str, &str> = HashMap::new(); + for source in &project.sources { + if let Some(loader) = source.entry.loader.as_deref() { + sources.insert(loader, source.entry.name.as_str()); + } + } + sources +} + +/// The source node id of a loader a source declares, or its loader node id. +pub(crate) fn loader_or_source_id(name: &str, sources: &HashMap<&str, &str>) -> String { + sources.get(name).map_or_else( + || loader_node_id(name), + |source| node_id(&(SOURCE_KIND.to_owned(), (*source).to_owned())), + ) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs new file mode 100644 index 0000000000..aa8c96dd3e --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs @@ -0,0 +1,101 @@ +//! Building artifact values, dropping `None` and empty optional fields, and encoding them. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::dag_artifact::constants::REQUIRED_COLLECTION_KEYS; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::DagColumnInput; +use crate::dag_artifact::types::DagValue; +use crate::python_values::main::encode_python_value::encode_python_value; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::PythonKey; + +/// An object whose keys are all field names. +pub(crate) fn object(entries: Vec<(&str, DagValue)>) -> DagValue { + DagValue::Object( + entries + .into_iter() + .map(|(key, value)| (text_key(key), value)) + .collect(), + ) +} + +/// A string field. +pub(crate) fn text(value: &str) -> DagValue { + DagValue::String(value.to_owned()) +} + +/// An optional string field. +pub(crate) fn optional_text(value: Option<&str>) -> DagValue { + value.map_or(DagValue::Null, text) +} + +/// A list of strings. +pub(crate) fn texts<'a>(values: impl IntoIterator) -> DagValue { + DagValue::Array(values.into_iter().map(text).collect()) +} + +/// An integer field. +pub(crate) fn integer(value: i64) -> DagValue { + DagValue::Integer(JsonInteger::from(value)) +} + +/// A `DagColumn` of an authored column. +pub(crate) fn column(column: &DagColumnInput) -> DagValue { + object(vec![ + ("name", text(&column.name)), + ("type", optional_text(column.data_type.as_deref())), + ( + "nullable", + column.nullable.map_or(DagValue::Null, DagValue::Bool), + ), + ("description", optional_text(column.description.as_deref())), + ("meta", column.meta.clone()), + ]) +} + +/// A `DagColumn` of only a name and type. +pub(crate) fn typed_column((name, data_type): &(String, Option)) -> DagValue { + object(vec![ + ("name", text(name)), + ("type", optional_text(data_type.as_deref())), + ("nullable", DagValue::Null), + ("description", DagValue::Null), + ("meta", DagValue::Object(Vec::new())), + ]) +} + +fn is_required_collection(key: &PythonKey) -> bool { + key.text + .as_ref() + .is_ok_and(|key| REQUIRED_COLLECTION_KEYS.contains(&key.as_str())) +} + +fn is_empty_collection(value: &DagValue) -> bool { + match value { + DagValue::Array(items) => items.is_empty(), + DagValue::Object(entries) => entries.is_empty(), + _ => false, + } +} + +/// `_drop_none`: drop `None` fields, then empty optional collections, at every depth. +pub(crate) fn drop_none(value: DagValue) -> DagValue { + match value { + DagValue::Array(items) => DagValue::Array(items.into_iter().map(drop_none).collect()), + DagValue::Object(entries) => DagValue::Object( + entries + .into_iter() + .filter(|(_, item)| *item != DagValue::Null) + .map(|(key, item)| (key, drop_none(item))) + .filter(|(key, item)| is_required_collection(key) || !is_empty_collection(item)) + .collect(), + ), + other => other, + } +} + +/// The value `json.dumps` encodes, or the `TypeError` it raises first in output order. +pub(crate) fn encoded(value: DagValue) -> Result { + encode_python_value(value).map_err(DagArtifactError::Unencodable) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs new file mode 100644 index 0000000000..8b877efc20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -0,0 +1,29 @@ +//! Versions, kinds and keys of the static DAG artifact. + +/// The artifact's `version`. +pub(crate) const DAG_VERSION: i64 = 1; +/// Keys whose empty collections are written instead of dropped. +pub(crate) const REQUIRED_COLLECTION_KEYS: [&str; 3] = ["nodes", "edges", "checks"]; +/// `json.dumps(..., indent=2)`. +pub(crate) const DAG_INDENT: usize = 2; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const LOADER_KIND: &str = "loader"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub(crate) const ASSET_KIND: &str = "asset"; +pub(crate) const CHECK_KIND: &str = "check"; +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; +pub(crate) const AUDIT_KIND: &str = "audit"; +pub(crate) const SCENARIO_KIND: &str = "scenario"; +pub(crate) const PYTHON_CHECK_KIND: &str = "python_check"; +pub(crate) const DEFAULT_MATERIALIZATION_PYTHON_ASSET: &str = "python_asset"; +pub(crate) const TABLE_RETURN_KIND: &str = "table"; +pub(crate) const SCALAR_RETURN_KIND: &str = "scalar"; +/// The meta of a loader node no source declares. +pub(crate) const LOADER_NODE_META_KEY: &str = "sqlbuild_loader_node"; +/// The destination of a loader without one. +pub(crate) const LOADER_DESTINATION_PREFIX: &str = "__loader__"; +/// The most parts a loader destination may have. +pub(crate) const LOADER_MAX_PARTS: usize = 3; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs new file mode 100644 index 0000000000..b37aa7a6eb --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs @@ -0,0 +1,16 @@ +//! Why the DAG artifact could not be built, named after the Python error each reproduces. + +use sqlbuild_core::json::errors::JsonEmitError; + +/// A failure building or encoding the DAG artifact. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DagArtifactError { + /// `SpecConfigError` for a loader destination that is not 1 to 3 non-empty parts. + LoaderDestination(String), + /// `KeyError` for a bound SQL test parameter missing from the parameter schema. + MissingParameterType(String), + /// `TypeError` for the first value `json.dumps` cannot encode. + Unencodable(String), + /// The emitter refused the encoded artifact. + Emit(JsonEmitError), +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs new file mode 100644 index 0000000000..8ec9694e35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs @@ -0,0 +1,65 @@ +//! The static DAG artifact JSON of a compiled project and its Python node graph. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::checks::checks; +use crate::dag_artifact::_helpers::edges::edges; +use crate::dag_artifact::_helpers::nodes::{ + function_node, loader_node, model_node, python_node, seed_node, source_node, sources_by_loader, +}; +use crate::dag_artifact::_helpers::values::{drop_none, encoded, integer, object, text}; +use crate::dag_artifact::constants::{DAG_INDENT, DAG_VERSION, LOADER_KIND}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use crate::dag_artifact::types::DagValue; +use crate::graph::models::ProjectGraph; + +fn nodes( + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> Result, DagArtifactError> { + let declared: HashMap<&str, &str> = sources_by_loader(project); + let mut nodes: Vec = project.sources.iter().map(source_node).collect(); + for loader in &project.loaders { + if !declared.contains_key(loader.name.as_str()) { + nodes.push(loader_node(loader, project)?); + } + } + nodes.extend(project.seeds.iter().map(seed_node)); + nodes.extend(project.functions.iter().map(function_node)); + nodes.extend(project.models.iter().map(model_node)); + if let Some(python) = python { + nodes.extend( + python + .nodes + .iter() + .filter(|node| node.kind != LOADER_KIND) + .map(python_node), + ); + } + Ok(nodes) +} + +/// `json.dumps` of the artifact with `None` and empty optional fields dropped, indented by two. +pub fn build_dag_json( + graph: &ProjectGraph, + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, + project_name: &str, +) -> Result { + let artifact: DagValue = object(vec![ + ("version", integer(DAG_VERSION)), + ("project_name", text(project_name)), + ("nodes", DagValue::Array(nodes(project, python)?)), + ("edges", edges(graph, project, python)), + ("checks", checks(project, python)?), + ]); + let value: JsonValue = encoded(drop_none(artifact))?; + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(DAG_INDENT))), + ) + .map_err(DagArtifactError::Emit) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs new file mode 100644 index 0000000000..60780629b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -0,0 +1 @@ +pub mod build_dag_json; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/mod.rs new file mode 100644 index 0000000000..38a9736573 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs new file mode 100644 index 0000000000..660a118be2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -0,0 +1,218 @@ +//! The compiled project, Python node graph and authored values the DAG artifact is built from. + +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::types::{DagColumnLineage, DagValue}; + +/// One authored column: name, type, nullability, description and meta. +#[derive(Clone, Debug, PartialEq)] +pub struct DagColumnInput { + pub name: String, + pub data_type: Option, + pub nullable: Option, + pub description: Option, + pub meta: DagValue, +} + +/// A compiled relation location. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagRelationInput { + pub database: Option, + pub schema: Option, + pub name: Option, + pub qualified_name: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +/// A source declaration's entry. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSourceEntryInput { + pub name: String, + pub database: Option, + pub schema: Option, + pub table: Option, + pub description: Option, + pub loader: Option, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled source. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSourceInput { + pub key: ObjectKey, + pub name: String, + /// `str(source_file.relative_path)`. + pub path: String, + pub entry: DagSourceEntryInput, +} + +/// A discovered loader function. +#[derive(Clone, Debug, PartialEq)] +pub struct DagLoaderInput { + pub name: String, + pub path: String, + /// The authored destination, if any. + pub destination: Option, + pub columns: Vec, + /// Names of the project loaders this loader depends on, in dependency order. + pub dependency_names: Vec, +} + +/// A compiled seed. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSeedInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub description: Option, + pub tags: Vec, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled SQL or Python function. +#[derive(Clone, Debug, PartialEq)] +pub struct DagFunctionInput { + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub body_sql: String, + pub language: String, + pub tags: Vec, + pub returns: String, + /// `(name, type)` of each returned column; a table function has at least one. + pub return_columns: Vec<(String, Option)>, + pub arguments: Vec<(String, String)>, +} + +/// The parts of a model's schema entry the artifact reads. +#[derive(Clone, Debug, PartialEq)] +pub struct DagModelSchemaInput { + pub description: Option, + pub tags: Vec, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled model. +#[derive(Clone, Debug, PartialEq)] +pub struct DagModelInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub authored_sql: String, + pub schema: Option, + /// `str()` of each configured tag, when the `tags` config value is a list or tuple. + pub config_tags: Option>, + /// `str()` of the `materialized` config value, `view` by default. + pub materialization: String, + pub inferred_columns: Option)>>, +} + +/// A Python asset's declared columns and column lineage. +#[derive(Clone, Debug, PartialEq)] +pub struct DagPythonAssetInput { + pub columns: Vec, + /// Column to `(upstream node, upstream column)` references, when declared. + pub column_lineage: Option, +} + +/// A discovered Python node. +#[derive(Clone, Debug, PartialEq)] +pub struct DagPythonNodeInput { + pub kind: String, + pub name: String, + pub path: String, + pub description: Option, + pub tags: Vec, + pub group: Option, + /// `node.meta or {}`. + pub meta: DagValue, + pub asset: Option, + /// The `AuditSeverity` value of a check's severity. + pub check_severity: Option, + /// Graph keys of the SQL resources the node reads. + pub sql_deps: Vec, +} + +/// The discovered Python node graph. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DagPythonGraphInput { + pub nodes: Vec, + /// `(upstream name, downstream name)` of each dependency edge. + pub dependency_edges: Vec<(String, String)>, + /// Each name's node kind, as `nodes_by_name` maps it. + pub kinds_by_name: Vec<(String, String)>, +} + +/// One case of a parameterized SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSqlTestCaseInput { + pub case_name: String, + pub block_index: Option, + pub parent_name: Option, + pub case_index: Option, + pub case_fingerprint: Option, + /// `(name, value type, nullable)` of each declared parameter. + pub parameter_schema: Vec<(String, String, bool)>, + /// `(name, value)` of each bound parameter, decimals already as text. + pub parameters: Vec<(String, DagValue)>, +} + +/// A compiled SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSqlTestInput { + pub key: ObjectKey, + pub name: String, + /// `str(test_file.relative_path)`. + pub path: String, + /// `(source_path or test_file.relative_path).as_posix()`. + pub source_path: String, + pub mode: String, + pub scope_deps: Vec, + pub case: Option, +} + +/// A compiled audit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DagAuditInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub scope_deps: Vec, + pub attached_target_kind: Option, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub severity: Option, +} + +/// A compiled SQL scenario. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DagScenarioInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub expected_model_names: Vec, + pub authored_cte_names: Vec, + pub assertion_names: Vec, + /// Source, ref, seed and dbt ref fixture names, each group in authored order. + pub fixture_names: [Vec; 4], +} + +/// The project's compiled resources and target defaults. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DagProjectInput { + pub sources: Vec, + pub loaders: Vec, + pub seeds: Vec, + pub functions: Vec, + pub models: Vec, + pub sql_tests: Vec, + pub audits: Vec, + pub scenarios: Vec, + pub target_database: Option, + pub target_schema: Option, +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs new file mode 100644 index 0000000000..e7bae12d29 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_dag_json.rs"] +mod build_dag_json; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs new file mode 100644 index 0000000000..c41a2a4f7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -0,0 +1,98 @@ +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::main::build_dag_json::build_dag_json; +use crate::dag_artifact::models::{ + DagColumnInput, DagLoaderInput, DagPythonNodeInput, DagSourceEntryInput, DagSourceInput, +}; +use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::types::DagValue; +use crate::graph::models::ProjectGraph; +use crate::python_values::main::text_key::text_key; + +/// A `(resource type, name)` key. +pub(crate) fn key(kind: &str, name: &str) -> ObjectKey { + (kind.to_owned(), name.to_owned()) +} + +/// A typed column without meta. +pub(crate) fn column(name: &str, data_type: &str) -> DagColumnInput { + DagColumnInput { + name: name.to_owned(), + data_type: Some(data_type.to_owned()), + nullable: None, + description: None, + meta: DagValue::Object(Vec::new()), + } +} + +/// A source in `raw` loaded by `loader`, when given. +pub(crate) fn source(name: &str, loader: Option<&str>, meta: DagValue) -> DagSourceInput { + DagSourceInput { + key: key("source", name), + name: name.to_owned(), + path: "sources/raw.yml".to_owned(), + entry: DagSourceEntryInput { + name: name.to_owned(), + database: None, + schema: Some("raw".to_owned()), + table: None, + description: None, + loader: loader.map(str::to_owned), + meta, + columns: vec![column("id", "INTEGER")], + }, + } +} + +/// A loader at `destination` depending on `dependencies`. +pub(crate) fn loader( + name: &str, + destination: Option<&str>, + dependencies: &[&str], +) -> DagLoaderInput { + DagLoaderInput { + name: name.to_owned(), + path: "python/loaders.py".to_owned(), + destination: destination.map(str::to_owned), + columns: Vec::new(), + dependency_names: dependencies.iter().map(|name| (*name).to_owned()).collect(), + } +} + +/// A Python node of `kind` reading the `sql_deps` keys. +pub(crate) fn python_node(kind: &str, name: &str, sql_deps: Vec) -> DagPythonNodeInput { + DagPythonNodeInput { + kind: kind.to_owned(), + name: name.to_owned(), + path: "python/nodes.py".to_owned(), + description: None, + tags: Vec::new(), + group: None, + meta: DagValue::Object(Vec::new()), + asset: None, + check_severity: (kind == "check").then(|| "error".to_owned()), + sql_deps, + } +} + +/// The artifact JSON of a test case. +pub(crate) fn built(test_case: &BuildDagJsonTestCase) -> Result { + let graph: ProjectGraph = ProjectGraph { + upstream: test_case.upstream.clone(), + ..ProjectGraph::default() + }; + build_dag_json( + &graph, + &test_case.project, + test_case.python.as_ref(), + "orders_project", + ) +} + +/// Source meta with a `None` owner and the given team. +pub(crate) fn team_meta(value: DagValue) -> DagValue { + DagValue::Object(vec![ + (text_key("owner"), DagValue::Null), + (text_key("team"), value), + ]) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs new file mode 100644 index 0000000000..1c540c5ae9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -0,0 +1,174 @@ +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use crate::dag_artifact::tests::helpers::{built, key, loader, python_node, source, team_meta}; +use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::types::DagValue; + +const EMPTY_JSON: &str = "{ + \"version\": 1, + \"project_name\": \"orders_project\", + \"nodes\": [], + \"edges\": [], + \"checks\": [] +}"; + +const LOADED_JSON: &str = "{ + \"version\": 1, + \"project_name\": \"orders_project\", + \"nodes\": [ + { + \"id\": \"source:orders\", + \"kind\": \"source\", + \"name\": \"orders\", + \"asset_key\": [ + \"raw\", + \"orders\" + ], + \"target\": { + \"schema\": \"raw\", + \"name\": \"orders\", + \"qualified_name\": \"raw.orders\" + }, + \"path\": \"sources/raw.yml\", + \"meta\": { + \"team\": \"ops\" + }, + \"columns\": [ + { + \"name\": \"id\", + \"type\": \"INTEGER\" + } + ] + }, + { + \"id\": \"loader:load_customers\", + \"kind\": \"loader\", + \"name\": \"load_customers\", + \"asset_key\": [ + \"load_customers\" + ], + \"target\": { + \"schema\": \"main\", + \"name\": \"__loader__load_customers\", + \"qualified_name\": \"main.__loader__load_customers\" + }, + \"path\": \"python/loaders.py\", + \"meta\": { + \"sqlbuild_loader_node\": true + }, + \"loader\": \"load_customers\" + }, + { + \"id\": \"check:orders_check\", + \"kind\": \"check\", + \"name\": \"orders_check\", + \"asset_key\": [ + \"check\", + \"orders_check\" + ], + \"path\": \"python/nodes.py\" + } + ], + \"edges\": [ + { + \"from_id\": \"source:orders\", + \"to_id\": \"check:orders_check\" + }, + { + \"from_id\": \"source:orders\", + \"to_id\": \"loader:load_customers\" + } + ], + \"checks\": [ + { + \"id\": \"check:orders_check\", + \"kind\": \"python_check\", + \"name\": \"orders_check\", + \"checked_asset_ids\": [ + \"source:orders\" + ], + \"path\": \"python/nodes.py\", + \"severity\": \"error\" + } + ] +}"; + +#[test] +fn given_project_when_building_dag_json_then_python_artifact_agrees() { + let test_cases = [ + BuildDagJsonTestCase { + description: "an empty project keeps its required collections", + project: DagProjectInput::default(), + python: None, + upstream: Vec::new(), + expected_json: Ok(EMPTY_JSON), + }, + BuildDagJsonTestCase { + description: "declared loaders become sources and None and empty fields are dropped", + project: DagProjectInput { + sources: vec![source( + "orders", + Some("load_orders"), + team_meta(DagValue::String("ops".to_owned())), + )], + loaders: vec![ + loader("load_orders", None, &[]), + loader("load_customers", None, &["load_orders"]), + ], + target_schema: Some("main".to_owned()), + ..DagProjectInput::default() + }, + python: Some(DagPythonGraphInput { + nodes: vec![python_node( + "check", + "orders_check", + vec![key("source", "orders")], + )], + dependency_edges: Vec::new(), + kinds_by_name: vec![("orders_check".to_owned(), "check".to_owned())], + }), + upstream: Vec::new(), + expected_json: Ok(LOADED_JSON), + }, + BuildDagJsonTestCase { + description: "a loader destination with an empty part is refused", + project: DagProjectInput { + loaders: vec![loader("load_orders", Some("raw..orders"), &[])], + ..DagProjectInput::default() + }, + python: None, + upstream: Vec::new(), + expected_json: Err(DagArtifactError::LoaderDestination( + "Invalid loader destination 'raw..orders': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + .to_owned(), + )), + }, + BuildDagJsonTestCase { + description: "a meta value json cannot encode raises its TypeError", + project: DagProjectInput { + sources: vec![source( + "orders", + None, + team_meta(DagValue::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + )), + )], + ..DagProjectInput::default() + }, + python: None, + upstream: Vec::new(), + expected_json: Err(DagArtifactError::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + built(&test_case), + test_case.expected_json.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs new file mode 100644 index 0000000000..529ebbda57 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs @@ -0,0 +1,11 @@ +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; + +pub(crate) struct BuildDagJsonTestCase { + pub(crate) description: &'static str, + pub(crate) project: DagProjectInput, + pub(crate) python: Option, + pub(crate) upstream: Vec<(ObjectKey, Vec)>, + pub(crate) expected_json: Result<&'static str, DagArtifactError>, +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/types.rs b/crates/sqlbuild-analysis/src/dag_artifact/types.rs new file mode 100644 index 0000000000..a23241346a --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/types.rs @@ -0,0 +1,9 @@ +//! Keys and lineage shapes of the DAG artifact inputs. + +use crate::python_values::models::PythonValue; + +/// A Python value as `dataclasses.asdict` copies it, before `json.dumps` encodes it. +pub type DagValue = PythonValue; + +/// Column to `(upstream node, upstream column)` references. +pub type DagColumnLineage = Vec<(String, Vec<(String, String)>)>; diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs index d2c4b0a0b0..cdeac2df37 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs @@ -7,30 +7,22 @@ use crate::explicit_references::constants::{ AUDIT_GATE_CODE, MODEL_TYPE, READABLE_TYPES, SEED_TYPE, SOURCE_TYPE, TABLE_FUNCTION_TYPE, UDF_TYPE, }; -use crate::explicit_references::models::{AuditGate, ReferenceDiagnostic, ReferenceRequest}; +use crate::explicit_references::models::{ + AuditGate, GateEdge, GraphResource, ReferenceDiagnostic, ReferenceRequest, +}; use crate::explicit_references::types::ResourceKey; -/// One ordering edge: `gated` waits for `read`, which an audit on `target` reads. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct GateEdge { - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, -} - /// An audit's target and the other resources it reads. struct Gate<'a> { + audit_name: &'a str, target: &'a ResourceKey, reads: Vec<&'a ResourceKey>, } /// `attached_audit_gate_edges_impl`: edges to a fixed point, as sources gain triggering models. -pub(crate) fn gate_edges(request: &ReferenceRequest) -> Vec { +pub(crate) fn gate_edges(resources: &[GraphResource], audits: &[AuditGate]) -> Vec { let mut triggers: HashMap> = HashMap::new(); - for resource in request - .resources - .iter() - .filter(|item| item.key.0 == MODEL_TYPE) - { + for resource in resources.iter().filter(|item| item.key.0 == MODEL_TYPE) { for dep in resource.deps.iter().filter(|dep| dep.0 == SOURCE_TYPE) { triggers .entry(dep.clone()) @@ -38,7 +30,7 @@ pub(crate) fn gate_edges(request: &ReferenceRequest) -> Vec { .insert(resource.key.clone()); } } - let gates: Vec> = request.audits.iter().filter_map(gate).collect(); + let gates: Vec> = audits.iter().filter_map(gate).collect(); let mut edges: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); let mut changed = true; @@ -105,7 +97,11 @@ pub(crate) fn gate_diagnostics( fn gate(audit: &AuditGate) -> Option> { let target = audit.target.as_ref()?; let reads: Vec<&ResourceKey> = extra_reads(audit, target); - (!reads.is_empty()).then_some(Gate { target, reads }) + (!reads.is_empty()).then_some(Gate { + audit_name: &audit.name, + target, + reads, + }) } fn extra_reads<'a>(audit: &'a AuditGate, target: &ResourceKey) -> Vec<&'a ResourceKey> { @@ -135,6 +131,8 @@ fn edges_of( for read in &gate.reads { for key in &gated { edges.push(GateEdge { + audit_name: gate.audit_name.to_owned(), + target: gate.target.clone(), gated: key.clone(), read: (*read).clone(), }); diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs index a81fda7c77..077c9b0930 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs @@ -2,32 +2,16 @@ use std::collections::HashSet; -use crate::explicit_references::_helpers::audit_gates::GateEdge; use crate::explicit_references::_helpers::graph::{Upstream, lineage_upstream, reaches}; use crate::explicit_references::constants::{ HOOK_READ_CODE, MODEL_TYPE, PYTHON_DESTINATION, SEED_TYPE, SOURCE_TYPE, SQL_DESTINATION, }; -use crate::explicit_references::models::{PythonHook, ReferenceDiagnostic, ReferenceRequest}; +use crate::explicit_references::models::{ + GateEdge, HookEdge, HookType, ModelReferences, PythonHook, ReferenceDiagnostic, + ReferenceRequest, +}; use crate::explicit_references::types::ResourceKey; -/// How diagnostics name a hook. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum HookType { - Python, - Sql, - InlineSql, -} - -/// One ordering edge: model `gated` waits for `read`, which one of its hooks reads. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct HookEdge { - hook_name: String, - gated: ResourceKey, - read: ResourceKey, - hook_type: HookType, - path: Option, -} - /// The `CompileInputError` message for the first unknown read, hooks taken by name. pub(crate) fn unknown_read(request: &ReferenceRequest) -> Option { let mut hooks: Vec<&PythonHook> = request.hooks.iter().collect(); @@ -59,7 +43,7 @@ pub(crate) fn hook_diagnostics( request: &ReferenceRequest, gates: &[GateEdge], ) -> Vec { - let mut edges: Vec = hook_edges(request); + let mut edges: Vec = hook_edges(&request.models, &request.hooks); if edges.is_empty() { return Vec::new(); } @@ -91,19 +75,17 @@ pub(crate) fn hook_diagnostics( } /// `hook_read_edges_impl`: Python then SQL hook reads per model, deduplicated, minus self-reads. -fn hook_edges(request: &ReferenceRequest) -> Vec { +pub(crate) fn hook_edges(models: &[ModelReferences], hooks: &[PythonHook]) -> Vec { let mut candidates: Vec = Vec::new(); - for model in &request.models { + for model in models { let gated: ResourceKey = (MODEL_TYPE.to_owned(), model.name.clone()); for hook_name in &model.python_hooks { - let reads: &[ResourceKey] = request - .hooks + let reads: &[ResourceKey] = hooks .iter() .rev() .find(|hook| &hook.name == hook_name && !hook.reads.is_empty()) .map_or(&[], |hook| hook.reads.as_slice()); - let path: Option = request - .hooks + let path: Option = hooks .iter() .rev() .find(|hook| &hook.name == hook_name) diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs b/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs index 21f50afd64..c3e106bffe 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs @@ -23,7 +23,7 @@ fn checked(request: &ReferenceRequest) -> Result ExecutionGateEdges { + ExecutionGateEdges { + audit_gates: gate_edges(&request.resources, &request.audits), + hook_reads: hook_edges(&request.models, &request.hooks), + } +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs index 7cb27294a1..9e7e305311 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs @@ -1 +1,2 @@ pub mod check_explicit_references; +pub mod execution_gate_edges; diff --git a/crates/sqlbuild-analysis/src/explicit_references/models.rs b/crates/sqlbuild-analysis/src/explicit_references/models.rs index 6c730de62d..21021e13a4 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/models.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/models.rs @@ -34,7 +34,7 @@ pub struct GraphResource { } /// A relation name with the qualifiers written or configured. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Deserialize, Serialize)] pub struct Relation { pub name: String, pub schema: Option, @@ -46,6 +46,7 @@ pub struct Relation { pub struct ModelReferences { pub name: String, pub path: String, + #[serde(default)] pub destination: Relation, /// `model_python_hook_names(model)`. pub python_hooks: Vec, @@ -54,6 +55,50 @@ pub struct ModelReferences { pub migrate_from: Option, } +/// The facts execution ordering reads: models with hooks, audits and Python hooks. +#[derive(Debug, Clone, Deserialize)] +pub struct ExecutionGateRequest { + /// Models in project order with their dependencies. + pub resources: Vec, + pub models: Vec, + pub audits: Vec, + pub hooks: Vec, +} + +/// One ordering edge: `gated` waits for `read`, which audit `audit_name` on `target` reads. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GateEdge { + pub audit_name: String, + pub target: ResourceKey, + pub gated: ResourceKey, + pub read: ResourceKey, +} + +/// How diagnostics name a hook. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HookType { + Python, + Sql, + InlineSql, +} + +/// One ordering edge: model `gated` waits for `read`, which one of its hooks reads. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HookEdge { + pub hook_name: String, + pub gated: ResourceKey, + pub read: ResourceKey, + pub hook_type: HookType, + pub path: Option, +} + +/// Python's attached-audit gate edges and hook-read edges, in its order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ExecutionGateEdges { + pub audit_gates: Vec, + pub hook_reads: Vec, +} + /// One SQL hook entry with declared reads: its hook key, position and identity. #[derive(Debug, Clone, Deserialize)] pub struct SqlHook { diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs new file mode 100644 index 0000000000..5de163d71b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs @@ -0,0 +1,113 @@ +//! Standard-alphabet base64 as Python's `base64.b64encode` and strict `b64decode` apply it. + +const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const PAD: u8 = b'='; +const SECOND_BYTE: usize = 1; +const THIRD_BYTE: usize = 2; +const PAD_START_POSITION: usize = 2; +const QUAD_LENGTH: usize = 4; + +/// `base64.b64encode(value).decode("ascii")`. +pub(crate) fn encode(value: &[u8]) -> String { + let mut encoded = String::with_capacity(value.len().div_ceil(3) * 4); + for chunk in value.chunks(3) { + let first = chunk[0]; + let second = chunk.get(1).copied().unwrap_or(0); + let third = chunk.get(2).copied().unwrap_or(0); + encoded.push(char::from(ALPHABET[usize::from(first >> 2)])); + encoded.push(char::from( + ALPHABET[usize::from(((first & 0x03) << 4) | (second >> 4))], + )); + if chunk.len() > SECOND_BYTE { + encoded.push(char::from( + ALPHABET[usize::from(((second & 0x0f) << 2) | (third >> 6))], + )); + } else { + encoded.push(char::from(PAD)); + } + if chunk.len() > THIRD_BYTE { + encoded.push(char::from(ALPHABET[usize::from(third & 0x3f)])); + } else { + encoded.push(char::from(PAD)); + } + } + encoded +} + +/// `binascii.a2b_base64(text, strict_mode=True)`, with CPython's error messages. +pub(crate) fn decode_strict(text: &[u8]) -> Result, String> { + if text.first() == Some(&PAD) { + return Err("Leading padding not allowed".to_owned()); + } + let mut decoded: Vec = Vec::with_capacity(text.len() / 4 * 3); + let mut quad_position: usize = 0; + let mut left_char: u8 = 0; + let mut pads: usize = 0; + let mut padding_started = false; + for (index, &character) in text.iter().enumerate() { + if character == PAD { + padding_started = true; + if quad_position == 0 { + return Err("Excess padding not allowed".to_owned()); + } + if quad_position >= PAD_START_POSITION { + pads += 1; + if quad_position + pads >= QUAD_LENGTH { + if index + 1 < text.len() { + return Err("Excess data after padding".to_owned()); + } + return Ok(decoded); + } + } + continue; + } + let Some(value) = sextet(character) else { + return Err("Only base64 data is allowed".to_owned()); + }; + if padding_started { + return Err("Discontinuous padding not allowed".to_owned()); + } + pads = 0; + match quad_position { + 0 => { + quad_position = 1; + left_char = value; + } + 1 => { + quad_position = 2; + decoded.push((left_char << 2) | (value >> 4)); + left_char = value & 0x0f; + } + 2 => { + quad_position = 3; + decoded.push((left_char << 4) | (value >> 2)); + left_char = value & 0x03; + } + _ => { + quad_position = 0; + decoded.push((left_char << 6) | value); + left_char = 0; + } + } + } + match quad_position { + 0 => Ok(decoded), + 1 => Err(format!( + "Invalid base64-encoded string: number of data characters ({}) cannot be 1 more \ + than a multiple of 4", + decoded.len() / 3 * 4 + 1 + )), + _ => Err("Incorrect padding".to_owned()), + } +} + +fn sextet(character: u8) -> Option { + match character { + b'A'..=b'Z' => Some(character - b'A'), + b'a'..=b'z' => Some(character - b'a' + 26), + b'0'..=b'9' => Some(character - b'0' + 52), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs new file mode 100644 index 0000000000..96ece2e4d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs @@ -0,0 +1,20 @@ +//! SQL string literals for fingerprint rows. + +/// `'value'` with single quotes doubled. +pub(crate) fn string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// `NULL`, or the quoted literal. +pub(crate) fn optional_string_literal(value: Option<&str>) -> String { + value.map_or_else(|| "NULL".to_owned(), string_literal) +} + +/// `'a', 'b'` for an `IN` list. +pub(crate) fn literal_list(values: &[String]) -> String { + values + .iter() + .map(|value| string_literal(value)) + .collect::>() + .join(", ") +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs new file mode 100644 index 0000000000..7a2f4a0fa2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod base64; +pub(crate) mod literals; +pub(crate) mod table_columns; +pub(crate) mod utf8; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs new file mode 100644 index 0000000000..4fd7db8669 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs @@ -0,0 +1,8 @@ +//! The stored column list. + +use crate::fingerprint_storage::constants::FINGERPRINT_COLUMNS; + +/// Every stored column, comma separated. +pub(crate) fn column_list() -> String { + FINGERPRINT_COLUMNS.join(", ") +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs new file mode 100644 index 0000000000..b872db77d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs @@ -0,0 +1,31 @@ +//! UTF-8 decoding failures described as CPython's decoder reports them. + +use crate::fingerprint_storage::errors::StorageDecodeError; + +/// `bytes` as text, or Python's `UnicodeDecodeError` range and reason. +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + let error = match String::from_utf8(bytes) { + Ok(text) => return Ok(text), + Err(error) => error, + }; + let utf8_error = error.utf8_error(); + let bytes: Vec = error.into_bytes(); + let start: usize = utf8_error.valid_up_to(); + let (end, reason): (usize, &'static str) = match utf8_error.error_len() { + None => (bytes.len(), "unexpected end of data"), + Some(length) if length == 1 && invalid_start_byte(bytes[start]) => { + (start + 1, "invalid start byte") + } + Some(length) => (start + length, "invalid continuation byte"), + }; + Err(StorageDecodeError::Utf8 { + bytes, + start, + end, + reason, + }) +} + +fn invalid_start_byte(byte: u8) -> bool { + matches!(byte, 0x80..=0xC1 | 0xF5..=0xFF) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs new file mode 100644 index 0000000000..17aad5e650 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -0,0 +1,33 @@ +//! Fingerprint table column names, in table order. + +pub const COLUMN_NODE_TYPE: &str = "node_type"; +pub const COLUMN_NODE_NAME: &str = "node_name"; +pub const COLUMN_TARGET_DATABASE: &str = "target_database"; +pub const COLUMN_TARGET_SCHEMA: &str = "target_schema"; +pub const COLUMN_TARGET_NAME: &str = "target_name"; +pub const COLUMN_RUN_ID: &str = "run_id"; +pub const COLUMN_DEFINITION_HASH: &str = "definition_hash"; +pub const COLUMN_VERSION_HASH: &str = "version_hash"; +pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; +pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; +pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; +pub const COLUMN_TIMESTAMP: &str = "ts"; + +/// Every stored column, in table order. +pub const FINGERPRINT_COLUMNS: [&str; 12] = [ + COLUMN_NODE_TYPE, + COLUMN_NODE_NAME, + COLUMN_TARGET_DATABASE, + COLUMN_TARGET_SCHEMA, + COLUMN_TARGET_NAME, + COLUMN_RUN_ID, + COLUMN_DEFINITION_HASH, + COLUMN_VERSION_HASH, + COLUMN_SCHEMA_FINGERPRINT, + COLUMN_DEFINITION_B64, + COLUMN_METADATA_JSON_B64, + COLUMN_TIMESTAMP, +]; + +/// The node type that wins a node-name collision between latest rows. +pub const NODE_TYPE_MODEL: &str = "model"; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs new file mode 100644 index 0000000000..6a10d0fb8b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs @@ -0,0 +1,37 @@ +//! Failures of encoding and decoding fingerprint rows. + +use std::fmt::{self, Display}; + +/// Invalid insert input, with Python's `FingerprintInputError` message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FingerprintStorageError { + EmptyInsert, + NonPositiveBatchSize, + NonPositiveBatchBytes, +} + +impl Display for FingerprintStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::EmptyInsert => "fingerprint insert requires at least one row", + Self::NonPositiveBatchSize => "fingerprint batch size must be positive", + Self::NonPositiveBatchBytes => "fingerprint batch byte limit must be positive", + }) + } +} + +impl std::error::Error for FingerprintStorageError {} + +/// Why stored base64 text is not base64-encoded UTF-8, as Python's decoders report it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageDecodeError { + /// `binascii.Error` from strict base64 decoding. + Base64(String), + /// `UnicodeDecodeError` from UTF-8 decoding the base64 payload. + Utf8 { + bytes: Vec, + start: usize, + end: usize, + reason: &'static str, + }, +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs new file mode 100644 index 0000000000..7c0a00587e --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -0,0 +1,26 @@ +//! Fingerprint table DDL. + +use crate::fingerprint_storage::constants::{ + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, + COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, + COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, +}; + +/// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. +pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: &str) -> String { + format!( + "CREATE TABLE IF NOT EXISTS {qualified_name} (\ + {COLUMN_NODE_TYPE} {string_type} NOT NULL, \ + {COLUMN_NODE_NAME} {string_type} NOT NULL, \ + {COLUMN_TARGET_DATABASE} {string_type}, \ + {COLUMN_TARGET_SCHEMA} {string_type}, \ + {COLUMN_TARGET_NAME} {string_type}, \ + {COLUMN_RUN_ID} {string_type} NOT NULL, \ + {COLUMN_DEFINITION_HASH} {string_type} NOT NULL, \ + {COLUMN_VERSION_HASH} {string_type} NOT NULL, \ + {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ + {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ + {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + ) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs new file mode 100644 index 0000000000..3c224af96d --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs @@ -0,0 +1,10 @@ +//! Stored definition text decoding. + +use crate::fingerprint_storage::_helpers::base64::decode_strict; +use crate::fingerprint_storage::_helpers::utf8::decode_utf8; +use crate::fingerprint_storage::errors::StorageDecodeError; + +/// Stored ASCII base64 text decoded to UTF-8, failing as Python's strict decoders do. +pub fn decode_storage(storage: &str) -> Result { + decode_utf8(decode_strict(storage.as_bytes()).map_err(StorageDecodeError::Base64)?) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs new file mode 100644 index 0000000000..b6f2f90abb --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs @@ -0,0 +1,42 @@ +//! Bounded fingerprint insert batches. + +use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::insert_sql::insert_sql; +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Batch lengths in order, each within `batch_size` rows and `batch_max_bytes` of row SQL. +pub fn insert_batch_lengths( + qualified_name: &str, + fingerprints: &[StoredFingerprint], + batch_size: i64, + batch_max_bytes: i64, +) -> Result, FingerprintStorageError> { + if batch_size < 1 { + return Err(FingerprintStorageError::NonPositiveBatchSize); + } + if batch_max_bytes < 1 { + return Err(FingerprintStorageError::NonPositiveBatchBytes); + } + let mut lengths: Vec = Vec::new(); + let mut current_length: usize = 0; + let mut current_bytes: i64 = 0; + for fingerprint in fingerprints { + let single_row_bytes: i64 = + i64::try_from(insert_sql(qualified_name, std::slice::from_ref(fingerprint))?.len()) + .unwrap_or(i64::MAX); + if current_length > 0 + && (i64::try_from(current_length).unwrap_or(i64::MAX) >= batch_size + || current_bytes.saturating_add(single_row_bytes) > batch_max_bytes) + { + lengths.push(current_length); + current_length = 0; + current_bytes = 0; + } + current_length += 1; + current_bytes = current_bytes.saturating_add(single_row_bytes); + } + if current_length > 0 { + lengths.push(current_length); + } + Ok(lengths) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs new file mode 100644 index 0000000000..25e54bf696 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -0,0 +1,41 @@ +//! Append-only fingerprint inserts. + +use crate::fingerprint_storage::_helpers::base64::encode; +use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; +use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::models::StoredFingerprint; + +/// One multi-row `INSERT` for a non-empty batch. +pub fn insert_sql( + qualified_name: &str, + fingerprints: &[StoredFingerprint], +) -> Result { + if fingerprints.is_empty() { + return Err(FingerprintStorageError::EmptyInsert); + } + let rows: Vec = fingerprints.iter().map(row_values).collect(); + Ok(format!( + "INSERT INTO {qualified_name} ({}) VALUES {}", + column_list(), + rows.join(", ") + )) +} + +fn row_values(fingerprint: &StoredFingerprint) -> String { + let values: [String; 12] = [ + string_literal(&fingerprint.node_type), + string_literal(&fingerprint.node_name), + optional_string_literal(fingerprint.target_database.as_deref()), + optional_string_literal(fingerprint.target_schema.as_deref()), + optional_string_literal(fingerprint.target_name.as_deref()), + format!("'{}'", fingerprint.run_id), + format!("'{}'", fingerprint.definition_hash), + format!("'{}'", fingerprint.version_hash), + format!("'{}'", fingerprint.schema_fingerprint), + format!("'{}'", encode(fingerprint.definition.as_bytes())), + format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), + format!("'{}'", fingerprint.ts), + ]; + format!("({})", values.join(", ")) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs new file mode 100644 index 0000000000..6a3a61f131 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs @@ -0,0 +1,39 @@ +//! Latest fingerprint rows indexed by identity and by node name. + +use std::collections::HashMap; + +use crate::fingerprint_storage::constants::NODE_TYPE_MODEL; +use crate::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; + +/// Kept row positions in Python dict order, a model winning a node-name collision. +pub fn latest_index(identities: &[FingerprintIdentity<'_>]) -> LatestIndex { + let mut by_name: Vec = Vec::new(); + let mut name_slots: HashMap<&str, usize> = HashMap::new(); + let mut by_identity: Vec = Vec::new(); + let mut identity_slots: HashMap<(&str, &str), usize> = HashMap::new(); + for (position, identity) in identities.iter().enumerate() { + let key = (identity.node_type, identity.node_name); + match identity_slots.get(&key) { + Some(&slot) => by_identity[slot] = position, + None => { + let _ = identity_slots.insert(key, by_identity.len()); + by_identity.push(position); + } + } + match name_slots.get(identity.node_name) { + Some(&slot) => { + if identity.node_type == NODE_TYPE_MODEL { + by_name[slot] = position; + } + } + None => { + let _ = name_slots.insert(identity.node_name, by_name.len()); + by_name.push(position); + } + } + } + LatestIndex { + by_name, + by_identity, + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs new file mode 100644 index 0000000000..54c1f22e6f --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -0,0 +1,8 @@ +pub mod create_table_sql; +pub mod decode_storage; +pub mod insert_batch_lengths; +pub mod insert_sql; +pub mod latest_index; +pub mod read_latest_sql; +pub mod relevant_read_sql; +pub mod selected_positions; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs new file mode 100644 index 0000000000..b6d1baeac6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs @@ -0,0 +1,18 @@ +//! The latest-row read of the fingerprint table. + +use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::{ + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_TIMESTAMP, +}; + +/// A windowed `SELECT` of the latest row per node identity. +pub fn read_latest_sql(qualified_name: &str) -> String { + let columns: String = column_list(); + format!( + "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (\ + PARTITION BY {COLUMN_NODE_TYPE}, {COLUMN_NODE_NAME} \ + ORDER BY {COLUMN_TIMESTAMP} DESC, {COLUMN_RUN_ID} DESC) AS __sqlbuild_latest_rank \ + FROM {qualified_name}) AS __sqlbuild_latest_fingerprints \ + WHERE __sqlbuild_latest_rank = 1" + ) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs new file mode 100644 index 0000000000..487a79bea2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs @@ -0,0 +1,34 @@ +//! The latest-row read narrowed to the relevant nodes. + +use crate::fingerprint_storage::_helpers::literals::literal_list; +use crate::fingerprint_storage::models::RelevantRead; + +/// The read limited to `node_names` plus unfiltered node types; `None` names keep every row. +pub fn relevant_read_sql( + read_sql: &str, + node_names: Option<&[String]>, + filtered_node_types: &[String], +) -> RelevantRead { + let Some(node_names) = node_names else { + return RelevantRead::Sql(read_sql.to_owned()); + }; + if node_names.is_empty() && filtered_node_types.is_empty() { + return RelevantRead::Empty; + } + let mut predicate = String::new(); + if !node_names.is_empty() { + predicate = format!("node_name IN ({})", literal_list(node_names)); + } + if !filtered_node_types.is_empty() { + let identity_predicate: String = + format!("node_type NOT IN ({})", literal_list(filtered_node_types)); + predicate = if predicate.is_empty() { + identity_predicate + } else { + format!("{predicate} OR {identity_predicate}") + }; + } + RelevantRead::Sql(format!( + "SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE {predicate}" + )) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs new file mode 100644 index 0000000000..b38d2bda84 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs @@ -0,0 +1,24 @@ +//! In-memory node selection over unfiltered latest rows. + +use std::collections::HashSet; + +use crate::fingerprint_storage::models::FingerprintIdentity; + +/// Positions of the rows a narrowed latest read would have returned. +pub fn selected_positions( + identities: &[FingerprintIdentity<'_>], + node_names: &[String], + filtered_node_types: &[String], +) -> Vec { + let names: HashSet<&str> = node_names.iter().map(String::as_str).collect(); + let types: HashSet<&str> = filtered_node_types.iter().map(String::as_str).collect(); + identities + .iter() + .enumerate() + .filter(|(_, identity)| { + names.contains(identity.node_name) + || (!types.is_empty() && !types.contains(identity.node_type)) + }) + .map(|(position, _)| position) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs new file mode 100644 index 0000000000..35a2bdafe7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs @@ -0,0 +1,10 @@ +//! The append-only fingerprint table: its DDL, latest-row reads, batched inserts and row decoding. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs new file mode 100644 index 0000000000..fdd309019b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs @@ -0,0 +1,41 @@ +//! Fingerprint rows, read narrowing and latest-row indexes. + +/// One fingerprint row to insert, with its timestamp already in ISO format. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredFingerprint { + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: String, +} + +/// How a latest-fingerprint read is narrowed to the relevant nodes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RelevantRead { + /// Read every latest row with this SQL. + Sql(String), + /// Nothing can match; skip the read. + Empty, +} + +/// One latest fingerprint's identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FingerprintIdentity<'a> { + pub node_type: &'a str, + pub node_name: &'a str, +} + +/// Positions of the latest rows kept per node name and per identity, in Python dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LatestIndex { + pub by_name: Vec, + pub by_identity: Vec, +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs new file mode 100644 index 0000000000..2f1e84c032 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_storage.rs"] +mod storage; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs new file mode 100644 index 0000000000..9af85b3a35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs @@ -0,0 +1,19 @@ +use crate::fingerprint_storage::models::StoredFingerprint; + +/// A model fingerprint row with `definition` and fixed other fields. +pub(crate) fn stored_fingerprint(definition: String) -> StoredFingerprint { + StoredFingerprint { + node_type: "model".to_owned(), + node_name: "o'rders".to_owned(), + target_database: None, + target_schema: Some("main".to_owned()), + target_name: Some("orders".to_owned()), + run_id: "run".to_owned(), + definition_hash: "d".to_owned(), + version_hash: String::new(), + schema_fingerprint: "s".to_owned(), + definition, + metadata_json: "{}".to_owned(), + ts: "2026-01-01T00:00:00".to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs new file mode 100644 index 0000000000..59c72c1dd6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -0,0 +1,209 @@ +use crate::fingerprint_storage::errors::{FingerprintStorageError, StorageDecodeError}; +use crate::fingerprint_storage::main::decode_storage::decode_storage; +use crate::fingerprint_storage::main::insert_batch_lengths::insert_batch_lengths; +use crate::fingerprint_storage::main::insert_sql::insert_sql; +use crate::fingerprint_storage::main::latest_index::latest_index; +use crate::fingerprint_storage::models::{FingerprintIdentity, LatestIndex, StoredFingerprint}; +use crate::fingerprint_storage::tests::helpers::stored_fingerprint; +use crate::fingerprint_storage::tests::test_types::{ + BatchTestCase, DecodeTestCase, InsertTestCase, LatestIndexTestCase, +}; + +#[test] +fn given_stored_text_when_decoding_then_python_strict_base64_and_utf8_rules_apply() { + let test_cases = [ + DecodeTestCase { + description: "padded multi-byte text decodes", + storage: "4oKsQQ==", + expected_decoded: Ok("\u{20ac}A"), + }, + DecodeTestCase { + description: "empty storage is empty text", + storage: "", + expected_decoded: Ok(""), + }, + DecodeTestCase { + description: "leading padding is rejected", + storage: "=", + expected_decoded: Err(StorageDecodeError::Base64( + "Leading padding not allowed".to_owned(), + )), + }, + DecodeTestCase { + description: "a missing pad is incorrect padding", + storage: "QQ=", + expected_decoded: Err(StorageDecodeError::Base64("Incorrect padding".to_owned())), + }, + DecodeTestCase { + description: "one data character past a quad reports the count", + storage: "QUJDR", + expected_decoded: Err(StorageDecodeError::Base64( + "Invalid base64-encoded string: number of data characters (5) cannot be 1 more \ + than a multiple of 4" + .to_owned(), + )), + }, + DecodeTestCase { + description: "data after padding is rejected", + storage: "QQ=A", + expected_decoded: Err(StorageDecodeError::Base64( + "Discontinuous padding not allowed".to_owned(), + )), + }, + DecodeTestCase { + description: "a truncated sequence is an unexpected end", + storage: "4oI=", + expected_decoded: Err(StorageDecodeError::Utf8 { + bytes: vec![0xe2, 0x82], + start: 0, + end: 2, + reason: "unexpected end of data", + }), + }, + DecodeTestCase { + description: "a surrogate lead is an invalid continuation", + storage: "7aCA", + expected_decoded: Err(StorageDecodeError::Utf8 { + bytes: vec![0xed, 0xa0, 0x80], + start: 0, + end: 1, + reason: "invalid continuation byte", + }), + }, + ]; + for test_case in test_cases { + assert_eq!( + decode_storage(test_case.storage).as_deref(), + test_case.expected_decoded.as_ref().copied(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprints_when_batching_then_row_and_byte_limits_split_batches() { + let test_cases = [ + BatchTestCase { + description: "the row limit splits batches", + definition_lengths: &[1, 1, 1, 1, 1], + batch_size: 2, + batch_max_bytes: 1_000_000, + expected_lengths: Ok(&[2, 2, 1]), + }, + BatchTestCase { + description: "an oversized row still forms its own batch", + definition_lengths: &[10, 5_000, 10], + batch_size: 50, + batch_max_bytes: 1_000, + expected_lengths: Ok(&[1, 1, 1]), + }, + BatchTestCase { + description: "no fingerprints form no batches", + definition_lengths: &[], + batch_size: 50, + batch_max_bytes: 1_000, + expected_lengths: Ok(&[]), + }, + BatchTestCase { + description: "a zero batch size is rejected", + definition_lengths: &[1], + batch_size: 0, + batch_max_bytes: 1_000, + expected_lengths: Err(FingerprintStorageError::NonPositiveBatchSize), + }, + ]; + for test_case in test_cases { + let fingerprints: Vec = test_case + .definition_lengths + .iter() + .map(|length| stored_fingerprint("x".repeat(*length))) + .collect(); + assert_eq!( + insert_batch_lengths( + "main._sqlbuild_fingerprints", + &fingerprints, + test_case.batch_size, + test_case.batch_max_bytes, + ), + test_case.expected_lengths.map(<[usize]>::to_vec), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() { + let test_cases = [ + InsertTestCase { + description: "one row quotes names and encodes the definition", + definitions: &["ab"], + expected_sql: Ok( + "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ + target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ + definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + ), + }, + InsertTestCase { + description: "an empty batch is rejected", + definitions: &[], + expected_sql: Err(FingerprintStorageError::EmptyInsert), + }, + ]; + for test_case in test_cases { + let fingerprints: Vec = test_case + .definitions + .iter() + .map(|definition| stored_fingerprint((*definition).to_owned())) + .collect(); + assert_eq!( + insert_sql("main.t", &fingerprints), + test_case.expected_sql.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_name_collisions_when_indexing_latest_rows_then_models_win_and_order_is_kept() { + let test_cases = [ + LatestIndexTestCase { + description: "a later model replaces an earlier name; other types do not", + identities: &[ + ("udf", "orders"), + ("seed", "customers"), + ("model", "orders"), + ("hook", "orders"), + ("udf", "orders"), + ], + expected_index: LatestIndex { + by_name: vec![2, 1], + by_identity: vec![4, 1, 2, 3], + }, + }, + LatestIndexTestCase { + description: "no rows index nothing", + identities: &[], + expected_index: LatestIndex::default(), + }, + ]; + for test_case in test_cases { + let identities: Vec> = test_case + .identities + .iter() + .map(|(node_type, node_name)| FingerprintIdentity { + node_type, + node_name, + }) + .collect(); + assert_eq!( + latest_index(&identities), + test_case.expected_index, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs new file mode 100644 index 0000000000..048e5cf649 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::fingerprint_storage::errors::{FingerprintStorageError, StorageDecodeError}; +use crate::fingerprint_storage::models::LatestIndex; + +pub(crate) struct DecodeTestCase { + pub(crate) description: &'static str, + pub(crate) storage: &'static str, + pub(crate) expected_decoded: Result<&'static str, StorageDecodeError>, +} + +pub(crate) struct BatchTestCase { + pub(crate) description: &'static str, + pub(crate) definition_lengths: &'static [usize], + pub(crate) batch_size: i64, + pub(crate) batch_max_bytes: i64, + pub(crate) expected_lengths: Result<&'static [usize], FingerprintStorageError>, +} + +pub(crate) struct InsertTestCase { + pub(crate) description: &'static str, + pub(crate) definitions: &'static [&'static str], + pub(crate) expected_sql: Result<&'static str, FingerprintStorageError>, +} + +pub(crate) struct LatestIndexTestCase { + pub(crate) description: &'static str, + pub(crate) identities: &'static [(&'static str, &'static str)], + pub(crate) expected_index: LatestIndex, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index b474f08964..36186f5894 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -9,9 +9,14 @@ pub mod compiled_project; pub mod compiler; mod constants; pub mod contracts; +pub mod dag_artifact; pub mod explicit_references; +pub mod fingerprint_storage; pub mod graph; pub mod lineage; +pub mod lineage_graph; +pub mod manifest; +pub mod python_values; pub mod query_analysis; pub mod relation_names; pub mod required_columns; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/constants.rs b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs new file mode 100644 index 0000000000..8dad7393e9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs @@ -0,0 +1,26 @@ +//! Rich lineage dialect aliases. + +/// Aliases Python maps before trying the name as a Polyglot dialect value. +pub const ANALYSIS_DIALECT_ALIASES: [(&str, &str); 12] = [ + ("", "generic"), + ("ansi", "generic"), + ("generic", "generic"), + ("bigquery", "bigquery"), + ("snowflake", "snowflake"), + ("duckdb", "duckdb"), + ("motherduck", "duckdb"), + ("databricks", "databricks"), + ("postgres", "postgresql"), + ("tsql", "tsql"), + ("sqlserver", "tsql"), + ("postgresql", "postgresql"), +]; + +/// The branch before a node's last child. +pub const TREE_BRANCH_LAST: &str = "└── "; +/// The branch before every other child. +pub const TREE_BRANCH_MID: &str = "├── "; +/// The indent under a last child. +pub const TREE_CONTINUATION_LAST: &str = " "; +/// The indent under every other child. +pub const TREE_CONTINUATION_MID: &str = "│ "; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs new file mode 100644 index 0000000000..6cee9ed107 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs @@ -0,0 +1,14 @@ +//! The Polyglot analysis dialect for a stripped, lowercased adapter dialect name. + +use crate::lineage_graph::constants::ANALYSIS_DIALECT_ALIASES; +use crate::lineage_graph::models::AnalysisDialect; + +/// The aliased dialect value, or the name to look up as a dialect value. +pub fn analysis_dialect(normalized: &str) -> AnalysisDialect<'_> { + ANALYSIS_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == normalized) + .map_or(AnalysisDialect::Named(normalized), |(_, dialect)| { + AnalysisDialect::Alias(dialect) + }) +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs new file mode 100644 index 0000000000..8aff42a153 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs @@ -0,0 +1,30 @@ +//! Edge indexes for the project column lineage graph. + +use std::collections::HashMap; + +use crate::lineage_graph::models::{EdgeIndex, EdgeKey}; + +/// Index `edges` by target model and source resource; equal edges share their first position. +pub fn edge_index(edges: &[EdgeKey]) -> EdgeIndex { + let mut index = EdgeIndex { + first_equal: Vec::with_capacity(edges.len()), + ..EdgeIndex::default() + }; + let mut first_positions: HashMap<&EdgeKey, usize> = HashMap::with_capacity(edges.len()); + for (position, edge) in edges.iter().enumerate() { + index + .by_target_model + .entry(edge.target_model.clone()) + .or_default() + .push(position); + index + .by_source_resource + .entry(edge.source_name.clone()) + .or_default() + .push(position); + index + .first_equal + .push(*first_positions.entry(edge).or_insert(position)); + } + index +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs new file mode 100644 index 0000000000..2fd9d3193b --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs @@ -0,0 +1,4 @@ +pub mod analysis_dialect; +pub mod edge_index; +pub mod trace_limit_note; +pub mod tree_lines; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs new file mode 100644 index 0000000000..07a7d4af70 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs @@ -0,0 +1,13 @@ +//! The note shown when a column trace is truncated. + +/// The note's lines, before styling, when `total` columns exceed the `limit` shown. +pub fn trace_limit_note(total: i64, limit: i64) -> Vec { + if total <= limit { + return Vec::new(); + } + vec![ + format!("Showing {limit} of {total} columns."), + "Use --depth 1 to show direct column dependencies only.".to_owned(), + "Use --format json for the full trace.".to_owned(), + ] +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs new file mode 100644 index 0000000000..3e81952f41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs @@ -0,0 +1,63 @@ +//! Box-drawing tree traversal that expands each node once across the tree. + +use std::collections::HashSet; +use std::hash::Hash; + +use crate::lineage_graph::constants::{ + TREE_BRANCH_LAST, TREE_BRANCH_MID, TREE_CONTINUATION_LAST, TREE_CONTINUATION_MID, +}; +use crate::lineage_graph::models::TreeLine; +use crate::lineage_graph::types::TreeVisitor; + +/// Visit `children` depth first in order, emitting one line per node; repeats are not expanded. +pub fn tree_lines( + children: Vec, + seen: HashSet, + visitor: &mut impl TreeVisitor, +) -> Result<(), E> +where + K: Eq + Hash, +{ + let mut shown: HashSet = seen; + let mut pending: Vec<(N, String, bool)> = pending_children(children, ""); + while let Some((node, prefix, is_last)) = pending.pop() { + let key: K = visitor.key(&node)?; + let repeated: bool = shown.contains(&key); + visitor.line(TreeLine { + node: &node, + prefix: &prefix, + branch: if is_last { + TREE_BRANCH_LAST + } else { + TREE_BRANCH_MID + }, + repeated, + })?; + if repeated { + continue; + } + let _ = shown.insert(key); + let continuation: &str = if is_last { + TREE_CONTINUATION_LAST + } else { + TREE_CONTINUATION_MID + }; + let grandchildren: Vec = visitor.children(&node)?; + pending.extend(pending_children( + grandchildren, + &format!("{prefix}{continuation}"), + )); + } + Ok(()) +} + +fn pending_children(children: Vec, prefix: &str) -> Vec<(N, String, bool)> { + let last: usize = children.len().saturating_sub(1); + let mut pending: Vec<(N, String, bool)> = children + .into_iter() + .enumerate() + .map(|(index, child)| (child, prefix.to_owned(), index == last)) + .collect(); + pending.reverse(); + pending +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs new file mode 100644 index 0000000000..55f895632d --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs @@ -0,0 +1,9 @@ +//! The project column lineage graph: edge indexes and the rich analysis dialect. + +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/models.rs b/crates/sqlbuild-analysis/src/lineage_graph/models.rs new file mode 100644 index 0000000000..679d5c425e --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/models.rs @@ -0,0 +1,41 @@ +//! Column lineage graph edges and their indexes. + +use std::collections::HashMap; + +/// One collapsed edge's identity, every part as Python's `str()` renders it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EdgeKey { + pub source_type: String, + pub source_name: String, + pub source_column: String, + pub target_model: String, + pub target_column: String, + pub transform: String, + pub confidence: String, +} + +/// Edge positions by target model and source resource, and each edge's first equal edge. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EdgeIndex { + pub by_target_model: HashMap>, + pub by_source_resource: HashMap>, + pub first_equal: Vec, +} + +/// How an adapter dialect name maps onto a Polyglot analysis dialect. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnalysisDialect<'a> { + /// A known alias of this Polyglot dialect value. + Alias(&'static str), + /// Not an alias: the name itself must be a Polyglot dialect value. + Named(&'a str), +} + +/// One tree line: its node, indent, branch, and whether the node was already shown. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TreeLine<'a, N> { + pub node: &'a N, + pub prefix: &'a str, + pub branch: &'static str, + pub repeated: bool, +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests.rs new file mode 100644 index 0000000000..a9d1498b0f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_graph.rs"] +mod graph; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs new file mode 100644 index 0000000000..2aafbda202 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs @@ -0,0 +1,37 @@ +use std::convert::Infallible; + +use crate::lineage_graph::models::{EdgeKey, TreeLine}; +use crate::lineage_graph::tests::test_types::StringTree; +use crate::lineage_graph::types::TreeVisitor; + +/// A direct, high-confidence edge from `source.column` into `target.column`. +pub(crate) fn edge_key(source: (&str, &str), target: (&str, &str)) -> EdgeKey { + EdgeKey { + source_type: "model".to_owned(), + source_name: source.0.to_owned(), + source_column: source.1.to_owned(), + target_model: target.0.to_owned(), + target_column: target.1.to_owned(), + transform: "direct".to_owned(), + confidence: "high".to_owned(), + } +} + +impl TreeVisitor<&'static str, &'static str, Infallible> for StringTree { + fn key(&mut self, node: &&'static str) -> Result<&'static str, Infallible> { + Ok(node) + } + + fn line(&mut self, line: TreeLine<'_, &'static str>) -> Result<(), Infallible> { + let suffix: &str = if line.repeated { " (shown)" } else { "" }; + self.lines.push(format!( + "{}{}{}{suffix}", + line.prefix, line.branch, line.node + )); + Ok(()) + } + + fn children(&mut self, node: &&'static str) -> Result, Infallible> { + Ok(self.children.get(node).cloned().unwrap_or_default()) + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs new file mode 100644 index 0000000000..0000185abe --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs @@ -0,0 +1,127 @@ +use crate::lineage_graph::main::analysis_dialect::analysis_dialect; +use crate::lineage_graph::main::edge_index::edge_index; +use crate::lineage_graph::main::tree_lines::tree_lines; +use crate::lineage_graph::models::{AnalysisDialect, EdgeIndex, EdgeKey}; +use crate::lineage_graph::tests::helpers::edge_key; +use crate::lineage_graph::tests::test_types::{ + AnalysisDialectTestCase, EdgeIndexTestCase, StringTree, TreeLinesTestCase, +}; + +#[test] +fn given_edges_when_indexing_then_positions_keep_edge_order_and_equal_edges_share_one() { + let test_cases = [EdgeIndexTestCase { + description: "repeated and interleaved edges", + edges: &[ + (("orders", "id"), ("customers", "order_id")), + (("products", "id"), ("orders", "product_id")), + (("orders", "id"), ("customers", "order_id")), + (("orders", "total"), ("customers", "spend")), + ], + expected_targeting_orders: &[0, 2, 3], + expected_sourced_from_orders: &[0, 2, 3], + expected_first_equal: &[0, 1, 0, 3], + }]; + for test_case in test_cases { + let edges: Vec = test_case + .edges + .iter() + .map(|(source, target)| edge_key(*source, *target)) + .collect(); + let index: EdgeIndex = edge_index(&edges); + assert_eq!( + index.by_target_model.get("customers").map(Vec::as_slice), + Some(test_case.expected_targeting_orders), + "{}", + test_case.description + ); + assert_eq!( + index.by_source_resource.get("orders").map(Vec::as_slice), + Some(test_case.expected_sourced_from_orders), + "{}", + test_case.description + ); + assert_eq!( + index.first_equal.as_slice(), + test_case.expected_first_equal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dialect_names_when_resolving_then_aliases_map_and_others_stay_named() { + let test_cases = [ + AnalysisDialectTestCase { + description: "an empty name is generic", + normalized: "", + expected_dialect: AnalysisDialect::Alias("generic"), + }, + AnalysisDialectTestCase { + description: "MotherDuck analyzes as DuckDB", + normalized: "motherduck", + expected_dialect: AnalysisDialect::Alias("duckdb"), + }, + AnalysisDialectTestCase { + description: "SQL Server analyzes as T-SQL", + normalized: "sqlserver", + expected_dialect: AnalysisDialect::Alias("tsql"), + }, + AnalysisDialectTestCase { + description: "other names are looked up as given", + normalized: "mysql", + expected_dialect: AnalysisDialect::Named("mysql"), + }, + ]; + for test_case in test_cases { + assert_eq!( + analysis_dialect(test_case.normalized), + test_case.expected_dialect, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_graph_when_drawing_a_tree_then_each_node_expands_once() { + let test_cases = [TreeLinesTestCase { + description: "a diamond expands its shared node once and skips the focus", + children: &[ + ("orders", &["customers", "products"]), + ("customers", &["regions", "orders"]), + ("products", &["regions"]), + ("regions", &["countries"]), + ], + roots: &["customers", "products"], + seen: &["orders"], + expected_lines: &[ + "├── customers", + "│ ├── regions", + "│ │ └── countries", + "│ └── orders (shown)", + "└── products", + " └── regions (shown)", + ], + }]; + for test_case in test_cases { + let mut tree = StringTree { + children: test_case + .children + .iter() + .map(|(node, children)| (*node, children.to_vec())) + .collect(), + lines: Vec::new(), + }; + let Ok(()) = tree_lines( + test_case.roots.to_vec(), + test_case.seen.iter().copied().collect(), + &mut tree, + ); + assert_eq!( + tree.lines, test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs new file mode 100644 index 0000000000..91192c2d77 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs @@ -0,0 +1,34 @@ +use std::collections::HashMap; + +use crate::lineage_graph::models::AnalysisDialect; + +/// `(resource, column)`. +pub(crate) type ColumnRef = (&'static str, &'static str); + +pub(crate) struct EdgeIndexTestCase { + pub(crate) description: &'static str, + pub(crate) edges: &'static [(ColumnRef, ColumnRef)], + pub(crate) expected_targeting_orders: &'static [usize], + pub(crate) expected_sourced_from_orders: &'static [usize], + pub(crate) expected_first_equal: &'static [usize], +} + +pub(crate) struct AnalysisDialectTestCase { + pub(crate) description: &'static str, + pub(crate) normalized: &'static str, + pub(crate) expected_dialect: AnalysisDialect<'static>, +} + +pub(crate) struct TreeLinesTestCase { + pub(crate) description: &'static str, + pub(crate) children: &'static [(&'static str, &'static [&'static str])], + pub(crate) roots: &'static [&'static str], + pub(crate) seen: &'static [&'static str], + pub(crate) expected_lines: &'static [&'static str], +} + +/// A string tree whose lines render as `prefix + branch + node`, repeats marked `(shown)`. +pub(crate) struct StringTree { + pub(crate) children: HashMap<&'static str, Vec<&'static str>>, + pub(crate) lines: Vec, +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/types.rs b/crates/sqlbuild-analysis/src/lineage_graph/types.rs new file mode 100644 index 0000000000..c85a12a94c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/types.rs @@ -0,0 +1,13 @@ +//! Caller-supplied tree behaviour. + +use crate::lineage_graph::models::TreeLine; + +/// The caller's view of a tree: node identity, child order and line output. +pub trait TreeVisitor { + /// The identity a node is shown once by. + fn key(&mut self, node: &N) -> Result; + /// Record one line. + fn line(&mut self, line: TreeLine<'_, N>) -> Result<(), E>; + /// A node's children, in display order. + fn children(&mut self, node: &N) -> Result, E>; +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs new file mode 100644 index 0000000000..bd806d5b13 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs @@ -0,0 +1,134 @@ +//! dbt-compatible macro nodes of project macros and SQL hook definitions. + +use std::collections::HashMap; + +use crate::manifest::_helpers::values::{ + docs, empty_array, keyed, object, optional_text, text, text_or_empty, zero_float, +}; +use crate::manifest::constants::{RESOURCE_TYPE_MACRO, SQL_HOOK_PACKAGE_SUFFIX}; +use crate::manifest::models::{ + ManifestInput, ManifestMacroDeclarationInput, ManifestMacroInput, ManifestSqlHookInput, +}; +use crate::python_values::models::PythonValue; + +fn macro_node( + loaded: &ManifestMacroInput, + project_name: &str, + declaration: (Option<&Vec>, Option<&ManifestMacroDeclarationInput>), +) -> PythonValue { + let (dependencies, record) = declaration; + let path: &str = &loaded.relative_path; + object(vec![ + ("name", text(&loaded.name)), + ("resource_type", text(RESOURCE_TYPE_MACRO)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_MACRO}.{project_name}.{}", + loaded.name + )), + ), + ("macro_sql", text(&loaded.raw_source)), + ( + "depends_on", + object(vec![( + "macros", + PythonValue::Array( + dependencies + .map(Vec::as_slice) + .unwrap_or_default() + .iter() + .map(|name| text(&format!("{RESOURCE_TYPE_MACRO}.{project_name}.{name}"))) + .collect(), + ), + )]), + ), + ("description", text("")), + ( + "meta", + object(vec![ + ( + "sqlbuild_visibility", + text(record.map_or("global", |record| record.scope.as_str())), + ), + ( + "sqlbuild_scope_path", + optional_text(record.and_then(|record| record.owning_path.as_deref())), + ), + ]), + ), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("arguments", empty_array()), + ("created_at", zero_float()), + ("supported_languages", PythonValue::Null), + ]) +} + +fn sql_hook_node(hook: &ManifestSqlHookInput, project_name: &str) -> PythonValue { + let package: String = format!("{project_name}{SQL_HOOK_PACKAGE_SUFFIX}"); + object(vec![ + ("name", text(&hook.name)), + ("resource_type", text(RESOURCE_TYPE_MACRO)), + ("package_name", text(&package)), + ("path", text(&hook.relative_path)), + ("original_file_path", text(&hook.relative_path)), + ( + "unique_id", + text(&format!("{RESOURCE_TYPE_MACRO}.{package}.{}", hook.name)), + ), + ("macro_sql", text(&hook.sql_body)), + ("depends_on", object(vec![("macros", empty_array())])), + ("description", text_or_empty(hook.description.as_deref())), + ( + "meta", + object(vec![ + ("sqlbuild_resource_type", text("sql_hook")), + ("sqlbuild_hook_name", text(&hook.name)), + ]), + ), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("arguments", empty_array()), + ("created_at", zero_float()), + ("supported_languages", PythonValue::Null), + ]) +} + +/// Every macro node: project macros, then SQL hook definitions. +pub(crate) fn macro_nodes(input: &ManifestInput) -> PythonValue { + let project_name: &str = &input.project_name; + let mut dependencies: HashMap<&str, &Vec> = HashMap::new(); + let mut records: HashMap<&str, &ManifestMacroDeclarationInput> = HashMap::new(); + for record in &input.macro_declarations { + if let Some(names) = &record.dependencies { + dependencies.insert(&record.name, names); + } + records.insert(&record.name, record); + } + let mut nodes: Vec<(String, PythonValue)> = Vec::new(); + for loaded in &input.macros { + let name: &str = &loaded.key; + nodes.push(( + format!("{RESOURCE_TYPE_MACRO}.{project_name}.{name}"), + macro_node( + loaded, + project_name, + (dependencies.get(name).copied(), records.get(name).copied()), + ), + )); + } + for hook in &input.sql_hooks { + nodes.push(( + format!( + "{RESOURCE_TYPE_MACRO}.{project_name}{SQL_HOOK_PACKAGE_SUFFIX}.{}", + hook.name + ), + sql_hook_node(hook, project_name), + )); + } + keyed(nodes) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs new file mode 100644 index 0000000000..cf93695262 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod macros; +pub(crate) mod model_nodes; +pub(crate) mod node_fields; +pub(crate) mod payload; +pub(crate) mod resource_nodes; +pub(crate) mod test_nodes; +pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs new file mode 100644 index 0000000000..ec47ec1936 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs @@ -0,0 +1,176 @@ +//! The dbt-compatible model node. + +use crate::manifest::_helpers::node_fields::{ + columns, config, config_tags, depends_on, fqn, lifecycle_hooks, +}; +use crate::manifest::_helpers::values::{ + docs, empty_object, inserted, object, optional_text, text, text_or_empty, +}; +use crate::manifest::constants::{CHECKSUM_HASH_NAME, RESOURCE_TYPE_MODEL, SQLBUILD_META_KEY}; +use crate::manifest::models::{ManifestModelInput, ManifestPythonHookInput}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +/// The model node's inputs besides the model. +pub(crate) struct ModelNodeContext<'a> { + pub(crate) project_name: &'a str, + pub(crate) compiled_code: Option<&'a str>, + pub(crate) query_hash: &'a str, + pub(crate) python_hooks: &'a [ManifestPythonHookInput], +} + +fn meta_entry(meta: &[(PythonKey, PythonValue)], key: &str) -> PythonValue { + meta.iter() + .find(|(existing, _)| *existing == text_key(key)) + .map_or(PythonValue::Null, |(_, value)| value.clone()) +} + +/// The `str`-keyed entries of an authored `sqlbuild` dict, or its non-dict value as `authored_value`. +fn authored_sqlbuild(authored: PythonValue) -> Vec<(PythonKey, PythonValue)> { + match authored { + PythonValue::Object(entries) => entries + .into_iter() + .filter(|(key, _)| key.is_str) + .fold(Vec::new(), |fields, (key, value)| { + inserted(fields, key, value) + }), + PythonValue::Null => Vec::new(), + other => vec![(text_key("authored_value"), other)], + } +} + +fn with_sqlbuild_fact( + meta: Vec<(PythonKey, PythonValue)>, + (field, value): (&str, PythonValue), +) -> Vec<(PythonKey, PythonValue)> { + let fields: Vec<(PythonKey, PythonValue)> = inserted( + authored_sqlbuild(meta_entry(&meta, SQLBUILD_META_KEY)), + text_key(field), + value, + ); + inserted( + meta, + text_key(SQLBUILD_META_KEY), + PythonValue::Object(fields), + ) +} + +fn meta(model: &ManifestModelInput, python_hooks: &[ManifestPythonHookInput]) -> PythonValue { + let mut meta: Vec<(PythonKey, PythonValue)> = Vec::new(); + if let Some(PythonValue::Object(authored)) = + model.schema.as_ref().and_then(|schema| schema.meta.clone()) + { + for (key, value) in authored { + meta = inserted(meta, key, value); + } + } + let mut hooks: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, entries) in [ + ("pre_hooks", model.config.pre_hooks.as_deref()), + ("post_hooks", model.config.post_hooks.as_deref()), + ] { + let serialized: Vec = lifecycle_hooks(entries, python_hooks); + if !serialized.is_empty() { + hooks.push((text_key(key), PythonValue::Array(serialized))); + } + } + if !hooks.is_empty() { + meta = with_sqlbuild_fact(meta, ("lifecycle_hooks", PythonValue::Object(hooks))); + } + let families = model + .schema + .as_ref() + .map(|schema| schema.dynamic_columns.as_slice()) + .unwrap_or_default(); + if !families.is_empty() { + let families: Vec = families + .iter() + .map(|family| { + object(vec![ + ("name", family.name.clone()), + ("pivot_column", family.pivot_column.clone()), + ("value_column", family.value_column.clone()), + ("aggregate", family.aggregate.clone()), + ("type", family.data_type.clone()), + ("name_pattern", family.name_pattern.clone()), + ]) + }) + .collect(); + meta = with_sqlbuild_fact(meta, ("dynamic_columns", PythonValue::Array(families))); + } + PythonValue::Object(meta) +} + +/// One dbt-compatible model node. +pub(crate) fn model_node( + model: &ManifestModelInput, + context: &ModelNodeContext<'_>, +) -> PythonValue { + let project_name: &str = context.project_name; + let path: &str = &model.path.text; + object(vec![ + ( + "database", + optional_text(model.destination.database.as_deref()), + ), + ("schema", optional_text(model.destination.schema.as_deref())), + ("name", text(&model.name)), + ("resource_type", text(RESOURCE_TYPE_MODEL)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_MODEL}.{project_name}.{}", + model.name + )), + ), + ("fqn", fqn(project_name, &model.path)), + ("alias", optional_text(model.destination.name.as_deref())), + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text(context.query_hash)), + ]), + ), + ("config", config(&model.config)), + ("tags", config_tags(&model.config)), + ( + "description", + text_or_empty( + model + .schema + .as_ref() + .and_then(|schema| schema.description.as_deref()), + ), + ), + ( + "columns", + model + .schema + .as_ref() + .map_or_else(empty_object, |schema| columns(&schema.columns)), + ), + ("meta", meta(model, context.python_hooks)), + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", PythonValue::Float(0.0)), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ( + "relation_name", + optional_text(model.destination.qualified_name.as_deref()), + ), + ("raw_code", text(&model.query_sql)), + ( + "compiled_code", + text(context.compiled_code.unwrap_or(&model.query_sql)), + ), + ("depends_on", depends_on(&model.deps, project_name)), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs new file mode 100644 index 0000000000..8c9aed05f2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs @@ -0,0 +1,201 @@ +//! Shared manifest node pieces: fqn, columns, config, hooks and dependency ids. + +use crate::assembly::project::types::ObjectKey; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, keyed, object, optional_text, text, text_or_empty, texts, +}; +use crate::manifest::constants::{DEFAULT_RESOURCE_PREFIX, RESOURCE_TYPE_PREFIXES}; +use crate::manifest::models::{ + ManifestColumnInput, ManifestConfigInput, ManifestHookInput, ManifestPathInput, + ManifestPythonHookInput, +}; +use crate::python_values::models::PythonValue; + +/// `[project, *relative_path.parent.parts, relative_path.stem]`. +pub(crate) fn fqn(project_name: &str, path: &ManifestPathInput) -> PythonValue { + texts( + std::iter::once(project_name) + .chain(path.parent_parts.iter().map(String::as_str)) + .chain(std::iter::once(path.stem.as_str())), + ) +} + +/// The dbt-style columns dict, a later column replacing an earlier one of the same name. +pub(crate) fn columns(columns: &[ManifestColumnInput]) -> PythonValue { + keyed( + columns + .iter() + .map(|column| { + ( + column.name.clone(), + object(vec![ + ("name", text(&column.name)), + ("description", text_or_empty(column.description.as_deref())), + ("meta", column.meta.clone().unwrap_or_else(empty_object)), + ("data_type", optional_text(column.data_type.as_deref())), + ("constraints", empty_array()), + ("quote", PythonValue::Null), + ("tags", empty_array()), + ]), + ) + }) + .collect(), + ) +} + +/// `prefix.project.name` of a compiled key. +pub(crate) fn unique_id((resource_type, name): &ObjectKey, project_name: &str) -> String { + let prefix: &str = RESOURCE_TYPE_PREFIXES + .iter() + .find(|(kind, _)| kind == resource_type) + .map_or(DEFAULT_RESOURCE_PREFIX, |(_, prefix)| prefix); + format!("{prefix}.{project_name}.{name}") +} + +/// The unique ids of `keys`. +pub(crate) fn unique_ids(keys: &[ObjectKey], project_name: &str) -> PythonValue { + PythonValue::Array( + keys.iter() + .map(|key| text(&unique_id(key, project_name))) + .collect(), + ) +} + +fn dbt_sql_hooks(hooks: Option<&[ManifestHookInput]>) -> PythonValue { + let statements = hooks + .unwrap_or_default() + .iter() + .filter_map(|hook| match hook { + ManifestHookInput::Sql { statement, .. } => Some(statement), + _ => None, + }); + PythonValue::Array( + statements + .enumerate() + .map(|(index, statement)| { + object(vec![ + ("sql", text(statement)), + ("transaction", PythonValue::Bool(true)), + ( + "index", + PythonValue::Integer(sqlbuild_core::json::models::JsonInteger::from( + i64::try_from(index).unwrap_or(i64::MAX), + )), + ), + ]) + }) + .collect(), + ) +} + +/// The config tags, or an empty list. +pub(crate) fn config_tags(config: &ManifestConfigInput) -> PythonValue { + texts(config.tags.iter().flatten().map(String::as_str)) +} + +/// The dbt-compatible config dict of a model. +pub(crate) fn config(config: &ManifestConfigInput) -> PythonValue { + object(vec![ + ("enabled", config.enabled.clone()), + ("alias", config.alias.clone()), + ("schema", config.schema.clone()), + ("database", config.database.clone()), + ("tags", config_tags(config)), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text(&config.materialized)), + ("incremental_strategy", config.incremental_strategy.clone()), + ( + "merge_exclude_columns", + config.merge_exclude_columns.clone(), + ), + ("persist_docs", empty_object()), + ("post-hook", dbt_sql_hooks(config.post_hooks.as_deref())), + ("pre-hook", dbt_sql_hooks(config.pre_hooks.as_deref())), + ("quoting", empty_object()), + ("column_types", empty_object()), + ("full_refresh", config.full_refresh.clone()), + ("unique_key", config.unique_key.clone()), + ("on_schema_change", config.on_schema_change.clone()), + ("on_configuration_change", text("apply")), + ("grants", empty_object()), + ("packages", empty_array()), + ("docs", docs()), + ( + "contract", + object(vec![ + ("enforced", PythonValue::Bool(false)), + ("alias_types", PythonValue::Bool(true)), + ]), + ), + ]) +} + +fn optional_field(name: &str, value: Option) -> Option<(String, PythonValue)> { + value.map(|value| (name.to_owned(), value)) +} + +/// `serialize_hook_entries` with every SQL hook output field and the Python hook identities. +pub(crate) fn lifecycle_hooks( + hooks: Option<&[ManifestHookInput]>, + python_hooks: &[ManifestPythonHookInput], +) -> Vec { + let mut serialized: Vec = Vec::new(); + for hook in hooks.unwrap_or_default() { + match hook { + ManifestHookInput::Sql { + statement, + name, + relative_path, + definition_sql, + kwargs, + description, + } => { + let fields: Vec<(String, PythonValue)> = [ + Some(("type".to_owned(), text("sql"))), + Some(("statement".to_owned(), text(statement))), + optional_field("name", name.as_deref().map(text)), + optional_field("relative_path", relative_path.as_deref().map(text)), + optional_field("definition_sql", definition_sql.as_deref().map(text)), + optional_field( + "kwargs", + (*kwargs != PythonValue::Null).then(|| kwargs.clone()), + ), + optional_field("description", description.as_deref().map(text)), + ] + .into_iter() + .flatten() + .collect(); + serialized.push(keyed(fields)); + } + ManifestHookInput::Python { name, kwargs } => { + let mut fields: Vec<(String, PythonValue)> = vec![ + ("type".to_owned(), text("python")), + ("name".to_owned(), text(name)), + ("kwargs".to_owned(), kwargs.clone()), + ]; + if let Some(identity) = python_hooks.iter().rev().find(|hook| hook.name == *name) { + fields.extend([ + ("relative_path".to_owned(), text(&identity.relative_path)), + ( + "definition_hash".to_owned(), + text(&identity.definition_hash), + ), + ("version_hash".to_owned(), text(&identity.version_hash)), + ]); + } + serialized.push(keyed(fields)); + } + ManifestHookInput::Other => {} + } + } + serialized +} + +/// `{"macros": [], "nodes": [...]}`. +pub(crate) fn depends_on(deps: &[ObjectKey], project_name: &str) -> PythonValue { + object(vec![ + ("macros", empty_array()), + ("nodes", unique_ids(deps, project_name)), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs new file mode 100644 index 0000000000..32087c8f0e --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs @@ -0,0 +1,156 @@ +//! The complete manifest payload. + +use std::collections::{HashMap, HashSet}; + +use crate::graph::main::unique_id_edges::unique_id_edges; +use crate::graph::models::{EdgeDirection, ProjectGraph}; +use crate::manifest::_helpers::macros::macro_nodes; +use crate::manifest::_helpers::model_nodes::{ModelNodeContext, model_node}; +use crate::manifest::_helpers::resource_nodes::{seed_node, source_node}; +use crate::manifest::_helpers::test_nodes::{audit_test_node, sql_test_node}; +use crate::manifest::_helpers::values::{empty_object, keyed, object, text}; +use crate::manifest::constants::{ + DBT_MANIFEST_SCHEMA_VERSION, DBT_VERSION, RESOURCE_TYPE_MODEL, RESOURCE_TYPE_PREFIXES, + RESOURCE_TYPE_SEED, RESOURCE_TYPE_SOURCE, +}; +use crate::manifest::errors::ManifestError; +use crate::manifest::models::{ManifestInput, ManifestPlanInput}; +use crate::python_values::models::PythonValue; +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// Each planned model's resolved SQL; a later entry of the same name wins. +fn resolved_sql(plan: &ManifestPlanInput) -> HashMap<&str, &str> { + plan.models + .iter() + .map(|(name, sql)| (name.as_str(), sql.as_str())) + .collect() +} + +fn id_list(ids: &[String]) -> PythonValue { + PythonValue::Array(ids.iter().map(|id| text(id)).collect()) +} + +fn nodes(input: &ManifestInput) -> Result { + let project_name: &str = &input.project_name; + let plan: Option<&ManifestPlanInput> = input.plan.as_ref(); + let resolved: HashMap<&str, &str> = plan.map(resolved_sql).unwrap_or_default(); + let mut nodes: Vec<(String, PythonValue)> = Vec::new(); + for model in &input.models { + let query_hash: String = + compiler_query_fingerprint(&model.query_sql, input.sql_analysis_dialect.as_deref()) + .map_err(ManifestError::QueryFingerprint)?; + nodes.push(( + format!("{RESOURCE_TYPE_MODEL}.{project_name}.{}", model.name), + model_node( + model, + &ModelNodeContext { + project_name, + compiled_code: resolved.get(model.name.as_str()).copied(), + query_hash: &query_hash, + python_hooks: &input.python_hooks, + }, + ), + )); + } + for seed in &input.seeds { + nodes.push(( + format!("{RESOURCE_TYPE_SEED}.{project_name}.{}", seed.name), + seed_node(seed, project_name), + )); + } + if let Some(plan) = plan { + nodes.extend( + plan.audits + .iter() + .map(|audit| audit_test_node(audit, project_name)), + ); + for test in &plan.sql_tests { + nodes.push(sql_test_node(test, project_name)?); + } + } + Ok(keyed(nodes)) +} + +fn sources(input: &ManifestInput) -> PythonValue { + let project_name: &str = &input.project_name; + let loaded: HashSet<&str> = input + .plan + .as_ref() + .map(|plan| plan.source_load_names.iter().map(String::as_str).collect()) + .unwrap_or_default(); + keyed( + input + .sources + .iter() + .map(|source| { + ( + format!("{RESOURCE_TYPE_SOURCE}.{project_name}.{}", source.name), + source_node(source, project_name, loaded.contains(source.name.as_str())), + ) + }) + .collect(), + ) +} + +fn id_map(graph: &ProjectGraph, direction: EdgeDirection, project_name: &str) -> PythonValue { + let prefixes: HashMap = RESOURCE_TYPE_PREFIXES + .iter() + .map(|(kind, prefix)| ((*kind).to_owned(), (*prefix).to_owned())) + .collect(); + keyed( + unique_id_edges(graph, direction, project_name, &prefixes) + .into_iter() + .map(|(id, neighbours)| (id, id_list(&neighbours))) + .collect(), + ) +} + +fn metadata(input: &ManifestInput) -> PythonValue { + object(vec![ + ("dbt_schema_version", text(DBT_MANIFEST_SCHEMA_VERSION)), + ("dbt_version", text(DBT_VERSION)), + ("generated_at", text(&input.generated_at)), + ("invocation_id", text(&input.run_id)), + ("env", empty_object()), + ("project_name", text(&input.project_name)), + ("project_id", PythonValue::Null), + ("user_id", PythonValue::Null), + ("send_anonymous_usage_stats", PythonValue::Bool(false)), + ("adapter_type", text(&input.adapter_type)), + ]) +} + +/// The manifest dict `build_manifest_payload` returned. +pub(crate) fn payload( + input: &ManifestInput, + graph: &ProjectGraph, +) -> Result { + let macros: PythonValue = macro_nodes(input); + let nodes: PythonValue = nodes(input)?; + let sources: PythonValue = sources(input); + let project_name: &str = &input.project_name; + Ok(object(vec![ + ("metadata", metadata(input)), + ("nodes", nodes), + ("sources", sources), + ("macros", macros), + ("exposures", empty_object()), + ("metrics", empty_object()), + ("groups", empty_object()), + ("selectors", empty_object()), + ("disabled", empty_object()), + ( + "parent_map", + id_map(graph, EdgeDirection::Upstream, project_name), + ), + ( + "child_map", + id_map(graph, EdgeDirection::Downstream, project_name), + ), + ("group_map", empty_object()), + ("docs", empty_object()), + ("saved_queries", empty_object()), + ("semantic_models", empty_object()), + ("unit_tests", empty_object()), + ])) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs new file mode 100644 index 0000000000..cf82fa841e --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs @@ -0,0 +1,172 @@ +//! The dbt-compatible source and seed nodes. + +use crate::manifest::_helpers::node_fields::{columns, fqn}; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, inserted, object, optional_text, text, text_or_empty, texts, + zero_float, +}; +use crate::manifest::constants::{ + CHECKSUM_HASH_NAME, RESOURCE_TYPE_SEED, RESOURCE_TYPE_SOURCE, SQLBUILD_META_KEY, +}; +use crate::manifest::models::{ManifestSeedInput, ManifestSourceInput}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +fn object_entries(value: Option) -> Vec<(PythonKey, PythonValue)> { + match value { + Some(PythonValue::Object(entries)) => entries, + _ => Vec::new(), + } +} + +fn source_meta(source: &ManifestSourceInput, auto_load: bool) -> PythonValue { + let meta: Vec<(PythonKey, PythonValue)> = object_entries(source.meta.clone()); + let Some(loader) = &source.loader else { + return PythonValue::Object(meta); + }; + let mut fields: Vec<(&str, PythonValue)> = vec![ + ("loader", text(loader)), + ("auto_load", PythonValue::Bool(auto_load)), + ]; + if let Some(strategy) = &source.write_strategy { + fields.push(("write_strategy", text(strategy))); + } + if let Some(cursor) = &source.cursor_column { + fields.push(("cursor_column", cursor.clone())); + } + if let Some(unique_key) = &source.unique_key { + fields.push(("unique_key", unique_key.clone())); + } + PythonValue::Object(inserted(meta, text_key(SQLBUILD_META_KEY), object(fields))) +} + +/// One dbt-compatible source node. +pub(crate) fn source_node( + source: &ManifestSourceInput, + project_name: &str, + auto_load: bool, +) -> PythonValue { + let name: &str = &source.entry_name; + object(vec![ + ("database", optional_text(source.database.as_deref())), + ("schema", text_or_empty(source.schema.as_deref())), + ("name", text(name)), + ("resource_type", text(RESOURCE_TYPE_SOURCE)), + ("package_name", text(project_name)), + ("path", text(&source.path)), + ("original_file_path", text(&source.path)), + ( + "unique_id", + text(&format!("{RESOURCE_TYPE_SOURCE}.{project_name}.{name}")), + ), + ("fqn", texts([project_name, name])), + ("source_name", text(name)), + ("source_description", text("")), + ("loader", text("")), + ("identifier", text(source.table.as_deref().unwrap_or(name))), + ("quoting", empty_object()), + ("loaded_at_field", PythonValue::Null), + ("freshness", PythonValue::Null), + ("external", PythonValue::Null), + ("description", text_or_empty(source.description.as_deref())), + ("columns", columns(&source.columns)), + ("meta", source_meta(source, auto_load)), + ("source_meta", empty_object()), + ("tags", empty_array()), + ("config", empty_object()), + ("patch_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("relation_name", PythonValue::Null), + ("created_at", zero_float()), + ]) +} + +fn seed_config(seed: &ManifestSeedInput) -> PythonValue { + object(vec![ + ("enabled", PythonValue::Bool(true)), + ("alias", PythonValue::Null), + ("schema", optional_text(seed.destination.schema.as_deref())), + ( + "database", + optional_text(seed.destination.database.as_deref()), + ), + ("tags", texts(seed.tags.iter().map(String::as_str))), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text("seed")), + ("persist_docs", empty_object()), + ("post-hook", empty_array()), + ("pre-hook", empty_array()), + ("quoting", empty_object()), + ("column_types", empty_object()), + ("full_refresh", PythonValue::Null), + ("unique_key", PythonValue::Null), + ("on_schema_change", text("ignore")), + ("on_configuration_change", text("apply")), + ("grants", empty_object()), + ("packages", empty_array()), + ("docs", docs()), + ( + "contract", + object(vec![ + ("enforced", PythonValue::Bool(false)), + ("alias_types", PythonValue::Bool(true)), + ]), + ), + ("delimiter", text(",")), + ("quote_columns", PythonValue::Null), + ]) +} + +/// One dbt-compatible seed node. +pub(crate) fn seed_node(seed: &ManifestSeedInput, project_name: &str) -> PythonValue { + let path: &str = &seed.path.text; + object(vec![ + ( + "database", + optional_text(seed.destination.database.as_deref()), + ), + ("schema", optional_text(seed.destination.schema.as_deref())), + ("name", text(&seed.name)), + ("resource_type", text(RESOURCE_TYPE_SEED)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_SEED}.{project_name}.{}", + seed.name + )), + ), + ("fqn", fqn(project_name, &seed.path)), + ("alias", optional_text(seed.destination.name.as_deref())), + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text("")), + ]), + ), + ("config", seed_config(seed)), + ("tags", texts(seed.tags.iter().map(String::as_str))), + ("description", text_or_empty(seed.description.as_deref())), + ("columns", columns(&seed.columns)), + ("meta", seed.meta.clone().unwrap_or_else(empty_object)), + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", zero_float()), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ( + "relation_name", + optional_text(seed.destination.qualified_name.as_deref()), + ), + ("raw_code", text("")), + ("root_path", PythonValue::Null), + ("depends_on", object(vec![("macros", empty_array())])), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs new file mode 100644 index 0000000000..9dea3f2dea --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs @@ -0,0 +1,222 @@ +//! dbt-compatible generic and singular test nodes of planned audits and SQL tests. + +use std::collections::HashMap; + +use crate::manifest::_helpers::node_fields::{unique_id, unique_ids}; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, keyed, object, optional_text, text, texts, zero_float, +}; +use crate::manifest::constants::{CHECKSUM_HASH_NAME, RESOURCE_TYPE_TEST}; +use crate::manifest::errors::ManifestError; +use crate::manifest::models::{ManifestAuditPlanInput, ManifestSqlTestPlanInput}; +use crate::python_values::models::PythonValue; + +fn test_config() -> PythonValue { + object(vec![ + ("enabled", PythonValue::Bool(true)), + ("alias", PythonValue::Null), + ("schema", PythonValue::Null), + ("database", PythonValue::Null), + ("tags", empty_array()), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text("test")), + ("severity", text("ERROR")), + ("store_failures", PythonValue::Null), + ("store_failures_as", PythonValue::Null), + ("where", PythonValue::Null), + ("limit", PythonValue::Null), + ("fail_calc", text("count(*)")), + ("warn_if", text("!= 0")), + ("error_if", text("!= 0")), + ]) +} + +/// The test node fields shared by audits and SQL tests, after `fqn` and before `checksum`. +struct TestNodeHead<'a> { + unique_id: &'a str, + name: &'a str, + project_name: &'a str, +} + +fn head(node: &TestNodeHead<'_>) -> Vec<(&'static str, PythonValue)> { + vec![ + ("database", PythonValue::Null), + ("schema", text("")), + ("name", text(node.name)), + ("resource_type", text(RESOURCE_TYPE_TEST)), + ("package_name", text(node.project_name)), + ("path", text("")), + ("original_file_path", text("")), + ("unique_id", text(node.unique_id)), + ("fqn", texts([node.project_name, node.name])), + ("alias", text(node.name)), + ] +} + +fn tail(compiled_code: &str, depends_on: PythonValue) -> Vec<(&'static str, PythonValue)> { + vec![ + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", zero_float()), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ("relation_name", PythonValue::Null), + ("raw_code", text(compiled_code)), + ("compiled_code", text(compiled_code)), + ( + "depends_on", + object(vec![("macros", empty_array()), ("nodes", depends_on)]), + ), + ] +} + +/// The generic test node of one planned audit, keyed by its unique id. +pub(crate) fn audit_test_node( + audit: &ManifestAuditPlanInput, + project_name: &str, +) -> (String, PythonValue) { + let id: String = format!("{RESOURCE_TYPE_TEST}.{project_name}.{}", audit.name); + let mut kwargs: Vec<(String, PythonValue)> = Vec::new(); + if let Some(target) = &audit.attached_target_name { + kwargs.push(("model".to_owned(), text(target))); + } + if let Some(column) = &audit.attached_column_name { + kwargs.push(("column_name".to_owned(), text(column))); + } + let mut fields: Vec<(&str, PythonValue)> = head(&TestNodeHead { + unique_id: &id, + name: &audit.name, + project_name, + }); + fields.extend([ + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text("")), + ]), + ), + ("config", test_config()), + ("tags", empty_array()), + ("description", text("")), + ("columns", empty_object()), + ("meta", object(vec![("sqlbuild_test_type", text("audit"))])), + ]); + fields.extend(tail( + &audit.resolved_sql, + unique_ids(&audit.scope_deps, project_name), + )); + fields.extend([ + ( + "test_metadata", + object(vec![("name", text(&audit.name)), ("kwargs", keyed(kwargs))]), + ), + ( + "column_name", + optional_text(audit.attached_column_name.as_deref()), + ), + ]); + (id, object(fields)) +} + +fn sql_test_meta(test: &ManifestSqlTestPlanInput) -> Result { + let mut fields: Vec<(&str, PythonValue)> = vec![("sqlbuild_test_type", text("sql_native"))]; + let (Some(case_name), Some(source_path)) = (&test.case_name, &test.source_path) else { + return Ok(object(fields)); + }; + let types: HashMap<&str, &str> = test + .parameter_schema + .iter() + .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) + .collect(); + let parameters: Vec = test + .parameters + .iter() + .map(|(name, value)| { + let value_type: &str = types + .get(name.as_str()) + .ok_or_else(|| ManifestError::MissingParameterType(name.clone()))?; + Ok(object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("value", value.clone()), + ])) + }) + .collect::>()?; + fields.extend([ + ("sqlbuild_source_path", text(source_path)), + ("sqlbuild_block_index", test.block_index.clone()), + ("sqlbuild_parent_name", test.parent_name.clone()), + ("sqlbuild_case_name", text(case_name)), + ("sqlbuild_case_index", test.case_index.clone()), + ( + "sqlbuild_case_fingerprint", + optional_text(test.case_fingerprint.as_deref()), + ), + ( + "sqlbuild_parameter_schema", + PythonValue::Array( + test.parameter_schema + .iter() + .map(|(name, value_type, nullable)| { + object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("nullable", nullable.clone()), + ]) + }) + .collect(), + ), + ), + ("sqlbuild_parameters", PythonValue::Array(parameters)), + ]); + Ok(object(fields)) +} + +/// The singular test node of one planned SQL test, keyed by its unique id. +pub(crate) fn sql_test_node( + test: &ManifestSqlTestPlanInput, + project_name: &str, +) -> Result<(String, PythonValue), ManifestError> { + let id: String = unique_id(&test.key, project_name); + let mut parts: Vec = Vec::new(); + for (model_name, resolved_sql) in &test.chain { + if resolved_sql.is_empty() { + continue; + } + parts.push(format!("-- step: {model_name}")); + parts.push(resolved_sql.clone()); + } + let compiled_code: String = parts.join("\n\n"); + let mut fields: Vec<(&str, PythonValue)> = head(&TestNodeHead { + unique_id: &id, + name: &test.name, + project_name, + }); + fields.extend([ + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ( + "checksum", + text(test.case_fingerprint.as_deref().unwrap_or_default()), + ), + ]), + ), + ("config", test_config()), + ("tags", empty_array()), + ("description", text("")), + ("columns", empty_object()), + ("meta", sql_test_meta(test)?), + ]); + fields.extend(tail( + &compiled_code, + unique_ids(&test.scope_deps, project_name), + )); + Ok((id, object(fields))) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs new file mode 100644 index 0000000000..0c2575f9f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs @@ -0,0 +1,79 @@ +//! Building manifest values as Python dicts, lists and strings. + +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +/// A dict of field names, later duplicate keys replacing earlier values in place. +pub(crate) fn object(entries: Vec<(&str, PythonValue)>) -> PythonValue { + let mut fields: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, value) in entries { + fields = inserted(fields, text_key(key), value); + } + PythonValue::Object(fields) +} + +/// `d[key] = value`: replace the value in place, or append the entry. +pub(crate) fn inserted( + entries: Vec<(PythonKey, PythonValue)>, + key: PythonKey, + value: PythonValue, +) -> Vec<(PythonKey, PythonValue)> { + let mut entries: Vec<(PythonKey, PythonValue)> = entries; + match entries.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => entries.push((key, value)), + } + entries +} + +/// A dict of `str` keys built entry by entry, as a dict comprehension or `update` builds it. +pub(crate) fn keyed(entries: Vec<(String, PythonValue)>) -> PythonValue { + let mut fields: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, value) in entries { + fields = inserted(fields, text_key(&key), value); + } + PythonValue::Object(fields) +} + +/// A string. +pub(crate) fn text(value: &str) -> PythonValue { + PythonValue::String(value.to_owned()) +} + +/// A string or `None`. +pub(crate) fn optional_text(value: Option<&str>) -> PythonValue { + value.map_or(PythonValue::Null, text) +} + +/// `value or ""`. +pub(crate) fn text_or_empty(value: Option<&str>) -> PythonValue { + text(value.unwrap_or_default()) +} + +/// A list of strings. +pub(crate) fn texts<'a>(values: impl IntoIterator) -> PythonValue { + PythonValue::Array(values.into_iter().map(text).collect()) +} + +/// An empty dict. +pub(crate) fn empty_object() -> PythonValue { + PythonValue::Object(Vec::new()) +} + +/// An empty list. +pub(crate) fn empty_array() -> PythonValue { + PythonValue::Array(Vec::new()) +} + +/// `0.0`. +pub(crate) fn zero_float() -> PythonValue { + PythonValue::Float(0.0) +} + +/// `{"show": True, "node_color": None}`. +pub(crate) fn docs() -> PythonValue { + object(vec![ + ("show", PythonValue::Bool(true)), + ("node_color", PythonValue::Null), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/constants.rs b/crates/sqlbuild-analysis/src/manifest/constants.rs new file mode 100644 index 0000000000..e472e9c44d --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/constants.rs @@ -0,0 +1,27 @@ +//! dbt manifest v12 constants. + +pub(crate) const DBT_MANIFEST_SCHEMA_VERSION: &str = + "https://schemas.getdbt.com/dbt/manifest/v12.json"; +pub(crate) const DBT_VERSION: &str = "1.12.0"; +pub(crate) const RESOURCE_TYPE_MODEL: &str = "model"; +pub(crate) const RESOURCE_TYPE_SOURCE: &str = "source"; +pub(crate) const RESOURCE_TYPE_SEED: &str = "seed"; +pub(crate) const RESOURCE_TYPE_TEST: &str = "test"; +pub(crate) const RESOURCE_TYPE_MACRO: &str = "macro"; +pub(crate) const CHECKSUM_HASH_NAME: &str = "sha256"; +/// Compiled resource type to its dbt unique id prefix; any other type uses `model`. +pub(crate) const RESOURCE_TYPE_PREFIXES: [(&str, &str); 6] = [ + ("model", "model"), + ("source", "source"), + ("seed", "seed"), + ("dbt_ref", "model"), + ("audit", "test"), + ("sql_test", "test"), +]; +pub(crate) const DEFAULT_RESOURCE_PREFIX: &str = "model"; +/// The package suffix SQL hook macros are published under. +pub(crate) const SQL_HOOK_PACKAGE_SUFFIX: &str = "__sqlbuild_hooks"; +/// The meta key SQLBuild-owned facts are nested under. +pub(crate) const SQLBUILD_META_KEY: &str = "sqlbuild"; +/// `json.dumps(..., indent=2)`. +pub(crate) const MANIFEST_INDENT: usize = 2; diff --git a/crates/sqlbuild-analysis/src/manifest/errors.rs b/crates/sqlbuild-analysis/src/manifest/errors.rs new file mode 100644 index 0000000000..7067fc17d1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/errors.rs @@ -0,0 +1,21 @@ +//! Why the manifest could not be built or encoded, named after the Python error each reproduces. + +use sqlbuild_core::json::errors::JsonEmitError; + +/// A failure while building the manifest, raised where Python's builder raised it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ManifestError { + /// `QueryFingerprintError` of a model query that cannot be fingerprinted. + QueryFingerprint(String), + /// `KeyError` for a bound SQL test parameter missing from the parameter schema. + MissingParameterType(String), +} + +/// A failure encoding the built manifest, raised where `json.dumps` raised it: on writing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ManifestEncodeError { + /// `TypeError` for the first value `json.dumps` cannot encode. + Unencodable(String), + /// The emitter refused the encoded manifest. + Emit(JsonEmitError), +} diff --git a/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs b/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs new file mode 100644 index 0000000000..e4648fdb49 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs @@ -0,0 +1,29 @@ +//! The manifest bytes `json.dumps(manifest, indent=2) + "\n"` wrote, or the error encoding raised. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::graph::models::ProjectGraph; +use crate::manifest::_helpers::payload::payload; +use crate::manifest::constants::MANIFEST_INDENT; +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::models::ManifestInput; +use crate::python_values::main::encode_python_value::encode_python_value; + +/// Build the manifest; its encoding outcome is kept for the writer, where `json.dumps` raised. +pub fn build_manifest_json( + input: &ManifestInput, + graph: &ProjectGraph, +) -> Result, ManifestError> { + let manifest = payload(input, graph)?; + Ok(encode_python_value(manifest) + .map_err(ManifestEncodeError::Unencodable) + .and_then(|value: JsonValue| { + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(MANIFEST_INDENT))), + ) + .map_err(ManifestEncodeError::Emit) + }) + .map(|text| text + "\n")) +} diff --git a/crates/sqlbuild-analysis/src/manifest/main/mod.rs b/crates/sqlbuild-analysis/src/manifest/main/mod.rs new file mode 100644 index 0000000000..3d510c80d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/main/mod.rs @@ -0,0 +1 @@ +pub mod build_manifest_json; diff --git a/crates/sqlbuild-analysis/src/manifest/mod.rs b/crates/sqlbuild-analysis/src/manifest/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/manifest/models.rs b/crates/sqlbuild-analysis/src/manifest/models.rs new file mode 100644 index 0000000000..7065b8efcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/models.rs @@ -0,0 +1,236 @@ +//! The compiled project, plan entries and Python-only facts a dbt-style manifest is built from. + +use crate::assembly::project::types::ObjectKey; +use crate::python_values::models::PythonValue; + +/// A compiled relation location. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestRelationInput { + pub database: Option, + pub schema: Option, + pub name: Option, + pub qualified_name: Option, +} + +/// A path's `str()`, its parent's parts and its stem. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestPathInput { + pub text: String, + pub parent_parts: Vec, + pub stem: String, +} + +/// An authored column. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestColumnInput { + pub name: String, + pub description: Option, + /// `dict(column.meta)` when the meta is truthy. + pub meta: Option, + pub data_type: Option, +} + +/// One resolved lifecycle hook entry. +#[derive(Clone, Debug, PartialEq)] +pub enum ManifestHookInput { + Sql { + statement: String, + name: Option, + /// `relative_path.as_posix()`. + relative_path: Option, + definition_sql: Option, + kwargs: PythonValue, + description: Option, + }, + Python { + name: String, + kwargs: PythonValue, + }, + /// Any other entry, which serializations skip. + Other, +} + +/// The model config values the manifest reads, each as `values.get(key, default)` returned it. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestConfigInput { + pub enabled: PythonValue, + pub alias: PythonValue, + pub schema: PythonValue, + pub database: PythonValue, + /// `str()` of each tag when the `tags` value is a list or tuple. + pub tags: Option>, + /// `str()` of the `materialized` value, `view` when absent or `None`. + pub materialized: String, + pub incremental_strategy: PythonValue, + pub merge_exclude_columns: PythonValue, + pub full_refresh: PythonValue, + pub unique_key: PythonValue, + pub on_schema_change: PythonValue, + /// The entries when the hooks value is a list or tuple. + pub pre_hooks: Option>, + pub post_hooks: Option>, +} + +/// One dynamic column family of a model schema. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestDynamicColumnInput { + pub name: PythonValue, + pub pivot_column: PythonValue, + pub value_column: PythonValue, + pub aggregate: PythonValue, + pub data_type: PythonValue, + pub name_pattern: PythonValue, +} + +/// A model's attached schema entry. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestModelSchemaInput { + pub description: Option, + /// `dict(schema_entry.meta)` when the meta is not `None`. + pub meta: Option, + pub columns: Vec, + pub dynamic_columns: Vec, +} + +/// A compiled model. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestModelInput { + pub name: String, + pub path: ManifestPathInput, + pub destination: ManifestRelationInput, + pub query_sql: String, + pub config: ManifestConfigInput, + pub schema: Option, + pub deps: Vec, +} + +/// A compiled source. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSourceInput { + /// The compiled source's name, which keys the `sources` map. + pub name: String, + pub path: String, + pub entry_name: String, + pub database: Option, + pub schema: Option, + pub table: Option, + pub description: Option, + /// `dict(entry.meta)` when the meta is truthy. + pub meta: Option, + pub loader: Option, + pub write_strategy: Option, + pub cursor_column: Option, + /// The unique key, when truthy. + pub unique_key: Option, + pub columns: Vec, +} + +/// A compiled seed. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSeedInput { + pub name: String, + pub path: ManifestPathInput, + pub destination: ManifestRelationInput, + pub tags: Vec, + pub description: Option, + /// `dict(schema_entry.meta)` when the meta is truthy. + pub meta: Option, + pub columns: Vec, +} + +/// A loaded macro, keyed by its registry name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestMacroInput { + /// The registry name the macro is loaded under. + pub key: String, + pub name: String, + pub relative_path: String, + pub raw_source: String, +} + +/// A macro declaration of the scope index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestMacroDeclarationInput { + pub name: String, + pub scope: String, + pub owning_path: Option, + /// The macro's dependency names, when the record carries macro facts. + pub dependencies: Option>, +} + +/// A SQL hook definition file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestSqlHookInput { + pub name: String, + pub relative_path: String, + pub sql_body: String, + pub description: Option, +} + +/// A Python hook's path and source identity hashes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestPythonHookInput { + pub name: String, + pub relative_path: String, + pub definition_hash: String, + pub version_hash: String, +} + +/// A planned audit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestAuditPlanInput { + pub name: String, + pub resolved_sql: String, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub scope_deps: Vec, +} + +/// A planned SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSqlTestPlanInput { + pub key: ObjectKey, + pub name: String, + /// `(model name, resolved SQL)` of each chain step. + pub chain: Vec<(String, String)>, + pub scope_deps: Vec, + pub source_path: Option, + pub block_index: PythonValue, + pub parent_name: PythonValue, + pub case_name: Option, + pub case_index: PythonValue, + pub case_fingerprint: Option, + /// `(name, value type, nullable)` of each declared parameter. + pub parameter_schema: Vec<(String, String, PythonValue)>, + /// `(name, value)` of each bound parameter, decimals already as text. + pub parameters: Vec<(String, PythonValue)>, +} + +/// The plan entries a build manifest reads. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ManifestPlanInput { + /// `(model name, resolved SQL)` of each model entry. + pub models: Vec<(String, String)>, + pub source_load_names: Vec, + pub audits: Vec, + pub sql_tests: Vec, +} + +/// Everything the manifest is built from besides the graph. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ManifestInput { + pub project_name: String, + pub adapter_type: String, + pub run_id: String, + /// `datetime.now(tz=UTC).isoformat()`. + pub generated_at: String, + pub sql_analysis_dialect: Option, + pub models: Vec, + pub sources: Vec, + pub seeds: Vec, + pub macros: Vec, + pub macro_declarations: Vec, + pub sql_hooks: Vec, + pub python_hooks: Vec, + pub plan: Option, +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests.rs b/crates/sqlbuild-analysis/src/manifest/tests.rs new file mode 100644 index 0000000000..4045696844 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_manifest_json.rs"] +mod build_manifest_json; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs b/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs new file mode 100644 index 0000000000..8523893f3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs @@ -0,0 +1,96 @@ +use crate::manifest::models::{ + ManifestConfigInput, ManifestHookInput, ManifestInput, ManifestModelInput, + ManifestModelSchemaInput, ManifestPathInput, ManifestRelationInput, ManifestSqlTestPlanInput, +}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +pub(crate) fn config() -> ManifestConfigInput { + ManifestConfigInput { + enabled: PythonValue::Bool(true), + alias: PythonValue::Null, + schema: PythonValue::Null, + database: PythonValue::Null, + tags: None, + materialized: "table".to_owned(), + incremental_strategy: PythonValue::Null, + merge_exclude_columns: PythonValue::Null, + full_refresh: PythonValue::Null, + unique_key: PythonValue::Null, + on_schema_change: PythonValue::String("ignore".to_owned()), + pre_hooks: None, + post_hooks: None, + } +} + +pub(crate) fn sql_hook() -> ManifestHookInput { + ManifestHookInput::Sql { + statement: "select 1".to_owned(), + name: Some("audit_orders".to_owned()), + relative_path: None, + definition_sql: None, + kwargs: PythonValue::Null, + description: Some(String::new()), + } +} + +pub(crate) fn meta(entries: Vec<(PythonKey, PythonValue)>) -> Option { + Some(PythonValue::Object(entries)) +} + +pub(crate) fn model(meta: Option, config: ManifestConfigInput) -> ManifestModelInput { + ManifestModelInput { + name: "orders".to_owned(), + path: ManifestPathInput { + text: "models/orders.sql".to_owned(), + parent_parts: vec!["models".to_owned()], + stem: "orders".to_owned(), + }, + destination: ManifestRelationInput::default(), + query_sql: "SELECT 1 AS order_id".to_owned(), + config, + schema: Some(ManifestModelSchemaInput { + description: None, + meta, + columns: Vec::new(), + dynamic_columns: Vec::new(), + }), + deps: Vec::new(), + } +} + +pub(crate) fn project(models: Vec) -> ManifestInput { + ManifestInput { + project_name: "shop".to_owned(), + adapter_type: "duckdb".to_owned(), + run_id: "run".to_owned(), + generated_at: "2026-10-12T00:00:00+00:00".to_owned(), + models, + ..ManifestInput::default() + } +} + +pub(crate) fn authored(key: &str, value: PythonValue) -> (PythonKey, PythonValue) { + (text_key(key), value) +} + +pub(crate) fn sql_test(parameter: &str) -> ManifestSqlTestPlanInput { + ManifestSqlTestPlanInput { + key: ("sql_test".to_owned(), "orders_case".to_owned()), + name: "orders_case".to_owned(), + chain: Vec::new(), + scope_deps: Vec::new(), + source_path: Some("tests/orders.sql".to_owned()), + block_index: PythonValue::Null, + parent_name: PythonValue::Null, + case_name: Some("first".to_owned()), + case_index: PythonValue::Null, + case_fingerprint: None, + parameter_schema: vec![( + "amount".to_owned(), + "decimal".to_owned(), + PythonValue::Bool(false), + )], + parameters: vec![(parameter.to_owned(), PythonValue::String("1.50".to_owned()))], + } +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs b/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs new file mode 100644 index 0000000000..b68a4031e1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs @@ -0,0 +1,141 @@ +use crate::graph::models::ProjectGraph; +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::main::build_manifest_json::build_manifest_json; +use crate::manifest::models::ManifestPlanInput; +use crate::manifest::tests::helpers::{authored, config, meta, model, project, sql_hook, sql_test}; +use crate::manifest::tests::test_types::{BuildManifestJsonTestCase, ManifestFailureTestCase}; +use crate::python_values::models::{PythonKey, PythonValue}; + +#[test] +fn given_project_when_building_manifest_json_then_python_payload_agrees() { + let hooked = || { + let mut hooked = config(); + hooked.post_hooks = Some(vec![sql_hook()]); + hooked + }; + let test_cases = [ + BuildManifestJsonTestCase { + description: "an empty project keeps metadata first and ends with a newline", + input: project(Vec::new()), + expected_fragments: vec![ + "{\n \"metadata\": {\n \"dbt_schema_version\": \"https://schemas.getdbt.com/dbt/manifest/v12.json\",\n \"dbt_version\": \"1.12.0\",\n \"generated_at\": \"2026-10-12T00:00:00+00:00\",\n \"invocation_id\": \"run\",", + " \"unit_tests\": {}\n}\n", + ], + }, + BuildManifestJsonTestCase { + description: "an authored sqlbuild scalar is kept beside lifecycle hooks", + input: project(vec![model( + meta(vec![authored( + "sqlbuild", + PythonValue::String("team".to_owned()), + )]), + hooked(), + )]), + expected_fragments: vec![ + "\"sqlbuild\": {\n \"authored_value\": \"team\",\n \"lifecycle_hooks\": {\n \"post_hooks\": [", + "\"type\": \"sql\",\n \"statement\": \"select 1\",\n \"name\": \"audit_orders\",\n \"description\": \"\"\n", + "\"post-hook\": [\n {\n \"sql\": \"select 1\",\n \"transaction\": true,\n \"index\": 0\n", + "\"fqn\": [\n \"shop\",\n \"models\",\n \"orders\"\n ]", + "\"created_at\": 0.0,", + ], + }, + BuildManifestJsonTestCase { + description: "non-str keys of an authored sqlbuild dict are dropped", + input: project(vec![model( + meta(vec![authored( + "sqlbuild", + PythonValue::Object(vec![ + ( + PythonKey { + text: Ok("1".to_owned()), + is_str: false, + }, + PythonValue::Null, + ), + authored("owner", PythonValue::String("ops".to_owned())), + ]), + )]), + hooked(), + )]), + expected_fragments: vec![ + "\"sqlbuild\": {\n \"owner\": \"ops\",\n \"lifecycle_hooks\"", + ], + }, + ]; + for test_case in test_cases { + let json: String = build_manifest_json(&test_case.input, &ProjectGraph::default()) + .expect("manifest builds") + .expect("manifest encodes"); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| json.contains(fragment)), + "{}: missing a fragment of\n{json}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_values_when_building_manifest_json_then_python_error_agrees() { + let unencodable = + PythonValue::Unencodable("Object of type set is not JSON serializable".to_owned()); + let test_cases = [ + ManifestFailureTestCase { + description: "an unencodable meta value fails on encoding", + input: project(vec![model( + meta(vec![authored("owners", unencodable.clone())]), + config(), + )]), + expected_error: Ok(Some(ManifestEncodeError::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + ))), + }, + ManifestFailureTestCase { + description: "the first unencodable value in output order is reported", + input: { + let mut early = config(); + early.alias = PythonValue::Unencodable( + "Object of type frozenset is not JSON serializable".to_owned(), + ); + project(vec![model(meta(vec![authored("owners", unencodable)]), early)]) + }, + expected_error: Ok(Some(ManifestEncodeError::Unencodable( + "Object of type frozenset is not JSON serializable".to_owned(), + ))), + }, + ManifestFailureTestCase { + description: "a bound parameter missing from the schema fails while building", + input: { + let mut input = project(Vec::new()); + input.plan = Some(ManifestPlanInput { + sql_tests: vec![sql_test("other")], + ..ManifestPlanInput::default() + }); + input + }, + expected_error: Err(ManifestError::MissingParameterType("other".to_owned())), + }, + ManifestFailureTestCase { + description: "a query that cannot be fingerprinted fails while building", + input: { + let mut broken = model(None, config()); + broken.query_sql = "SELECT 'unterminated".to_owned(); + project(vec![broken]) + }, + expected_error: Err(ManifestError::QueryFingerprint( + "SQL tokenization failed: Tokenization error at line 1, column 21: Unterminated string".to_owned(), + )), + }, + ]; + for test_case in test_cases { + let actual: Result, ManifestError> = + build_manifest_json(&test_case.input, &ProjectGraph::default()).map(Result::err); + assert_eq!( + actual, test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs b/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs new file mode 100644 index 0000000000..d0ad2b87e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::models::ManifestInput; + +pub(crate) struct BuildManifestJsonTestCase { + pub(crate) description: &'static str, + pub(crate) input: ManifestInput, + pub(crate) expected_fragments: Vec<&'static str>, +} + +pub(crate) struct ManifestFailureTestCase { + pub(crate) description: &'static str, + pub(crate) input: ManifestInput, + /// The building error, or the encoding error kept for the writer. + pub(crate) expected_error: Result, ManifestError>, +} diff --git a/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs b/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs new file mode 100644 index 0000000000..23ee2aa843 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs @@ -0,0 +1,30 @@ +//! The JSON value `json.dumps` encodes, or the `TypeError` it raises first in output order. + +use sqlbuild_core::json::models::JsonValue; + +use crate::python_values::models::PythonValue; + +/// The encoded value, or the message of the first `TypeError` in output order. +pub fn encode_python_value(value: PythonValue) -> Result { + Ok(match value { + PythonValue::Null => JsonValue::Null, + PythonValue::Bool(flag) => JsonValue::Bool(flag), + PythonValue::Integer(number) => JsonValue::Integer(number), + PythonValue::Float(number) => JsonValue::Float(number), + PythonValue::String(text) => JsonValue::String(text), + PythonValue::Utf16(units) => JsonValue::Utf16(units), + PythonValue::Array(items) => JsonValue::Array( + items + .into_iter() + .map(encode_python_value) + .collect::>()?, + ), + PythonValue::Object(entries) => JsonValue::Object( + entries + .into_iter() + .map(|(key, item)| Ok((key.text?, encode_python_value(item)?))) + .collect::>()?, + ), + PythonValue::Unencodable(message) => return Err(message), + }) +} diff --git a/crates/sqlbuild-analysis/src/python_values/main/mod.rs b/crates/sqlbuild-analysis/src/python_values/main/mod.rs new file mode 100644 index 0000000000..ca5fee3739 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/mod.rs @@ -0,0 +1,2 @@ +pub mod encode_python_value; +pub mod text_key; diff --git a/crates/sqlbuild-analysis/src/python_values/main/text_key.rs b/crates/sqlbuild-analysis/src/python_values/main/text_key.rs new file mode 100644 index 0000000000..b898b05c9e --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/text_key.rs @@ -0,0 +1,12 @@ +//! A `str` dict key. + +use crate::python_values::models::PythonKey; + +/// The key of a `str` field name. +#[must_use] +pub fn text_key(text: &str) -> PythonKey { + PythonKey { + text: Ok(text.to_owned()), + is_str: true, + } +} diff --git a/crates/sqlbuild-analysis/src/python_values/mod.rs b/crates/sqlbuild-analysis/src/python_values/mod.rs new file mode 100644 index 0000000000..5355003ff2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/mod.rs @@ -0,0 +1,2 @@ +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-analysis/src/python_values/models.rs b/crates/sqlbuild-analysis/src/python_values/models.rs new file mode 100644 index 0000000000..a5bfa2bdc3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/models.rs @@ -0,0 +1,28 @@ +//! Python values on their way to `json.dumps`, keeping what Python's type checks see. + +use sqlbuild_core::json::models::JsonInteger; + +/// A Python value as `json.dumps` would encode it. +#[derive(Clone, Debug, PartialEq)] +pub enum PythonValue { + Null, + Bool(bool), + Integer(JsonInteger), + Float(f64), + String(String), + /// A `str` holding lone surrogates, as UTF-16 code units. + Utf16(Vec), + /// A `list` or `tuple`. + Array(Vec), + /// A `dict`, in insertion order. + Object(Vec<(PythonKey, PythonValue)>), + /// A value `json.dumps` refuses, with the message of its `TypeError`. + Unencodable(String), +} + +/// A `dict` key: the text `json.dumps` writes, or its `TypeError` message, and whether it is a `str`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonKey { + pub text: Result, + pub is_str: bool, +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index e166e32913..a771ef8731 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -24,6 +24,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; const ANALYSIS_WORKERS: usize = 4; const ANALYSIS_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; +const DEFAULT_BINDING_MESSAGE: &str = "SQL binding failed"; impl ProjectCatalog { /// Map inferred column types onto the binding names the query projects, in column order. @@ -200,6 +201,32 @@ impl ProjectCatalog { serde_json::to_string(&payloads).map_err(|error| error.to_string()) } + /// Validate each request on its own schema, as `validations_json` does, as diagnostic rows. + pub fn schema_validation_rows( + &self, + dialect: &str, + requests: Vec, + ) -> Result>, String> { + let sqls: Vec = requests.iter().map(|request| request.0.clone()).collect(); + let payload: String = self.validation_payload(requests)?; + let results: Vec = + serde_json::from_str(&validation::validations_json(&payload)?) + .map_err(|error| error.to_string())?; + if results.len() != sqls.len() { + return Err( + "native SQL schema validation returned an invalid batch response".to_owned(), + ); + } + sqls.iter() + .zip(results) + .map(|(sql, result)| { + let rows: Vec = + result.errors.into_iter().map(messaged_row).collect(); + diagnostics::binding_diagnostics(sql, dialect, rows) + }) + .collect() + } + /// Resolve a compact batch now; the job runs later while the catalog keeps changing. pub fn prepare_compact(&self, text: &str) -> Result { let analysis = @@ -586,6 +613,15 @@ impl ProjectCatalog { } } +/// The diagnostic row of `error`, naming the failure when the parser gives no message. +fn messaged_row(error: ValidationError) -> DiagnosticRow { + let mut row: DiagnosticRow = diagnostic_row(error); + if row.1.is_empty() { + DEFAULT_BINDING_MESSAGE.clone_into(&mut row.1); + } + row +} + pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { let severity = match error.severity { polyglot_sql::ValidationSeverity::Error => "error", diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs index e53d6ea07e..d28fbf2ed3 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod entry_names; pub(crate) mod failures; +pub(crate) mod suffixes; diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs new file mode 100644 index 0000000000..736e5419b4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs @@ -0,0 +1,16 @@ +//! `Path.suffix` and `Path.glob` name matching as Python applies them to listed names. + +use sqlbuild_core::text::main::glob_name_match::glob_name_match; + +/// Python's `PurePath.suffix` of a file name. +pub(crate) fn python_suffix(name: &str) -> &str { + match name.rfind('.') { + Some(index) if index > 0 && index + 1 < name.len() => &name[index..], + _ => "", + } +} + +/// Whether `name` matches a single-name glob with the platform's default case sensitivity. +pub(crate) fn name_matches(name: &str, pattern: &str) -> bool { + glob_name_match(name, pattern, cfg!(windows)) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/constants.rs b/crates/sqlbuild-discovery/src/yaml_files/constants.rs new file mode 100644 index 0000000000..726dfc8b1c --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/constants.rs @@ -0,0 +1,12 @@ +//! Where authored source and schema YAML files live. + +/// The directory whose direct YAML children declare sources. +pub(crate) const SOURCES_DIRECTORY: &str = "sources"; +pub(crate) const MODELS_DIRECTORY: &str = "models"; +pub(crate) const SEEDS_DIRECTORY: &str = "seeds"; +/// Python's `YAML_FILE_SUFFIXES`. +pub(crate) const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; +/// Python's `SCHEMA_FILE_NAME`. +pub(crate) const SCHEMA_FILE_NAME: &str = "schema.yml"; +pub(crate) const SEED_DECLARATION_PATTERN: &str = "*.yml"; +pub(crate) const UNSUPPORTED_SEED_DECLARATION_PATTERN: &str = "*.yaml"; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs index b6ed6a999a..a41c241ae9 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -1,3 +1,5 @@ pub mod load_yaml_files; pub mod load_yaml_text; +pub mod schema_yaml_paths; +pub mod source_yaml_paths; pub mod yaml_entry_lines; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs new file mode 100644 index 0000000000..21f8fe6ef9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs @@ -0,0 +1,38 @@ +//! The unscoped model `schema.yml` files and seed declaration files, in Python's order. + +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use crate::yaml_files::_helpers::suffixes::name_matches; +use crate::yaml_files::constants::{ + MODELS_DIRECTORY, SCHEMA_FILE_NAME, SEED_DECLARATION_PATTERN, SEEDS_DIRECTORY, + UNSUPPORTED_SEED_DECLARATION_PATTERN, +}; +use crate::yaml_files::models::YamlPathFailure; + +fn matching(tree: &ProjectTree, root: &str, pattern: &str) -> Result, YamlPathFailure> { + rglob(tree, root, |entry| name_matches(&entry.name, pattern)).map_err(YamlPathFailure::Stage) +} + +/// Python's `schema_file_paths`; a `.yaml` seed declaration fails as a schema error. +pub fn schema_yaml_paths(tree: &ProjectTree) -> Result, YamlPathFailure> { + let mut paths: Vec = Vec::new(); + if tree.is_dir(MODELS_DIRECTORY) { + paths.extend( + matching(tree, MODELS_DIRECTORY, SCHEMA_FILE_NAME)? + .into_iter() + .filter(|path| !is_in_scoped_declaration_tree(path)), + ); + } + if tree.is_dir(SEEDS_DIRECTORY) { + if let Some(path) = + matching(tree, SEEDS_DIRECTORY, UNSUPPORTED_SEED_DECLARATION_PATTERN)?.first() + { + return Err(YamlPathFailure::YamlSeedDeclaration(path.clone())); + } + paths.extend(matching(tree, SEEDS_DIRECTORY, SEED_DECLARATION_PATTERN)?); + } + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + paths.retain(|path| seen.insert(path.clone())); + Ok(paths) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs new file mode 100644 index 0000000000..6c33d2c98a --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs @@ -0,0 +1,19 @@ +//! The YAML files directly below `sources/`, as Python's source discovery lists them. + +use crate::models::StageFailure; +use crate::tree::main::children::children; +use crate::tree::models::ProjectTree; +use crate::yaml_files::_helpers::suffixes::python_suffix; +use crate::yaml_files::constants::{SOURCES_DIRECTORY, YAML_FILE_SUFFIXES}; + +/// `sorted(path for path in sources.iterdir() if path.suffix in YAML_FILE_SUFFIXES)`. +pub fn source_yaml_paths(tree: &ProjectTree) -> Result, StageFailure> { + if !tree.is_dir(SOURCES_DIRECTORY) { + return Ok(Vec::new()); + } + Ok(children(tree, SOURCES_DIRECTORY)? + .into_iter() + .filter(|(_, entry)| YAML_FILE_SUFFIXES.contains(&python_suffix(&entry.name))) + .map(|(path, _)| path) + .collect()) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/mod.rs index e6e5a52eb4..1e64f3d116 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-discovery/src/yaml_files/models.rs b/crates/sqlbuild-discovery/src/yaml_files/models.rs index 73d52e5c8a..f119786801 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/models.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/models.rs @@ -1,6 +1,6 @@ //! Native outcomes of reading and loading authored YAML files. -use crate::models::{DiscoveryFailure, ReadFailure}; +use crate::models::{DiscoveryFailure, ReadFailure, StageFailure}; use sqlbuild_config::models::ConfigValue; /// One YAML file read and loaded as YAML 1.2, or why it could not be. @@ -15,3 +15,11 @@ pub enum YamlFileOutcome { /// The file could not be read or decoded as Python's `read_text` would. Unreadable(ReadFailure), } + +/// Why the schema YAML files could not be listed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum YamlPathFailure { + Stage(StageFailure), + /// The first seed declaration file, by relative path, that uses `.yaml`. + YamlSeedDeclaration(String), +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests.rs b/crates/sqlbuild-discovery/src/yaml_files/tests.rs index 9026fd5196..cbe8b7097a 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests.rs @@ -6,3 +6,5 @@ mod load_yaml_files; mod test_types; #[path = "tests/test_yaml_entry_lines.rs"] mod yaml_entry_lines; +#[path = "tests/test_yaml_paths.rs"] +mod yaml_paths; diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs index f7330423e5..00de3a41db 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs @@ -37,3 +37,21 @@ pub(super) fn owned_failure( ) -> Option<(String, Option)> { expected.map(|(message, help)| (message.to_owned(), help.map(str::to_owned))) } + +/// The paths a listing of a project holding `files` (with empty contents) returns. +pub(super) fn listed_paths( + files: &[&str], + list: impl Fn(&ProjectTree) -> Result, crate::yaml_files::models::YamlPathFailure>, +) -> Result, crate::yaml_files::models::YamlPathFailure> { + let project = tempfile::tempdir().expect("temporary project"); + for file in files { + let path = project.path().join(file); + std::fs::create_dir_all(path.parent().expect("file parent")).expect("parent directory"); + std::fs::write(path, b"").expect("project file"); + } + list(&ProjectTree::new(project.path())) +} + +pub(super) fn owned_paths(paths: &[&str]) -> Vec { + paths.iter().map(|path| (*path).to_owned()).collect() +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs index 1d06fe9305..be46dd86c1 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::yaml_files::models::YamlFileOutcome; +use crate::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; pub(super) struct YamlFileTestCase { pub(super) description: &'static str, @@ -19,3 +19,9 @@ pub(super) struct EntryLinesTestCase { /// The line of each named entry the case checks; `None` where no entry has the name. pub(super) expected_lines: &'static [(&'static str, Option)], } + +pub(super) struct YamlPathsTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) expected_paths: Result, YamlPathFailure>, +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs new file mode 100644 index 0000000000..075258031b --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs @@ -0,0 +1,75 @@ +use crate::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use crate::yaml_files::main::source_yaml_paths::source_yaml_paths; +use crate::yaml_files::models::YamlPathFailure; +use crate::yaml_files::tests::helpers::{listed_paths, owned_paths}; +use crate::yaml_files::tests::test_types::YamlPathsTestCase; + +#[test] +fn given_sources_directory_when_listing_source_yaml_then_direct_yaml_children_are_sorted() { + let test_cases = [ + YamlPathsTestCase { + description: "yml and yaml files directly below sources/ in path order", + files: &[ + "sources/b.yaml", + "sources/a.yml", + "sources/notes.txt", + "sources/.yml", + "sources/nested/c.yml", + ], + expected_paths: Ok(owned_paths(&["sources/a.yml", "sources/b.yaml"])), + }, + YamlPathsTestCase { + description: "no sources directory", + files: &["models/orders.sql"], + expected_paths: Ok(Vec::new()), + }, + ]; + for test_case in test_cases { + let listed = listed_paths(test_case.files, |tree| { + source_yaml_paths(tree).map_err(YamlPathFailure::Stage) + }); + assert_eq!( + listed, test_case.expected_paths, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_models_and_seeds_when_listing_schema_yaml_then_unscoped_schemas_and_seed_files_are_kept() { + let test_cases = [ + YamlPathsTestCase { + description: "unscoped schema files then seed declarations, each in path order", + files: &[ + "models/marts/schema.yml", + "models/schema.yml", + "models/marts/_sqlbuild/schemas/schema.yml", + "models/other.yml", + "seeds/products.yml", + "seeds/customers/customers.yml", + ], + expected_paths: Ok(owned_paths(&[ + "models/marts/schema.yml", + "models/schema.yml", + "seeds/customers/customers.yml", + "seeds/products.yml", + ])), + }, + YamlPathsTestCase { + description: "a seed declaration using .yaml fails naming the first one", + files: &["seeds/b.yaml", "seeds/a.yaml", "seeds/c.yml"], + expected_paths: Err(YamlPathFailure::YamlSeedDeclaration( + "seeds/a.yaml".to_owned(), + )), + }, + ]; + for test_case in test_cases { + let listed = listed_paths(test_case.files, schema_yaml_paths); + assert_eq!( + listed, test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs new file mode 100644 index 0000000000..e04e5cfbf0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs @@ -0,0 +1,388 @@ +//! The children CPython's `ast.iter_child_nodes` yields, in `_fields` order. + +use ruff_python_ast::{ + ElifElseClause, ExceptHandler, Expr, Parameters, Pattern, Stmt, Suite, TypeParam, TypeParams, +}; + +use crate::cpython_tree::_helpers::joined::{element_pieces, merged_values}; +use crate::cpython_tree::main::joined_values::joined_values; +use crate::cpython_tree::models::CpythonNode; + +/// The child nodes of `node` that can hold other nodes; operator and context leaves are omitted. +pub(crate) fn children<'a>(node: &CpythonNode<'a>) -> Vec> { + match node { + CpythonNode::Stmt(statement) => statement_children(statement), + CpythonNode::Elif(clauses) => elif_children(clauses), + CpythonNode::Expr(expression) => expression_children(expression), + CpythonNode::Arguments(parameters) => argument_children(parameters), + CpythonNode::Arg(parameter) => optional(parameter.annotation.as_deref()), + CpythonNode::Keyword(keyword) => vec![CpythonNode::Expr(&keyword.value)], + CpythonNode::WithItem(item) => [ + vec![CpythonNode::Expr(&item.context_expr)], + optional(item.optional_vars.as_deref()), + ] + .concat(), + CpythonNode::MatchCase(case) => [ + vec![CpythonNode::Pattern(&case.pattern)], + optional(case.guard.as_deref()), + statements(&case.body), + ] + .concat(), + CpythonNode::Pattern(pattern) => pattern_children(pattern), + CpythonNode::Comprehension(comprehension) => [ + vec![ + CpythonNode::Expr(&comprehension.target), + CpythonNode::Expr(&comprehension.iter), + ], + expressions(&comprehension.ifs), + ] + .concat(), + CpythonNode::ExceptHandler(ExceptHandler::ExceptHandler(handler)) => [ + optional(handler.type_.as_deref()), + statements(&handler.body), + ] + .concat(), + CpythonNode::TypeParam(parameter) => type_param_children(parameter), + CpythonNode::Text(_) => Vec::new(), + CpythonNode::Interpolation(interpolation) => { + let spec: Vec> = interpolation + .format_spec + .iter() + .map(|spec| CpythonNode::FormatSpec(spec)) + .collect(); + [vec![CpythonNode::Expr(&interpolation.expression)], spec].concat() + } + CpythonNode::FormatSpec(spec) => merged_values(element_pieces(&spec.elements)), + } +} + +fn optional(expression: Option<&Expr>) -> Vec> { + expression.map(CpythonNode::Expr).into_iter().collect() +} + +fn expressions(expressions: &[Expr]) -> Vec> { + expressions.iter().map(CpythonNode::Expr).collect() +} + +fn statements(body: &Suite) -> Vec> { + body.iter().map(CpythonNode::Stmt).collect() +} + +fn decorators(decorator_list: &[ruff_python_ast::Decorator]) -> Vec> { + decorator_list + .iter() + .map(|decorator| CpythonNode::Expr(&decorator.expression)) + .collect() +} + +fn type_params(parameters: Option<&TypeParams>) -> Vec> { + parameters + .map(|parameters| parameters.iter().map(CpythonNode::TypeParam).collect()) + .unwrap_or_default() +} + +fn statement_children(statement: &Stmt) -> Vec> { + match statement { + Stmt::FunctionDef(function) => [ + vec![CpythonNode::Arguments(&function.parameters)], + statements(&function.body), + decorators(&function.decorator_list), + optional(function.returns.as_deref()), + type_params(function.type_params.as_deref()), + ] + .concat(), + Stmt::ClassDef(class) => [ + expressions(class.bases()), + class.keywords().iter().map(CpythonNode::Keyword).collect(), + statements(&class.body), + decorators(&class.decorator_list), + type_params(class.type_params.as_deref()), + ] + .concat(), + Stmt::Return(statement) => optional(statement.value.as_deref()), + Stmt::Delete(statement) => expressions(&statement.targets), + Stmt::TypeAlias(statement) => [ + vec![CpythonNode::Expr(&statement.name)], + type_params(statement.type_params.as_deref()), + vec![CpythonNode::Expr(&statement.value)], + ] + .concat(), + Stmt::Assign(statement) => [ + expressions(&statement.targets), + vec![CpythonNode::Expr(&statement.value)], + ] + .concat(), + Stmt::AugAssign(statement) => vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.value), + ], + Stmt::AnnAssign(statement) => [ + vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.annotation), + ], + optional(statement.value.as_deref()), + ] + .concat(), + Stmt::For(statement) => [ + vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.iter), + ], + statements(&statement.body), + statements(&statement.orelse), + ] + .concat(), + Stmt::While(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + statements(&statement.body), + statements(&statement.orelse), + ] + .concat(), + Stmt::If(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + statements(&statement.body), + else_children(&statement.elif_else_clauses), + ] + .concat(), + Stmt::With(statement) => [ + statement.items.iter().map(CpythonNode::WithItem).collect(), + statements(&statement.body), + ] + .concat(), + Stmt::Match(statement) => [ + vec![CpythonNode::Expr(&statement.subject)], + statement.cases.iter().map(CpythonNode::MatchCase).collect(), + ] + .concat(), + Stmt::Raise(statement) => [ + optional(statement.exc.as_deref()), + optional(statement.cause.as_deref()), + ] + .concat(), + Stmt::Try(statement) => [ + statements(&statement.body), + statement + .handlers + .iter() + .map(CpythonNode::ExceptHandler) + .collect(), + statements(&statement.orelse), + statements(&statement.finalbody), + ] + .concat(), + Stmt::Assert(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + optional(statement.msg.as_deref()), + ] + .concat(), + Stmt::Expr(statement) => vec![CpythonNode::Expr(&statement.value)], + Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// The `orelse` of an `If`: a nested `If` for an `elif`, or the `else` body. +fn else_children(clauses: &[ElifElseClause]) -> Vec> { + match clauses.first() { + Some(clause) if clause.test.is_some() => vec![CpythonNode::Elif(clauses)], + Some(clause) => statements(&clause.body), + None => Vec::new(), + } +} + +fn elif_children(clauses: &[ElifElseClause]) -> Vec> { + clauses + .split_first() + .map(|(clause, rest)| { + [ + optional(clause.test.as_ref()), + statements(&clause.body), + else_children(rest), + ] + .concat() + }) + .unwrap_or_default() +} + +fn expression_children(expression: &Expr) -> Vec> { + match expression { + Expr::BoolOp(node) => expressions(&node.values), + Expr::Named(node) => vec![ + CpythonNode::Expr(&node.target), + CpythonNode::Expr(&node.value), + ], + Expr::BinOp(node) => vec![ + CpythonNode::Expr(&node.left), + CpythonNode::Expr(&node.right), + ], + Expr::UnaryOp(node) => vec![CpythonNode::Expr(&node.operand)], + Expr::Lambda(node) => [ + node.parameters + .iter() + .map(|parameters| CpythonNode::Arguments(parameters)) + .collect(), + vec![CpythonNode::Expr(&node.body)], + ] + .concat(), + Expr::If(node) => vec![ + CpythonNode::Expr(&node.test), + CpythonNode::Expr(&node.body), + CpythonNode::Expr(&node.orelse), + ], + Expr::Dict(node) => node + .iter_keys() + .flatten() + .chain(node.iter_values()) + .map(CpythonNode::Expr) + .collect(), + Expr::Set(node) => expressions(&node.elts), + Expr::ListComp(node) => comprehension_children(&node.elt, &node.generators), + Expr::SetComp(node) => comprehension_children(&node.elt, &node.generators), + Expr::Generator(node) => comprehension_children(&node.elt, &node.generators), + Expr::DictComp(node) => [ + optional(node.key.as_deref()), + comprehension_children(&node.value, &node.generators), + ] + .concat(), + Expr::Await(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Yield(node) => optional(node.value.as_deref()), + Expr::YieldFrom(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Compare(node) => [ + vec![CpythonNode::Expr(&node.left)], + expressions(&node.comparators), + ] + .concat(), + Expr::Call(node) => [ + vec![CpythonNode::Expr(&node.func)], + expressions(&node.arguments.args), + node.arguments + .keywords + .iter() + .map(CpythonNode::Keyword) + .collect(), + ] + .concat(), + Expr::FString(_) | Expr::TString(_) => joined_values(expression), + Expr::Attribute(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Subscript(node) => vec![ + CpythonNode::Expr(&node.value), + CpythonNode::Expr(&node.slice), + ], + Expr::Starred(node) => vec![CpythonNode::Expr(&node.value)], + Expr::List(node) => expressions(&node.elts), + Expr::Tuple(node) => expressions(&node.elts), + Expr::Slice(node) => [ + optional(node.lower.as_deref()), + optional(node.upper.as_deref()), + optional(node.step.as_deref()), + ] + .concat(), + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::Name(_) + | Expr::IpyEscapeCommand(_) => Vec::new(), + } +} + +fn comprehension_children<'a>( + element: &'a Expr, + generators: &'a [ruff_python_ast::Comprehension], +) -> Vec> { + [ + vec![CpythonNode::Expr(element)], + generators.iter().map(CpythonNode::Comprehension).collect(), + ] + .concat() +} + +/// `arguments` fields: posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults. +fn argument_children(parameters: &Parameters) -> Vec> { + let positional = || parameters.posonlyargs.iter().chain(parameters.args.iter()); + [ + positional() + .map(|parameter| CpythonNode::Arg(¶meter.parameter)) + .collect(), + parameters + .vararg + .iter() + .map(|parameter| CpythonNode::Arg(parameter)) + .collect(), + parameters + .kwonlyargs + .iter() + .map(|parameter| CpythonNode::Arg(¶meter.parameter)) + .collect(), + parameters + .kwonlyargs + .iter() + .filter_map(|parameter| parameter.default.as_deref()) + .map(CpythonNode::Expr) + .collect(), + parameters + .kwarg + .iter() + .map(|parameter| CpythonNode::Arg(parameter)) + .collect(), + positional() + .filter_map(|parameter| parameter.default.as_deref()) + .map(CpythonNode::Expr) + .collect::>>(), + ] + .concat() +} + +fn pattern_children(pattern: &Pattern) -> Vec> { + match pattern { + Pattern::MatchValue(node) => vec![CpythonNode::Expr(&node.value)], + Pattern::MatchSequence(node) => node.patterns.iter().map(CpythonNode::Pattern).collect(), + Pattern::MatchMapping(node) => [ + expressions(&node.keys), + node.patterns.iter().map(CpythonNode::Pattern).collect(), + ] + .concat(), + Pattern::MatchClass(node) => [ + vec![CpythonNode::Expr(&node.cls)], + node.arguments + .patterns + .iter() + .map(CpythonNode::Pattern) + .collect(), + node.arguments + .keywords + .iter() + .map(|keyword| CpythonNode::Pattern(&keyword.pattern)) + .collect(), + ] + .concat(), + Pattern::MatchAs(node) => node + .pattern + .iter() + .map(|inner| CpythonNode::Pattern(inner)) + .collect(), + Pattern::MatchOr(node) => node.patterns.iter().map(CpythonNode::Pattern).collect(), + Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => Vec::new(), + } +} + +fn type_param_children(parameter: &TypeParam) -> Vec> { + match parameter { + TypeParam::TypeVar(node) => [ + optional(node.bound.as_deref()), + optional(node.default.as_deref()), + ] + .concat(), + TypeParam::ParamSpec(node) => optional(node.default.as_deref()), + TypeParam::TypeVarTuple(node) => optional(node.default.as_deref()), + } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs new file mode 100644 index 0000000000..4ae730df7f --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs @@ -0,0 +1,64 @@ +//! CPython's merging of adjacent string pieces in f-strings, t-strings and format specs. + +use ruff_python_ast::{InterpolatedStringElement, InterpolatedStringElements}; +use ruff_text_size::{TextLen, TextSize}; + +use crate::cpython_tree::models::{CpythonNode, TextConstant}; + +/// One piece CPython concatenates: literal text at an offset, or an interpolation. +pub(crate) enum JoinedPiece<'a> { + Text(&'a str, TextSize), + Interpolation(&'a ruff_python_ast::InterpolatedElement), +} + +/// The pieces of f-string elements; a self-documenting `{x=}` contributes its text first. +pub(crate) fn element_pieces(elements: &InterpolatedStringElements) -> Vec> { + elements.iter().flat_map(pieces).collect() +} + +fn pieces(element: &InterpolatedStringElement) -> Vec> { + match element { + InterpolatedStringElement::Literal(literal) => { + vec![JoinedPiece::Text(&literal.value, literal.range.start())] + } + InterpolatedStringElement::Interpolation(interpolation) => { + let after_brace: TextSize = interpolation.range.start() + '{'.text_len(); + let debug: Option> = interpolation + .debug_text + .as_ref() + .map(|debug_text| JoinedPiece::Text(debug_text.as_str(), after_brace)); + debug + .into_iter() + .chain([JoinedPiece::Interpolation(interpolation)]) + .collect() + } + } +} + +/// Merge runs of text into constants located at the run's first piece, dropping empty runs. +pub(crate) fn merged_values(pieces: Vec>) -> Vec> { + let mut values: Vec> = Vec::new(); + let mut run: Option = None; + for piece in pieces { + match piece { + JoinedPiece::Text(text, start) => { + let constant: &mut TextConstant = run.get_or_insert_with(|| TextConstant { + value: String::new(), + start, + }); + constant.value.push_str(text); + } + JoinedPiece::Interpolation(interpolation) => { + values.extend(non_empty(run.take())); + values.push(CpythonNode::Interpolation(interpolation)); + } + } + } + values.extend(non_empty(run)); + values +} + +fn non_empty<'a>(run: Option) -> Option> { + run.filter(|constant| !constant.value.is_empty()) + .map(CpythonNode::Text) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs new file mode 100644 index 0000000000..520323cac8 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs @@ -0,0 +1,10 @@ +//! The line starts of a source whose newlines are already translated. + +use crate::cpython_tree::models::SourceLines; + +/// Where each `\n`-terminated line of `source` starts. +pub(crate) fn source_lines(source: &str) -> SourceLines { + let mut starts: Vec = vec![0]; + starts.extend(source.match_indices('\n').map(|(index, _)| index + 1)); + SourceLines { starts } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs new file mode 100644 index 0000000000..b6ef371a6a --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod children; +pub(crate) mod joined; +pub(crate) mod lines; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/constants.rs b/crates/sqlbuild-pysource/src/cpython_tree/constants.rs new file mode 100644 index 0000000000..8d13b92cfd --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/constants.rs @@ -0,0 +1,4 @@ +//! Constants of CPython's parse of a source text. + +/// The byte order mark CPython rejects as a non-printable character at the start of a `str` source. +pub(crate) const BYTE_ORDER_MARK: char = '\u{feff}'; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs new file mode 100644 index 0000000000..2c892bbf20 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs @@ -0,0 +1,29 @@ +//! The `values` CPython gives an f-string's `JoinedStr` or a t-string's `TemplateStr`. + +use ruff_python_ast::{Expr, FStringPart}; + +use crate::cpython_tree::_helpers::joined::{JoinedPiece, element_pieces, merged_values}; +use crate::cpython_tree::models::CpythonNode; + +/// The merged constants and interpolations of an f-string or t-string; empty for other nodes. +pub(crate) fn joined_values(expression: &Expr) -> Vec> { + let pieces: Vec> = match expression { + Expr::FString(fstring) => fstring + .value + .iter() + .flat_map(|part| match part { + FStringPart::Literal(literal) => { + vec![JoinedPiece::Text(&literal.value, literal.range.start())] + } + FStringPart::FString(part) => element_pieces(&part.elements), + }) + .collect(), + Expr::TString(tstring) => tstring + .value + .iter() + .flat_map(|part| element_pieces(&part.elements)) + .collect(), + _ => Vec::new(), + }; + merged_values(pieces) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs new file mode 100644 index 0000000000..67ecd869e5 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod joined_values; +pub(crate) mod parse_cpython_module; +pub(crate) mod source_position; +pub(crate) mod walk_breadth_first; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs new file mode 100644 index 0000000000..935723d9b7 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs @@ -0,0 +1,28 @@ +//! Parse a module the way CPython's `ast.parse` accepts a `str` source. + +use ruff_python_ast::PythonVersion; +use ruff_python_parser::{Mode, ParseOptions, Parsed}; + +use crate::cpython_tree::_helpers::lines::source_lines; +use crate::cpython_tree::constants::BYTE_ORDER_MARK; +use crate::cpython_tree::models::ParsedSource; + +/// `source` parsed for the `(major, minor)` Python with CPython's newline translation, if valid. +pub(crate) fn parse_cpython_module(source: &str, python_version: (u8, u8)) -> Option { + if source.starts_with(BYTE_ORDER_MARK) { + return None; + } + let translated: String = source.replace("\r\n", "\n").replace('\r', "\n"); + let (major, minor) = python_version; + let options: ParseOptions = + ParseOptions::from(Mode::Module).with_target_version(PythonVersion { major, minor }); + let parsed: Parsed<_> = ruff_python_parser::parse_unchecked(&translated, options); + if !parsed.has_no_syntax_errors() { + return None; + } + let module = parsed.try_into_module()?.into_syntax(); + Some(ParsedSource { + module, + lines: source_lines(&translated), + }) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs new file mode 100644 index 0000000000..d4e918ba05 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs @@ -0,0 +1,17 @@ +//! CPython's `lineno` and `col_offset` for a byte offset. + +use ruff_text_size::TextSize; + +use crate::cpython_tree::models::SourceLines; + +/// The 1-based line and the UTF-8 byte column of `offset`, as CPython reports a node start. +pub(crate) fn source_position(lines: &SourceLines, offset: TextSize) -> (usize, usize) { + let offset: usize = offset.to_usize(); + let line: usize = lines.starts.partition_point(|start| *start <= offset); + let start: usize = lines + .starts + .get(line.saturating_sub(1)) + .copied() + .unwrap_or_default(); + (line, offset - start) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs new file mode 100644 index 0000000000..4307efbf3c --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs @@ -0,0 +1,17 @@ +//! CPython's `ast.walk` order over mirrored nodes. + +use std::collections::VecDeque; + +use crate::cpython_tree::_helpers::children::children; +use crate::cpython_tree::models::CpythonNode; + +/// `roots` and every node below them, breadth first with children in CPython's `_fields` order. +pub(crate) fn walk_breadth_first<'a>(roots: Vec>) -> Vec> { + let mut queue: VecDeque> = VecDeque::from(roots); + let mut visited: Vec> = Vec::new(); + while let Some(node) = queue.pop_front() { + queue.extend(children(&node)); + visited.push(node); + } + visited +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/mod.rs new file mode 100644 index 0000000000..2ffdf440f3 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/models.rs b/crates/sqlbuild-pysource/src/cpython_tree/models.rs new file mode 100644 index 0000000000..bcbe85e2fd --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/models.rs @@ -0,0 +1,52 @@ +//! CPython `ast` nodes mirrored over the ruff tree, and the line starts of a source. + +use ruff_python_ast::{ + Comprehension, ElifElseClause, ExceptHandler, Expr, InterpolatedElement, + InterpolatedStringFormatSpec, Keyword, MatchCase, ModModule, Parameter, Parameters, Pattern, + Stmt, TypeParam, WithItem, +}; +use ruff_text_size::TextSize; + +/// One node of the tree CPython's `ast.parse` builds, borrowed from the ruff node it mirrors. +#[derive(Clone, Debug)] +pub(crate) enum CpythonNode<'a> { + Stmt(&'a Stmt), + /// The `If` an `elif` clause becomes; the clauses after it form its `orelse`. + Elif(&'a [ElifElseClause]), + Expr(&'a Expr), + Arguments(&'a Parameters), + Arg(&'a Parameter), + Keyword(&'a Keyword), + WithItem(&'a WithItem), + MatchCase(&'a MatchCase), + Pattern(&'a Pattern), + Comprehension(&'a Comprehension), + ExceptHandler(&'a ExceptHandler), + TypeParam(&'a TypeParam), + /// A string `Constant` of an f-string, t-string or format spec, merged as CPython merges it. + Text(TextConstant), + /// A `FormattedValue` or `Interpolation`. + Interpolation(&'a InterpolatedElement), + /// The `JoinedStr` of a format spec. + FormatSpec(&'a InterpolatedStringFormatSpec), +} + +/// A merged string constant and the offset CPython locates it at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TextConstant { + pub(crate) value: String, + pub(crate) start: TextSize, +} + +/// A module CPython accepts, parsed from its source with newlines translated as CPython does. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ParsedSource { + pub(crate) module: ModModule, + pub(crate) lines: SourceLines, +} + +/// The byte offsets where the lines of one newline-translated source start. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SourceLines { + pub(crate) starts: Vec, +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests.rs new file mode 100644 index 0000000000..4df003773b --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_joined_constants.rs"] +mod joined_constants; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs new file mode 100644 index 0000000000..3b6cc1f3c0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs @@ -0,0 +1,70 @@ +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::cpython_tree::tests::test_types::JoinedConstantTestCase; + +/// Expectations recorded from CPython 3.12, 3.13 and 3.14, which agree. +#[test] +fn given_interpolated_strings_when_walking_then_constants_merge_as_in_cpython() { + let test_cases = [ + JoinedConstantTestCase { + description: "debug text merges with the literal before it", + source: "f\"a{q=}b\"", + expected_constants: &[("aq=", 1, 2), ("b", 1, 7)], + }, + JoinedConstantTestCase { + description: "debug text keeps its spaces and starts after the brace", + source: "f\"{ q = }\"", + expected_constants: &[(" q = ", 1, 3)], + }, + JoinedConstantTestCase { + description: "implicit concatenation merges across parts", + source: "f\"a\" f\"b{x}\" \"c\" f\"d\"", + expected_constants: &[("ab", 1, 2), ("cd", 1, 13)], + }, + JoinedConstantTestCase { + description: "an empty plain string starts a run and an empty f-string does not", + source: "x = [\"\" f\"a{x}\", f\"{x}\" f\"\" \"b\"]", + expected_constants: &[("a", 1, 5), ("b", 1, 28)], + }, + JoinedConstantTestCase { + description: "format specs are walked one level deeper", + source: "f\"{x:>{y}d}\" \"\\u00e9\"", + expected_constants: &[("é", 1, 13), (">", 1, 5), ("d", 1, 9)], + }, + JoinedConstantTestCase { + description: "lines end at carriage returns", + source: "x = 1\r\ny = f\"\"\"a\r{x}\"\"\"", + expected_constants: &[("a\n", 2, 8)], + }, + ]; + for test_case in test_cases { + let parsed: Option = parse_cpython_module(test_case.source, (3, 12)); + let statements: Vec> = parsed + .iter() + .flat_map(|parsed| parsed.module.body.iter().map(CpythonNode::Stmt)) + .collect(); + let constants: Vec<(String, usize, usize)> = walk_breadth_first(statements) + .into_iter() + .filter_map(|node| { + let CpythonNode::Text(constant) = node else { + return None; + }; + let (line, column) = parsed.as_ref().map_or((0, 0), |parsed| { + source_position(&parsed.lines, constant.start) + }); + Some((constant.value, line, column)) + }) + .collect(); + let found: Vec<(&str, usize, usize)> = constants + .iter() + .map(|(value, line, column)| (value.as_str(), *line, *column)) + .collect(); + assert_eq!( + found, test_case.expected_constants, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs new file mode 100644 index 0000000000..9186ebc495 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct JoinedConstantTestCase { + pub(super) description: &'static str, + pub(super) source: &'static str, + pub(super) expected_constants: &'static [(&'static str, usize, usize)], +} diff --git a/crates/sqlbuild-pysource/src/definition_locations/constants.rs b/crates/sqlbuild-pysource/src/definition_locations/constants.rs new file mode 100644 index 0000000000..6b93361f5d --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/constants.rs @@ -0,0 +1,4 @@ +//! The location reported when no decorator is found. + +/// The line and column offset of a missing decorator. +pub(crate) const MISSING_DECORATOR_LOCATION: (usize, usize) = (1, 1); diff --git a/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs b/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs new file mode 100644 index 0000000000..384afa2f18 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs @@ -0,0 +1,45 @@ +//! Where a named decorator of a function sits in a module source. + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::definition_locations::constants::MISSING_DECORATOR_LOCATION; + +/// The `lineno` and byte `col_offset` of the first `@decorator` in `ast.walk` order, else `(1, 1)`. +#[must_use] +pub fn decorator_location( + contents: &str, + decorator: &str, + python_version: (u8, u8), +) -> (usize, usize) { + let Some(parsed): Option = parse_cpython_module(contents, python_version) else { + return MISSING_DECORATOR_LOCATION; + }; + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let CpythonNode::Stmt(Stmt::FunctionDef(function)) = node else { + continue; + }; + for expression in function.decorator_list.iter().map(|item| &item.expression) { + if decorator_name(expression) == Some(decorator) { + return source_position(&parsed.lines, expression.range().start()); + } + } + } + MISSING_DECORATOR_LOCATION +} + +fn decorator_name(expression: &Expr) -> Option<&str> { + let target: &Expr = match expression { + Expr::Call(call) => &call.func, + _ => expression, + }; + match target { + Expr::Name(name) => Some(name.id.as_str()), + Expr::Attribute(attribute) => Some(attribute.attr.as_str()), + _ => None, + } +} diff --git a/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs b/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs new file mode 100644 index 0000000000..4cb8145dd4 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs @@ -0,0 +1 @@ +pub mod decorator_location; diff --git a/crates/sqlbuild-pysource/src/definition_locations/mod.rs b/crates/sqlbuild-pysource/src/definition_locations/mod.rs new file mode 100644 index 0000000000..9b926d1cd2 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests.rs b/crates/sqlbuild-pysource/src/definition_locations/tests.rs new file mode 100644 index 0000000000..6888c15fdf --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_decorator_location.rs"] +mod decorator_location; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs new file mode 100644 index 0000000000..74c26a1947 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs @@ -0,0 +1,77 @@ +use crate::definition_locations::main::decorator_location::decorator_location; +use crate::definition_locations::tests::test_types::DecoratorLocationTestCase; + +/// Expectations recorded from the CPython `ast` implementation this replaces. +#[test] +fn given_module_when_locating_udf_decorator_then_matches_cpython_position() { + let test_cases = [ + DecoratorLocationTestCase { + description: "a call decorator", + contents: "import sqlbuild\n\n\n@sqlbuild.udf(returns=\"INTEGER\")\ndef add_one(value):\n return value + 1\n", + python_version: (3, 12), + expected_location: (4, 1), + }, + DecoratorLocationTestCase { + description: "a bare decorator in a class, found breadth first", + contents: "class Box:\n @udf\n def method(self):\n pass\n\n@ udf\ndef top():\n pass\n", + python_version: (3, 12), + expected_location: (6, 2), + }, + DecoratorLocationTestCase { + description: "columns count UTF-8 bytes", + contents: "x = \"é\"; y = 1\nif x:\n\t@é.udf()\n\tdef f():\n\t\tpass\n", + python_version: (3, 12), + expected_location: (3, 2), + }, + DecoratorLocationTestCase { + description: "carriage returns end lines", + contents: "x = 1\r@udf\rdef f():\r pass\r", + python_version: (3, 12), + expected_location: (2, 1), + }, + DecoratorLocationTestCase { + description: "a parenthesized decorator starts at its parenthesis", + contents: "@(udf)(a=1)\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "a missing decorator", + contents: "@other\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "a syntax error", + contents: "x = 1\n@udf\ndef f(:\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "syntax newer than the interpreter is a syntax error", + contents: "x = t\"tpl\"\n@udf\ndef f():\n pass\n", + python_version: (3, 13), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "syntax the interpreter supports", + contents: "x = t\"tpl\"\n@udf\ndef f():\n pass\n", + python_version: (3, 14), + expected_location: (2, 1), + }, + DecoratorLocationTestCase { + description: "a leading byte order mark is not printable source", + contents: "\u{feff}@udf\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + ]; + for test_case in test_cases { + assert_eq!( + decorator_location(test_case.contents, "udf", test_case.python_version), + test_case.expected_location, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs b/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs new file mode 100644 index 0000000000..e8b0557873 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct DecoratorLocationTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) python_version: (u8, u8), + pub(super) expected_location: (usize, usize), +} diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index da8c2d78f5..89a638c28a 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -2,5 +2,9 @@ #![forbid(unsafe_code)] +pub(crate) mod cpython_tree; +pub mod definition_locations; +pub mod literal_sql; pub mod macro_modules; pub mod module_parse; +pub mod string_literals; diff --git a/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs b/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs new file mode 100644 index 0000000000..55b31aaa2a --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs @@ -0,0 +1,67 @@ +//! The context parameter, the SQL argument of a call and its literal text. + +use ruff_python_ast::{Expr, ExprCall, Parameters}; + +use crate::cpython_tree::main::joined_values::joined_values; +use crate::cpython_tree::models::CpythonNode; +use crate::literal_sql::constants::{CONTEXT_PARAMETER_NAMES, SQL_PARAMETER_NAME}; + +/// The first well-known context parameter name, else the first parameter, ignoring `*args` and `**kwargs`. +pub(crate) fn context_name(parameters: &Parameters) -> Option<&str> { + let names: Vec<&str> = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .map(|parameter| parameter.parameter.name.as_str()) + .collect(); + names + .iter() + .find(|name| CONTEXT_PARAMETER_NAMES.contains(name)) + .or_else(|| names.first()) + .copied() +} + +/// The first positional argument, else the `sql=` keyword, also inside a `**{...}` literal. +pub(crate) fn sql_argument(call: &ExprCall) -> Option<&Expr> { + if let Some(first) = call.arguments.args.first() { + return Some(first); + } + for keyword in &call.arguments.keywords { + match &keyword.arg { + Some(name) if name.as_str() == SQL_PARAMETER_NAME => return Some(&keyword.value), + Some(_) => {} + None => { + let Expr::Dict(dict) = &keyword.value else { + continue; + }; + for item in &dict.items { + if let Some(Expr::StringLiteral(key)) = &item.key + && key.value.to_str() == SQL_PARAMETER_NAME + { + return Some(&item.value); + } + } + } + } + } + None +} + +/// A string literal's text, or an f-string's with numbered placeholders for its values. +pub(crate) fn literal_text(expression: &Expr, placeholder: &str) -> Option { + match expression { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + Expr::FString(_) => Some( + joined_values(expression) + .iter() + .enumerate() + .map(|(index, value)| match value { + CpythonNode::Text(constant) => constant.value.clone(), + _ => format!("{placeholder}{index}"), + }) + .collect(), + ), + _ => None, + } +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs new file mode 100644 index 0000000000..0bcb87fca2 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod arguments; diff --git a/crates/sqlbuild-pysource/src/literal_sql/constants.rs b/crates/sqlbuild-pysource/src/literal_sql/constants.rs new file mode 100644 index 0000000000..f1177cfe21 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/constants.rs @@ -0,0 +1,10 @@ +//! The context methods and parameters that carry SQL text. + +/// Context methods whose first argument is SQL. +pub(crate) const SQL_METHOD_NAMES: [&str; 2] = ["query", "execute_sql"]; + +/// The keyword name of the SQL argument. +pub(crate) const SQL_PARAMETER_NAME: &str = "sql"; + +/// Parameter names preferred as the context over the first parameter. +pub(crate) const CONTEXT_PARAMETER_NAMES: [&str; 4] = ["ctx", "context", "_ctx", "hook_context"]; diff --git a/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs b/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs new file mode 100644 index 0000000000..7d4a8a9019 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs @@ -0,0 +1,61 @@ +//! Literal SQL passed to the context's SQL methods inside one function. + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::literal_sql::_helpers::arguments::{context_name, literal_text, sql_argument}; +use crate::literal_sql::constants::SQL_METHOD_NAMES; +use crate::literal_sql::models::{FunctionSource, LiteralSql}; + +/// The context's `query`/`execute_sql` calls with literal SQL, in `ast.walk` order; none when unparsable. +#[must_use] +pub fn literal_sql_calls(source: &FunctionSource<'_>) -> Vec { + let Some(parsed): Option = + parse_cpython_module(source.text, source.python_version) + else { + return Vec::new(); + }; + let Some(Stmt::FunctionDef(definition)) = parsed.module.body.first() else { + return Vec::new(); + }; + let Some(context) = context_name(&definition.parameters) else { + return Vec::new(); + }; + let mut calls: Vec = Vec::new(); + for node in walk_breadth_first( + parsed + .module + .body + .iter() + .take(1) + .map(CpythonNode::Stmt) + .collect(), + ) { + let CpythonNode::Expr(Expr::Call(call)) = node else { + continue; + }; + let Expr::Attribute(attribute) = call.func.as_ref() else { + continue; + }; + let is_context_method: bool = SQL_METHOD_NAMES.contains(&attribute.attr.as_str()) + && matches!(attribute.value.as_ref(), Expr::Name(name) if name.id.as_str() == context); + if !is_context_method { + continue; + } + if let Some(sql) = + sql_argument(call).and_then(|argument| literal_text(argument, source.placeholder)) + { + let (line, _) = source_position(&parsed.lines, call.range().start()); + calls.push(LiteralSql { + method: attribute.attr.to_string(), + sql, + line: source.start_line + line - 1, + }); + } + } + calls +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs new file mode 100644 index 0000000000..a1db6e1556 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs @@ -0,0 +1 @@ +pub mod literal_sql_calls; diff --git a/crates/sqlbuild-pysource/src/literal_sql/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/literal_sql/models.rs b/crates/sqlbuild-pysource/src/literal_sql/models.rs new file mode 100644 index 0000000000..e9db38c94c --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/models.rs @@ -0,0 +1,19 @@ +//! A function source to read literal SQL from, and the literal SQL calls found in it. + +/// A dedented function source, its first line in the file, and the interpreter it runs on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FunctionSource<'a> { + pub text: &'a str, + pub start_line: usize, + /// The prefix of the numbered stand-ins for interpolated f-string values. + pub placeholder: &'a str, + pub python_version: (u8, u8), +} + +/// One context SQL call with literal SQL: the method, the SQL text and its line in the file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiteralSql { + pub method: String, + pub sql: String, + pub line: usize, +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests.rs b/crates/sqlbuild-pysource/src/literal_sql/tests.rs new file mode 100644 index 0000000000..18c3b81c65 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_literal_sql_calls.rs"] +mod literal_sql_calls; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs b/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs new file mode 100644 index 0000000000..b7f13f05e1 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs @@ -0,0 +1,67 @@ +use crate::literal_sql::main::literal_sql_calls::literal_sql_calls; +use crate::literal_sql::models::{FunctionSource, LiteralSql}; +use crate::literal_sql::tests::test_types::LiteralSqlTestCase; + +const PLACEHOLDER: &str = "__value_"; + +/// Expectations recorded from the CPython 3.12 `ast` implementation this replaces. +#[test] +fn given_function_source_when_reading_literal_sql_then_matches_cpython_walk() { + let test_cases = [ + LiteralSqlTestCase { + description: "positional and keyword SQL in walk order", + source: "def task(ctx):\n ctx.execute_sql(sql=\"delete from orders\")\n if ready:\n ctx.query(\"select * from customers\")\n ctx.query(f\"select * from {table} where id = {key!r:>{width}}\")\n", + expected_calls: &[ + ("execute_sql", "delete from orders", 11), + ("query", "select * from __value_1 where id = __value_3", 14), + ("query", "select * from customers", 13), + ], + }, + LiteralSqlTestCase { + description: "a well-known context name wins over the first parameter", + source: "def hook(self, context):\n self.query(\"select 1\")\n context.query(**{\"sql\": \"select 2\"})\n", + expected_calls: &[("query", "select 2", 12)], + }, + LiteralSqlTestCase { + description: "implicit concatenation, debug text and bytes", + source: "async def check(ctx, /):\n ctx.query(\"select \" \"1\")\n ctx.query(f\"{ value = }\" \"tail\")\n ctx.query(b\"select 3\")\n ctx.query(sql)\n", + expected_calls: &[ + ("query", "select 1", 11), + ("query", " value = __value_1tail", 12), + ], + }, + LiteralSqlTestCase { + description: "nested calls come after their parents", + source: "def task(ctx):\n ctx.query(ctx.query(\"inner\"))\n ctx.query(\"outer\")\n", + expected_calls: &[("query", "outer", 12), ("query", "inner", 11)], + }, + LiteralSqlTestCase { + description: "a source that does not parse", + source: "def task(ctx):\n ctx.query(\"select 1\"\n", + expected_calls: &[], + }, + LiteralSqlTestCase { + description: "a source that is not a function", + source: "owner = lambda ctx: ctx.query(\"select 1\")\n", + expected_calls: &[], + }, + LiteralSqlTestCase { + description: "a function without parameters", + source: "def task():\n ctx.query(\"select 1\")\n", + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let calls: Vec = literal_sql_calls(&FunctionSource { + text: test_case.source, + start_line: 10, + placeholder: PLACEHOLDER, + python_version: (3, 12), + }); + let found: Vec<(&str, &str, usize)> = calls + .iter() + .map(|call| (call.method.as_str(), call.sql.as_str(), call.line)) + .collect(); + assert_eq!(found, test_case.expected_calls, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs b/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs new file mode 100644 index 0000000000..a8bc6b777c --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct LiteralSqlTestCase { + pub(super) description: &'static str, + pub(super) source: &'static str, + pub(super) expected_calls: &'static [(&'static str, &'static str, usize)], +} diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs new file mode 100644 index 0000000000..0daabc76c6 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs @@ -0,0 +1,22 @@ +//! Whether one string constant names a renamed model or column. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::string_literals::_helpers::python_re::{has_word, python_lower, sql_like}; +use crate::string_literals::models::StringQuery; + +/// A name equal to the stripped string, or a whole word of SQL-like text; a context must appear too. +pub(crate) fn flags(value: &str, query: &StringQuery<'_>) -> bool { + let text: Vec = value.chars().collect(); + let is_sql_like: bool = sql_like(&text); + if let Some(context) = query.context + && !(is_sql_like && has_word(&text, context)) + { + return false; + } + let stripped: String = python_lower(python_strip(value)); + query + .names + .iter() + .any(|name| stripped == python_lower(name) || (is_sql_like && has_word(&text, name))) +} diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs new file mode 100644 index 0000000000..0cf7c1e49c --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod flags; +pub(crate) mod python_re; diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs new file mode 100644 index 0000000000..3cd6e3b72e --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs @@ -0,0 +1,132 @@ +//! Python `re` with `re.IGNORECASE` and `str.lower` for the refactoring string checks. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; + +use crate::string_literals::constants::{ + EXTRA_CASES, IDENTIFIER_PUNCTUATION, SQL_KEYWORDS, UNCASED_IN_PYTHON, +}; + +/// `re.search(r"\b(select|from|join|insert|update)\b", text, re.I) is not None`. +pub(crate) fn sql_like(text: &[char]) -> bool { + let keywords: Vec> = SQL_KEYWORDS + .iter() + .map(|keyword| keyword.chars().collect()) + .collect(); + (0..=text.len()).any(|start| word_boundary(text, start) && keyword_at(text, start, &keywords)) +} + +fn keyword_at(text: &[char], start: usize, keywords: &[Vec]) -> bool { + keywords.iter().any(|keyword| { + matches_at(text, start, keyword) && word_boundary(text, start + keyword.len()) + }) +} + +/// `re.search(rf"(? bool { + let word: Vec = word.chars().collect(); + if word.len() > text.len() { + return false; + } + (0..=text.len() - word.len()).any(|start| { + let end: usize = start + word.len(); + (start == 0 || !identifier_character(text[start - 1])) + && matches_at(text, start, &word) + && text + .get(end) + .is_none_or(|next| !identifier_character(*next)) + }) +} + +/// Python's `text.lower()`, keeping code points CPython 3.12 does not case-map. +pub(crate) fn python_lower(text: &str) -> String { + let mut lowered: String = String::with_capacity(text.len()); + let mut segment_start: usize = 0; + for (index, character) in text.char_indices() { + if uncased_in_python(character) { + lowered.push_str(&text[segment_start..index].to_lowercase()); + lowered.push(character); + segment_start = index + character.len_utf8(); + } + } + lowered.push_str(&text[segment_start..].to_lowercase()); + lowered +} + +fn matches_at(text: &[char], start: usize, pattern: &[char]) -> bool { + pattern.iter().enumerate().all(|(offset, expected)| { + text.get(start + offset) + .is_some_and(|found| literal_matches(*expected, *found)) + }) +} + +/// Python's `\b` for `str` patterns: a word character on exactly one side. +fn word_boundary(text: &[char], at: usize) -> bool { + let before: bool = at > 0 + && text + .get(at - 1) + .is_some_and(|character| word_character(*character)); + let after: bool = text + .get(at) + .is_some_and(|character| word_character(*character)); + before != after +} + +fn word_character(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// `[A-Za-z0-9_$]` under `re.IGNORECASE`: the folded character is in the folded class. +fn identifier_character(character: char) -> bool { + let folded: char = sre_lower(character); + identifier_base(folded) + || EXTRA_CASES.iter().any(|(code, extras)| { + char::from_u32(*code).is_some_and(identifier_base) + && extras.contains(&u32::from(folded)) + }) +} + +fn identifier_base(character: char) -> bool { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || IDENTIFIER_PUNCTUATION.contains(&character) +} + +/// One pattern literal against one character under `re.IGNORECASE`, as `sre` compiles it. +fn literal_matches(pattern: char, character: char) -> bool { + if !sre_cased(pattern) { + return character == pattern; + } + let lower: char = sre_lower(pattern); + let folded: char = sre_lower(character); + folded == lower || extra_cases(lower).contains(&u32::from(folded)) +} + +fn extra_cases(lower: char) -> &'static [u32] { + EXTRA_CASES + .binary_search_by_key(&u32::from(lower), |(code, _)| *code) + .map_or(&[], |index| EXTRA_CASES[index].1) +} + +fn uncased_in_python(character: char) -> bool { + let code: u32 = u32::from(character); + UNCASED_IN_PYTHON + .iter() + .any(|(first, last)| (*first..=*last).contains(&code)) +} + +/// `_sre.unicode_tolower`: the first code point of the full lowercase mapping. +fn sre_lower(character: char) -> char { + if uncased_in_python(character) { + return character; + } + character.to_lowercase().next().unwrap_or(character) +} + +/// `_sre.unicode_iscased`: the first code point of either case mapping differs. +fn sre_cased(character: char) -> bool { + if uncased_in_python(character) { + return false; + } + sre_lower(character) != character + || character.to_uppercase().next().unwrap_or(character) != character +} diff --git a/crates/sqlbuild-pysource/src/string_literals/constants.rs b/crates/sqlbuild-pysource/src/string_literals/constants.rs new file mode 100644 index 0000000000..260d711f8d --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/constants.rs @@ -0,0 +1,75 @@ +//! Python `re` facts for the refactoring string checks, from CPython 3.12. + +/// The keywords of `\b(select|from|join|insert|update)\b`, which marks a string as SQL-like. +pub(crate) const SQL_KEYWORDS: [&str; 5] = ["select", "from", "join", "insert", "update"]; + +/// `[A-Za-z0-9_$]` before `re.IGNORECASE` folding: identifier characters around a name. +pub(crate) const IDENTIFIER_PUNCTUATION: [char; 2] = ['_', '$']; + +/// Code points Rust case-maps whose case CPython 3.12's Unicode data does not know. +pub(crate) const UNCASED_IN_PYTHON: [(u32, u32); 10] = [ + (0x019B, 0x019B), + (0x0264, 0x0264), + (0x1C89, 0x1C8A), + (0xA7CB, 0xA7CF), + (0xA7D2, 0xA7D5), + (0xA7DA, 0xA7DC), + (0x10D50, 0x10D65), + (0x10D70, 0x10D85), + (0x16EA0, 0x16EB8), + (0x16EBB, 0x16ED3), +]; + +/// `re._casefix._EXTRA_CASES`: lowercase code points that share an uppercase with others. +pub(crate) const EXTRA_CASES: [(u32, &[u32]); 50] = [ + (0x0069, &[0x0131]), + (0x0073, &[0x017F]), + (0x00B5, &[0x03BC]), + (0x0131, &[0x0069]), + (0x017F, &[0x0073]), + (0x0345, &[0x03B9, 0x1FBE]), + (0x0390, &[0x1FD3]), + (0x03B0, &[0x1FE3]), + (0x03B2, &[0x03D0]), + (0x03B5, &[0x03F5]), + (0x03B8, &[0x03D1]), + (0x03B9, &[0x0345, 0x1FBE]), + (0x03BA, &[0x03F0]), + (0x03BC, &[0x00B5]), + (0x03C0, &[0x03D6]), + (0x03C1, &[0x03F1]), + (0x03C2, &[0x03C3]), + (0x03C3, &[0x03C2]), + (0x03C6, &[0x03D5]), + (0x03D0, &[0x03B2]), + (0x03D1, &[0x03B8]), + (0x03D5, &[0x03C6]), + (0x03D6, &[0x03C0]), + (0x03F0, &[0x03BA]), + (0x03F1, &[0x03C1]), + (0x03F5, &[0x03B5]), + (0x0432, &[0x1C80]), + (0x0434, &[0x1C81]), + (0x043E, &[0x1C82]), + (0x0441, &[0x1C83]), + (0x0442, &[0x1C84, 0x1C85]), + (0x044A, &[0x1C86]), + (0x0463, &[0x1C87]), + (0x1C80, &[0x0432]), + (0x1C81, &[0x0434]), + (0x1C82, &[0x043E]), + (0x1C83, &[0x0441]), + (0x1C84, &[0x0442, 0x1C85]), + (0x1C85, &[0x0442, 0x1C84]), + (0x1C86, &[0x044A]), + (0x1C87, &[0x0463]), + (0x1C88, &[0xA64B]), + (0x1E61, &[0x1E9B]), + (0x1E9B, &[0x1E61]), + (0x1FBE, &[0x0345, 0x03B9]), + (0x1FD3, &[0x0390]), + (0x1FE3, &[0x03B0]), + (0xA64B, &[0x1C88]), + (0xFB05, &[0xFB06]), + (0xFB06, &[0xFB05]), +]; diff --git a/crates/sqlbuild-pysource/src/string_literals/main/mod.rs b/crates/sqlbuild-pysource/src/string_literals/main/mod.rs new file mode 100644 index 0000000000..e3ad60f534 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/main/mod.rs @@ -0,0 +1 @@ +pub mod string_literal_locations; diff --git a/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs b/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs new file mode 100644 index 0000000000..49d58000e0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs @@ -0,0 +1,38 @@ +//! String constants of a Python module that name a renamed model or column. + +use ruff_python_ast::Expr; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::string_literals::_helpers::flags::flags; +use crate::string_literals::models::StringQuery; + +/// The 1-based line and column of each flagged `str` constant in `ast.walk` order; none when unparsable. +#[must_use] +pub fn string_literal_locations( + contents: &str, + query: &StringQuery<'_>, + python_version: (u8, u8), +) -> Vec<(usize, usize)> { + let Some(parsed): Option = parse_cpython_module(contents, python_version) else { + return Vec::new(); + }; + let mut locations: Vec<(usize, usize)> = Vec::new(); + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let (value, start): (&str, TextSize) = match &node { + CpythonNode::Expr(Expr::StringLiteral(literal)) => { + (literal.value.to_str(), literal.range().start()) + } + CpythonNode::Text(constant) => (&constant.value, constant.start), + _ => continue, + }; + if flags(value, query) { + let (line, column) = source_position(&parsed.lines, start); + locations.push((line, column + 1)); + } + } + locations +} diff --git a/crates/sqlbuild-pysource/src/string_literals/mod.rs b/crates/sqlbuild-pysource/src/string_literals/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/string_literals/models.rs b/crates/sqlbuild-pysource/src/string_literals/models.rs new file mode 100644 index 0000000000..7ec1f6588d --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/models.rs @@ -0,0 +1,8 @@ +//! The names a refactoring looks for in Python string constants. + +/// Renamed names, and the name a SQL-like string must also mention, if any. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StringQuery<'a> { + pub names: &'a [String], + pub context: Option<&'a str>, +} diff --git a/crates/sqlbuild-pysource/src/string_literals/tests.rs b/crates/sqlbuild-pysource/src/string_literals/tests.rs new file mode 100644 index 0000000000..cd9100cc37 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_string_literal_locations.rs"] +mod string_literal_locations; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs b/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs new file mode 100644 index 0000000000..5dfacadfa0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs @@ -0,0 +1,69 @@ +use crate::string_literals::main::string_literal_locations::string_literal_locations; +use crate::string_literals::models::StringQuery; +use crate::string_literals::tests::test_types::StringLocationTestCase; + +/// Expectations recorded from the CPython 3.12 `ast` and `re` implementation this replaces. +#[test] +fn given_module_when_finding_named_strings_then_matches_cpython_constants() { + let test_cases = [ + StringLocationTestCase { + description: "an exact name, a SQL word and a docstring", + contents: "\"\"\"orders\"\"\"\nquery = \"SELECT * FROM orders o\"\nother = \"orders_archive\"\nname = \" ORDERS \"\n", + names: &["orders"], + context: None, + expected_locations: &[(1, 1), (2, 9), (4, 8)], + }, + StringLocationTestCase { + description: "f-string constants are merged and located at their first piece", + contents: "sql = f\"select {cols} from orders\" \" where\"\nlabel = f\"{orders=}\"\n", + names: &["orders"], + context: None, + expected_locations: &[(1, 22)], + }, + StringLocationTestCase { + description: "a context must appear in SQL-like text", + contents: "a = \"select id from orders\"\nb = \"select id from customers\"\nc = \"id\"\n", + names: &["id"], + context: Some("orders"), + expected_locations: &[(1, 5)], + }, + StringLocationTestCase { + description: "Python ignore-case folds the long s and dotted capital I", + contents: "a = \"Å¿elect x from t\"\nb = \"SELECT Ä°d FROM t\"\nc = \"select $id from t\"\n", + names: &["id", "x"], + context: None, + expected_locations: &[(1, 5), (2, 5)], + }, + StringLocationTestCase { + description: "an unparsable file", + contents: "x = \"orders\"\ndef f(:\n", + names: &["orders"], + context: None, + expected_locations: &[], + }, + StringLocationTestCase { + description: "match patterns and annotations", + contents: "match x:\n case {\"orders\": 1}:\n pass\ndef f(a: \"orders\"): pass\n", + names: &["orders"], + context: None, + expected_locations: &[(2, 11), (4, 10)], + }, + ]; + for test_case in test_cases { + let names: Vec = test_case + .names + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let query: StringQuery<'_> = StringQuery { + names: &names, + context: test_case.context, + }; + assert_eq!( + string_literal_locations(test_case.contents, &query, (3, 12)), + test_case.expected_locations, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs b/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs new file mode 100644 index 0000000000..248ad43efa --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(super) struct StringLocationTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) names: &'static [&'static str], + pub(super) context: Option<&'static str>, + pub(super) expected_locations: &'static [(usize, usize)], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 23790b3674..c578031f6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -5,8 +5,15 @@ use pyo3::{pyfunction, wrap_pyfunction}; use serde_json::{Value, json}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; -use sqlbuild_analysis::explicit_references::models::{ReferenceReport, ReferenceRequest}; +use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::models::{ + ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, ReferenceReport, + ReferenceRequest, +}; +use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; +use sqlbuild_pysource::literal_sql::models::FunctionSource; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; @@ -44,6 +51,66 @@ fn check_native_explicit_references(py: Python<'_>, request_json: &str) -> PyRes }) } +/// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. +#[pyfunction] +fn python_literal_sql_calls( + py: Python<'_>, + source: &str, + start_line: usize, + placeholder: &str, +) -> Vec<(String, String, usize)> { + let version = py.version_info(); + literal_sql_calls(&FunctionSource { + text: source, + start_line, + placeholder, + python_version: (version.major, version.minor), + }) + .into_iter() + .map(|call| (call.method, call.sql, call.line)) + .collect() +} + +/// `(audit name, target, gated, read)` for one attached-audit ordering edge. +type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); +/// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. +type HookRow = ( + String, + ResourceKey, + ResourceKey, + &'static str, + Option, +); + +/// The attached-audit gate edges and hook-read edges of one project's facts, in Python's order. +#[pyfunction] +fn native_execution_gate_edges(request_json: &str) -> PyResult<(Vec, Vec)> { + let request: ExecutionGateRequest = serde_json::from_str(request_json).map_err(|error| { + compiler_error(native_failure( + EXPLICIT_REFERENCES_CONTEXT, + &error.to_string(), + )) + })?; + let edges: ExecutionGateEdges = execution_gate_edges(&request); + Ok(( + edges.audit_gates.into_iter().map(gate_row).collect(), + edges.hook_reads.into_iter().map(hook_row).collect(), + )) +} + +fn gate_row(edge: GateEdge) -> GateRow { + (edge.audit_name, edge.target, edge.gated, edge.read) +} + +fn hook_row(edge: HookEdge) -> HookRow { + let hook_type: &'static str = match edge.hook_type { + HookType::Python => "Python", + HookType::Sql => "SQL", + HookType::InlineSql => "inline SQL", + }; + (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) +} + fn report_value(report: &ReferenceReport) -> PyResult { serde_json::to_value(report).map_err(|error| { compiler_error(native_failure( @@ -55,5 +122,7 @@ fn report_value(report: &ReferenceReport) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(check_native_explicit_references, module)?)?; + module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; + module.add_function(wrap_pyfunction!(native_execution_gate_edges, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index b398ef704f..1fb8cc1b19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -1,9 +1,12 @@ //! Validate, fingerprint and analyze SQL queries for the Python compiler. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRow}; -use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::analysis::conversions; +use crate::bindings::_helpers::boundary::panics::{compiler_error, value_error}; +use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -55,12 +58,17 @@ fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::validations_json(request_json) - }) - .map_err(value_error) +/// Each request validated on its own schema, as binding diagnostic rows mapped to its SQL. +#[pyfunction] +fn schema_validation_rows( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + dialect: &str, + #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, +) -> PyResult>> { + let inner: &sqlbuild_analysis::semantic_validation::models::ProjectCatalog = &catalog.inner; + py.compiler_detach(|| inner.schema_validation_rows(dialect, requests)) + .map_err(compiler_error) } #[pyfunction] @@ -113,7 +121,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(function_definition_hash, module)?)?; module.add_function(wrap_pyfunction!(normalize_query_text, module)?)?; module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; - module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; + module.add_function(wrap_pyfunction!(schema_validation_rows, module)?)?; module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 3cb764a981..8d9c267653 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; -const DISCOVER_MODULE: &str = "sqlbuild.compiler.discovery.main.discover"; +const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs new file mode 100644 index 0000000000..29a3595ccb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -0,0 +1,173 @@ +//! Planner-ready project assembly: diagnostics collection, the macro bridge and explicit references. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; +use crate::bindings::_helpers::compile_driver::project::lineage_mode; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; + +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const RECORD_ANALYSES_MODULE: &str = + "sqlbuild.compiler.compiled_project.main._record_model_analyses"; +const EXPLICIT_REFERENCES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.explicit_references.checks"; +const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; +const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge"; + +const OPTIONAL: [&str; 5] = [ + "inference_profile", + "skip_column_inference", + "column_lineage_mode", + "analysis_cache_dir", + "analysis_model_names", +]; + +/// The keyword arguments of one project assembly. +pub(crate) struct AssemblyRequest { + pub(crate) inputs: Py, + pub(crate) inference_profile: Py, + pub(crate) skip_column_inference: Py, + pub(crate) column_lineage_mode: Py, + pub(crate) analysis_cache_dir: Py, + pub(crate) analysis_model_names: Py, +} + +/// Convert compile inputs into the planner-ready project view. +#[pyfunction] +#[pyo3(signature = (**options))] +fn assemble_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "assemble_project", options, (&["inputs"], &OPTIONAL))?; + let request: AssemblyRequest = AssemblyRequest { + inputs: arguments.value("inputs")?.unbind(), + inference_profile: arguments.value("inference_profile")?.unbind(), + skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), + column_lineage_mode: lineage_mode(&arguments)?.unbind(), + analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), + analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + }; + let collector: Bound<'py, PyAny> = + imported(py, COLLECTOR_MODULE, "collect_compile_diagnostics")?.call0()?; + let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + within(&collector, |violations| { + Ok((referenced_project(py, request)?, violations.clone())) + })?; + let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + if !collected.is_truthy()? { + return Ok(built); + } + let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[("project", built), ("diagnostics", diagnostics.into_any())], + ) +} + +/// `first` followed by `second`, as one tuple. +pub(crate) fn chained<'py>( + first: &Bound<'py, PyAny>, + second: &Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new( + first.py(), + first + .try_iter()? + .chain(second.try_iter()?) + .collect::>>()?, + ) +} + +/// The undecorated `assemble_project`. +fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult> { + let inputs: Bound<'_, PyAny> = request.inputs.bind(py).clone(); + let project: Bound<'_, PyAny> = if imported(py, ACTIVE_BRIDGE_MODULE, "active_macro_bridge")? + .call0()? + .is_none() + { + let stage: Bound<'_, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + assembled_project(arguments.py(), &request).map(Bound::unbind) + }, + )?; + call_with( + &imported(py, RUN_BRIDGE_MODULE, "run_with_macro_bridge")?, + &[("stage", stage.into_any())], + )? + } else { + assembled_project(py, &request)? + }; + let project: Bound<'_, PyAny> = + with_native_project(&project, &inputs.getattr("native_project")?)?; + let references: Bound<'_, PyAny> = call_with( + &imported(py, EXPLICIT_REFERENCES_MODULE, "explicit_reference_report")?, + &[ + ("project", project.clone()), + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ( + "enforce_explicit", + inputs + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + let reference_diagnostics: Bound<'_, PyAny> = references.getattr("diagnostics")?; + let unmatched: Bound<'_, PyAny> = references.getattr("unmatched")?; + if !reference_diagnostics.is_truthy()? && !unmatched.is_truthy()? { + return Ok(project); + } + let diagnostics: Bound<'_, PyTuple> = + chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[ + ("project", project), + ("diagnostics", diagnostics.into_any()), + ("unmatched_literal_sql_relations", unmatched), + ], + ) +} + +/// Hand the compile's native project the assembled analyses and keep it on the project. +fn with_native_project<'py>( + project: &Bound<'py, PyAny>, + native_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + if native_project.is_none() { + return Ok(project.clone()); + } + call_with( + &imported(py, RECORD_ANALYSES_MODULE, "record_model_analyses")?, + &[ + ("project", native_project.clone()), + ("models", project.getattr("models")?), + ], + )?; + if project.getattr("native_project")?.is(native_project) { + return Ok(project.clone()); + } + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[ + ("project", project.clone()), + ("native_project", native_project.clone()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_project, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs new file mode 100644 index 0000000000..3a43e2bbde --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -0,0 +1,430 @@ +//! Project assembly's inference profile, binding catalog, source shapes and model analysis. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; + +use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; + +use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::python_calls::{ + call_with, imported, keywords, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +const NATIVE_DECLARATIONS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.assembly.native_declarations"; +const IDENTIFIER_CASE_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._identifier_case"; +const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; +pub(crate) const NATIVE_STAGES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.native_stages.assembly"; +const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; +pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const PROFILING_CPU_MODULE: &str = "sqlbuild.compiler.profiling.main._record_cpu"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const GENERIC_DIALECT: &str = "generic"; +const UDF_PREFIX: &str = "__sqlbuild_udf_"; + +/// The analysis state every assembled model and the project share. +pub(crate) struct AnalysisContext<'py> { + pub(crate) profile: Bound<'py, PyAny>, + /// Python's `settings.sql_analysis and not skip_column_inference`, as that value. + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) column_types: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) complete_schemas: Bound<'py, PyAny>, + pub(crate) dynamic_families: Bound<'py, PyAny>, + /// The `NativeModelAnalyses`, or `None` when no model was analysed. + pub(crate) analyses: Bound<'py, PyAny>, +} + +/// Whether `model_input` is selected for analysis by `names` (`None` selects every model). +pub(crate) fn selected_for_analysis( + model_input: &Bound<'_, PyAny>, + names: &Bound<'_, PyAny>, +) -> PyResult { + Ok(names.is_none() || names.contains(model_stem(model_input)?)?) +} + +/// The model's name: its file's stem. +pub(crate) fn model_stem<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { + model_input + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem") +} + +/// Python's `left and not right`. +pub(crate) fn and_not<'py>( + left: &Bound<'py, PyAny>, + right: &Bound<'py, PyAny>, +) -> PyResult> { + if !left.is_truthy()? { + return Ok(left.clone()); + } + Ok(PyBool::new(left.py(), !right.is_truthy()?) + .to_owned() + .into_any()) +} + +/// The profile, catalog, declared relations and model analyses of `request`'s project. +pub(crate) fn analysis_context<'py>( + py: Python<'py>, + request: &AssemblyRequest, +) -> PyResult> { + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let sql_analysis_enabled: Bound<'py, PyAny> = and_not( + &inputs + .getattr("effective_settings")? + .getattr("sql_analysis")?, + request.skip_column_inference.bind(py), + )?; + let relations: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "declared_relations")?.call1((inputs,))?; + let column_types: Bound<'py, PyAny> = relations.get_item(0)?; + let nullability: Bound<'py, PyAny> = relations.get_item(1)?; + let complete_schemas: Bound<'py, PyAny> = relations.get_item(2)?; + let dynamic_families: Bound<'py, PyAny> = dynamic_families_by_table(inputs)?.into_any(); + let profile: Bound<'py, PyAny> = + profile_with_catalog(inputs, request.inference_profile.bind(py), &column_types)?; + let mut context: AnalysisContext<'py> = AnalysisContext { + profile, + sql_analysis_enabled, + column_types, + nullability, + complete_schemas, + dynamic_families, + analyses: py.None().into_bound(py), + }; + if context.sql_analysis_enabled.is_truthy()? { + record_expression_shapes(inputs, &context)?; + } + let catalog: Bound<'py, PyAny> = context.profile.getattr("binding_catalog")?; + catalog + .getattr("native")? + .call_method1("update_relations", (&context.complete_schemas,))?; + catalog + .getattr("schemas")? + .call_method1("update", (&context.complete_schemas,))?; + context.analyses = model_analyses(request, &context)?; + Ok(context) +} + +/// `_build_dynamic_families_by_table`: enforced contract models' dynamic column families. +fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let enforced: Bound<'py, PyAny> = + imported(py, PLANNER_TYPES_MODULE, "ContractPolicy")?.getattr("ENFORCED")?; + let families: Bound<'py, PyDict> = PyDict::new(py); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let contract: Bound<'py, PyAny> = model_input + .getattr("config")? + .getattr("values")? + .call_method1("get", ("contract",))?; + if !contract.eq(&enforced)? { + continue; + } + let schema_entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + if schema_entry.is_none() { + continue; + } + let dynamic_columns: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if dynamic_columns.is_truthy()? { + families.set_item(schema_entry.getattr("name")?, dynamic_columns)?; + } + } + Ok(families) +} + +/// Sorted function names and declared types registered with native validation. +pub(crate) fn declared_names<'py>( + inputs: &Bound<'py, PyAny>, + column_types: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = inputs.py(); + let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; + let names: Bound<'py, PyAny> = + imported(py, NATIVE_DECLARATIONS_MODULE, "known_function_names")? + .call1((functions.clone(),))?; + let types: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_DECLARATIONS_MODULE, "known_declared_types")?, + &[ + ("functions", functions), + ("column_types", column_types.clone()), + ], + )?; + Ok((names, types)) +} + +/// The adapter's profile with project declarations, the native dialect and a binding catalog. +fn profile_with_catalog<'py>( + inputs: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + column_types: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let profile: Bound<'py, PyAny> = if inference_profile.is_truthy()? { + inference_profile.clone() + } else { + imported(py, ADAPTER_MODELS_MODULE, "ExpressionInferenceProfile")?.call0()? + }; + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, column_types)?; + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let ignore_case: Bound<'py, PyAny> = call_with( + &imported(py, IDENTIFIER_CASE_MODULE, "ignores_quoted_case")?, + &[ + ("connection", inputs.getattr("effective_connection")?), + ("dialect", dialect.clone()), + ], + )?; + let name: Option = if dialect.is_none() { + None + } else { + Some(dialect.extract()?) + }; + let native: Bound<'py, PyAny> = match native_dialect(name.as_deref()) { + Some(native) if Some(native) != name.as_deref() => PyString::new(py, native).into_any(), + _ => dialect, + }; + let profile: Bound<'py, PyAny> = replace.call( + (profile.clone(),), + Some(&keywords( + py, + &[ + ("semantic_known_functions", known_functions), + ("semantic_known_types", known_types), + ("quoted_identifiers_ignore_case", ignore_case), + ("sql_analysis_dialect", native), + ( + "function_return_types", + function_return_types(inputs, &profile)?.into_any(), + ), + ], + )?), + )?; + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let catalog: Bound<'py, PyAny> = call_with( + &imported(py, BINDING_CATALOG_MODULE, "create_binding_catalog")?, + &[ + ( + "dialect", + if dialect.is_truthy()? { + dialect + } else { + PyString::new(py, GENERIC_DIALECT).into_any() + }, + ), + ( + "quoted_ignore_case", + profile.getattr("quoted_identifiers_ignore_case")?, + ), + ( + "known_functions", + profile.getattr("semantic_known_functions")?, + ), + ("known_types", profile.getattr("semantic_known_types")?), + ("relations", PyDict::new(py).into_any()), + ], + )?; + replace.call( + (profile,), + Some(&keywords(py, &[("binding_catalog", catalog)])?), + ) +} + +/// The profile's function return types with each scalar project function's analysis name. +fn function_return_types<'py>( + inputs: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let types: Bound<'py, PyDict> = PyDict::new(py); + types.call_method1("update", (profile.getattr("function_return_types")?,))?; + let scalar: Bound<'py, PyDict> = PyDict::new(py); + for function in inputs.getattr("sql_function_inputs")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + if function.getattr("return_columns")?.is_truthy()? { + continue; + } + let name: Bound<'py, PyAny> = function.getattr("name")?.str()?.into_any(); + let key: Bound<'py, PyAny> = PyString::new(py, UDF_PREFIX) + .into_any() + .add(name)? + .call_method0("upper")?; + scalar.set_item(key, function.getattr("returns")?)?; + } + types.update(scalar.as_mapping())?; + Ok(types) +} + +/// Record each expression source's analysed shape as its declared relation. +fn record_expression_shapes<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, +) -> PyResult<()> { + let py: Python<'py> = inputs.py(); + let mut sources: Vec> = Vec::new(); + let mut expressions: Vec> = Vec::new(); + for source_input in inputs.getattr("source_inputs")?.try_iter()? { + let source_input: Bound<'py, PyAny> = source_input?; + let expression: Bound<'py, PyAny> = source_input + .getattr("source_entry")? + .getattr("expression")?; + if expression.is_truthy()? { + sources.push(source_input); + expressions.push(expression); + } + } + let shapes: Vec> = call_with( + &imported(py, NATIVE_STAGES_MODULE, "expression_source_shapes")?, + &[ + ("expressions", PyTuple::new(py, expressions)?.into_any()), + ("profile", context.profile.clone()), + ], + )? + .try_iter()? + .collect::>()?; + if shapes.len() != sources.len() { + let relation: &str = if shapes.len() < sources.len() { + "shorter" + } else { + "longer" + }; + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "zip() argument 2 is {relation} than argument 1" + ))); + } + let unknown: Bound<'py, PyAny> = + imported(py, LINEAGE_TYPES_MODULE, "InferredNullability")?.getattr("UNKNOWN")?; + for (source_input, shape) in sources.iter().zip(shapes) { + if shape.is_none() { + continue; + } + let name: Bound<'py, PyAny> = source_input.getattr("source_entry")?.getattr("name")?; + let declared_types: Bound<'py, PyAny> = context + .column_types + .call_method1("get", (&name, PyDict::new(py)))?; + let declared_shape: Bound<'py, PyDict> = PyDict::new(py); + for item in shape.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let column: Bound<'py, PyAny> = item.get_item(0)?; + declared_shape.set_item( + &column, + declared_types.call_method1("get", (&column, item.get_item(1)?))?, + )?; + } + context.complete_schemas.set_item(&name, &declared_shape)?; + context.column_types.set_item(&name, &declared_shape)?; + let nullability: Bound<'py, PyDict> = PyDict::new(py); + for column in declared_shape.keys() { + let known: Bound<'py, PyAny> = context + .nullability + .call_method1("get", (&name, PyDict::new(py)))?; + nullability.set_item(&column, known.call_method1("get", (&column, &unknown))?)?; + } + context.nullability.set_item(&name, nullability)?; + } + Ok(()) +} + +/// Each selected model's native analysis, timed as `model_analysis_ms`, when SQL analysis runs. +fn model_analyses<'py>( + request: &AssemblyRequest, + context: &AnalysisContext<'py>, +) -> PyResult> { + let py: Python<'py> = context.profile.py(); + if !context.sql_analysis_enabled.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); + let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); + let lineage_mode: Bound<'py, PyAny> = imported(py, LINEAGE_TYPES_MODULE, "ColumnLineageMode")?; + let rich: bool = mode.eq(lineage_mode.getattr("RICH")?)?; + let allow_compact: bool = mode.eq(lineage_mode.getattr("FAST")?)? || rich; + let flags: (Bound<'py, PyAny>, Bound<'py, PyAny>) = ( + PyBool::new(py, allow_compact).to_owned().into_any(), + PyBool::new(py, rich).to_owned().into_any(), + ); + let cache: Bound<'py, PyAny> = if names.ne(PyFrozenSet::empty(py)?)? { + analysis_cache(request, context, &flags)? + } else { + py.None().into_bound(py) + }; + let timing: Bound<'py, PyAny> = + imported(py, PROFILING_MODULE, "record_compile_timing")?.call1(("model_analysis_ms",))?; + within(&timing, |_| { + let cpu_timing: Bound<'py, PyAny> = + imported(py, PROFILING_CPU_MODULE, "record_compile_cpu_timing")? + .call1(("model_analysis_cpu_ms",))?; + within(&cpu_timing, |_| { + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, &context.column_types)?; + let mut model_inputs: Vec> = Vec::new(); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + if model_input.getattr("sql_validation_enabled")?.is_truthy()? + && selected_for_analysis(&model_input, names)? + { + model_inputs.push(model_input); + } + } + let analysis_request: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativeModelAnalysisRequest")?, + &[ + ("known_functions", known_functions), + ("known_types", known_types), + ("model_inputs", PyTuple::new(py, model_inputs)?.into_any()), + ("column_nullability_by_table", context.nullability.clone()), + ("column_types_by_table", context.column_types.clone()), + ("inference_profile", context.profile.clone()), + ("allow_compact_analysis", flags.0.clone()), + ("rich_type_inference", flags.1.clone()), + ("analysis_cache", cache.clone()), + ("complete_binding_schemas", context.complete_schemas.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + imported(py, NATIVE_STAGES_MODULE, "analyze_model_sql")?.call1((analysis_request,)) + }) + }) +} + +/// The analysis cache context keyed by the project's declarations, target and vars. +fn analysis_cache<'py>( + request: &AssemblyRequest, + context: &AnalysisContext<'py>, + (allow_compact, rich): &(Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = context.profile.py(); + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, &context.column_types)?; + let namespace: Bound<'py, PyDict> = keywords( + py, + &[ + ("known_functions", known_functions), + ("known_types", known_types), + ("target", inputs.getattr("effective_target_name")?), + ("vars", inputs.getattr("effective_vars")?), + ], + )?; + call_with( + &imported(py, ANALYSIS_CACHE_MODULE, "build_analysis_cache_context")?, + &[ + ("root", request.analysis_cache_dir.bind(py).clone()), + ("inference_profile", context.profile.clone()), + ("allow_compact_analysis", allow_compact.clone()), + ("rich_type_inference", rich.clone()), + ("signature_namespace", namespace.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs new file mode 100644 index 0000000000..dd0c9ae3ab --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -0,0 +1,188 @@ +//! SQL test macro-mock model queries and compiled SQL scenarios of an assembled project. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MODULE; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +/// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. +const SCENARIO_FIELDS: [&str; 12] = [ + "sql_body", + "authored_ctes", + "expected_ctes", + "assertion_ctes", + "source_fixture_names", + "ref_fixture_names", + "seed_fixture_names", + "dbt_ref_fixture_names", + "expected_model_names", + "assertion_names", + "assertion_target_model_names", + "target_model_names", +]; + +/// Re-expand each model from its pre-macro SQL with the test's macro mocks applied. +pub(crate) fn test_model_query_overrides<'py>( + test_input: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let overrides: Bound<'py, PyDict> = PyDict::new(py); + let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + if !payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompileModelSqlTestInputPayload", + )?)? { + return Ok(overrides); + } + let macro_mocks: Bound<'py, PyAny> = payload.getattr("macro_mocks")?; + if !macro_mocks.is_truthy()? { + return Ok(overrides); + } + let macro_context: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?.call( + (model_macro_context(inputs)?,), + Some(&keywords( + py, + &[( + "_enforce_explicit_references", + PyBool::new(py, false).to_owned().into_any(), + )], + )?), + )?; + let declaration_scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let lookup: Bound<'py, PyAny> = if declaration_scope.is_none() { + py.None().into_bound(py) + } else { + declaration_scope.getattr("resolver")?.getattr("lookup")? + }; + let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; + let declaration_resolver: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, + &[ + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ("scope_index", inputs.getattr("scope_index")?), + ("loaded_macros", loaded_macros.clone()), + ("lookup", lookup), + ], + )?; + let expand: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "expand_sql_macros")?; + let intrinsics: Bound<'py, PyAny> = imported( + py, + CURSOR_INTRINSICS_MODULE, + "get_validated_model_cursor_intrinsics", + )?; + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let file_path: Bound<'py, PyAny> = + model_input.getattr("model_file")?.getattr("file_path")?; + let model_name: Bound<'py, PyAny> = file_path.getattr("stem")?; + let macro_source_sql: Bound<'py, PyAny> = model_input.getattr("macro_source_sql")?; + let macro_source_sql: Bound<'py, PyAny> = if macro_source_sql.is_truthy()? { + macro_source_sql + } else { + model_input.getattr("query_sql")? + }; + let expanded: Bound<'py, PyAny> = call_with( + &expand, + &[ + ("sql", macro_source_sql), + ("file_path", file_path), + ("loaded_macros", loaded_macros.clone()), + ("macro_overrides", macro_mocks.clone()), + ("macro_context", macro_context.clone()), + ("declaration_resolver", declaration_resolver.clone()), + ], + )?; + let query: Bound<'py, PyAny> = call_with( + &intrinsics, + &[ + ("sql", expanded), + ( + "config_values", + model_input.getattr("config")?.getattr("values")?, + ), + ("model_name", model_name.clone()), + ], + )?; + overrides.set_item(model_name, query)?; + } + Ok(overrides) +} + +/// The compile's macro context, or one built from the effective adapter, settings and target. +fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let macro_context: Bound<'py, PyAny> = inputs.getattr("macro_context")?; + if macro_context.is_truthy()? { + return Ok(macro_context); + } + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", inputs.getattr("project_config")?), + ("local_config", inputs.getattr("local_config")?), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ("adapter_name", adapter_name), + ( + "sql_analysis_enabled", + inputs + .getattr("effective_settings")? + .getattr("sql_analysis")?, + ), + ("target_name", inputs.getattr("effective_target_name")?), + ("vars", inputs.getattr("effective_vars")?), + ], + ) +} + +/// Each scenario input as a `CompiledSqlScenario`. +pub(crate) fn compiled_sql_scenarios<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let scenario_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompiledSqlScenario")?; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?; + let resource_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_SCENARIO")?; + let mut scenarios: Vec> = Vec::new(); + for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { + let scenario_input: Bound<'py, PyAny> = scenario_input?; + let scenario_file: Bound<'py, PyAny> = scenario_input.getattr("scenario_file")?; + let name: Bound<'py, PyAny> = scenario_file.getattr("name")?; + let key: Bound<'py, PyAny> = call_with( + &key_class, + &[ + ("resource_type", resource_type.clone()), + ("name", name.clone()), + ], + )?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ("key", key), + ("name", name), + ("scenario_file", scenario_file.clone()), + ], + )?; + for field in SCENARIO_FIELDS { + fields.set_item(field, scenario_input.getattr(field)?)?; + } + fields.set_item("source_path", scenario_file.getattr("relative_path")?)?; + fields.set_item("ownership_root", scenario_file.getattr("ownership_root")?)?; + scenarios.push(scenario_class.call((), Some(&fields))?); + } + PyTuple::new(py, scenarios) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs new file mode 100644 index 0000000000..96aa7a4797 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -0,0 +1,274 @@ +//! Each compiled model of an assembled project, with its syntax checks and analysis fallback. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_driver::assembly_analysis::{ + ANALYSIS_MODELS_MODULE, AnalysisContext, model_stem, selected_for_analysis, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SYNTAX_CHECKS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.syntax_checks"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const INFER_ROWS_MODULE: &str = + "sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows"; +const HAND_BUILT_MODULE: &str = "sqlbuild.compiler.compile._helpers.assembly.hand_built_models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; +const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; + +/// The Python stages each model's assembly calls, looked up once per project. +struct ModelStages<'py> { + analysis_sql: Bound<'py, PyAny>, + placeholders: Bound<'py, PyAny>, + validate_hooks: Bound<'py, PyAny>, + validate_sql: Bound<'py, PyAny>, + macro_calls: Bound<'py, PyAny>, + native_input: Bound<'py, PyAny>, +} + +/// One model's assembly inputs: the attached model, its native facts and its two gates. +struct ModelRequest<'a, 'py> { + model_input: &'a Bound<'py, PyAny>, + native_model: Bound<'py, PyAny>, + sql_analysis_enabled: bool, + sql_validation_enabled: bool, +} + +/// Every model of the project, compiled in input order. +pub(crate) fn compiled_models<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + (native_models, names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let native: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.into_any(); + let stages: ModelStages<'py> = ModelStages { + analysis_sql: imported( + py, + CURSOR_INTRINSICS_MODULE, + "cursor_intrinsics_analysis_sql", + )?, + placeholders: imported(py, SYNTAX_CHECKS_MODULE, "model_placeholders")?, + validate_hooks: native.getattr("validate_hook_sql_syntax")?, + validate_sql: native.getattr("validate_sql_syntax")?, + macro_calls: imported(py, MACROS_MODULE, "find_macro_call_names")?, + native_input: native.getattr("NativeModelInput")?, + }; + let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; + let mut models: Vec> = Vec::new(); + for (index, model_input) in inputs.getattr("model_inputs")?.try_iter()?.enumerate() { + let model_input: Bound<'py, PyAny> = model_input?; + let sql_analysis_enabled: bool = analysis_enabled + && model_input.getattr("sql_validation_enabled")?.is_truthy()? + && selected_for_analysis(&model_input, names)?; + let request: ModelRequest<'_, 'py> = ModelRequest { + model_input: &model_input, + sql_analysis_enabled, + sql_validation_enabled: selected_for_analysis(&model_input, names)?, + native_model: native_models.get_item(index)?, + }; + models.push(compiled_model(&stages, context, &request)?); + } + PyTuple::new(py, models) +} + +/// `_assemble_compiled_model`. +fn compiled_model<'py>( + stages: &ModelStages<'py>, + context: &AnalysisContext<'py>, + request: &ModelRequest<'_, 'py>, +) -> PyResult> { + let model_input: &Bound<'py, PyAny> = request.model_input; + let py: Python<'py> = model_input.py(); + let model_name: Bound<'py, PyAny> = model_stem(model_input)?; + let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; + let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; + let analysis_query_sql: Bound<'py, PyAny> = call_with( + &stages.analysis_sql, + &[ + ("sql", model_input.getattr("query_sql")?), + ( + "cursor_type", + config_values.call_method1("get", ("cursor_type",))?, + ), + ], + )?; + let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; + let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; + let location: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + model_name.clone(), + model_input.getattr("model_file")?.getattr("file_path")?, + ], + )?; + let syntax: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + placeholders.clone(), + context.profile.getattr("sql_analysis_dialect")?, + ], + )?; + let validates: bool = input_validated && !syntax_validated; + if request.sql_validation_enabled && validates { + for hook_name in HOOK_NAMES { + stages.validate_hooks.call1(( + config_values.call_method1("get", (hook_name,))?, + hook_name, + &location, + &syntax, + ))?; + } + } + let mut session_analysis: Bound<'py, PyAny> = py.None().into_bound(py); + let mut fallback: Bound<'py, PyAny> = py.None().into_bound(py); + if request.sql_analysis_enabled { + let analyses: &Bound<'py, PyAny> = &context.analyses; + let position: Bound<'py, PyAny> = if analyses.is_none() { + py.None().into_bound(py) + } else { + analyses + .getattr("positions")? + .call_method1("get", (&model_name,))? + }; + if analyses.is_none() || position.is_none() { + return Err(NativeCompilerError::new_err(format!( + "{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no \ + analysis for model '{}'", + model_name.str()? + ))); + } + session_analysis = PyTuple::new(py, [analyses.getattr("session")?, position])?.into_any(); + if !analyses.getattr("succeeded")?.contains(&model_name)? { + if validates { + stages + .validate_sql + .call1((&analysis_query_sql, &location, &syntax))?; + } + fallback = fallback_columns(context, &analysis_query_sql, &placeholders)?; + } + } else if request.sql_validation_enabled && validates { + stages + .validate_sql + .call1((&analysis_query_sql, &location, &syntax))?; + } + let declared_macro_deps: Bound<'py, PyAny> = model_input.getattr("macro_deps")?; + let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { + declared_macro_deps + } else { + stages + .macro_calls + .call1((model_input.getattr("macro_source_sql")?,))? + }; + let native_model: &Bound<'py, PyAny> = &request.native_model; + if model_input.is_instance(&stages.native_input)? { + let macro_list: Bound<'py, PyList> = + PyList::new(py, macro_deps.try_iter()?.collect::>>()?)?; + return model_input.call_method1( + "compiled_model", + (( + native_model.getattr("deps")?, + session_analysis, + fallback, + native_model.getattr("dynamic_contract")?, + macro_list, + ),), + ); + } + let analysis: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "HandBuiltAnalysis")?, + &[ + ("session", session_analysis), + ("fallback", fallback), + ( + "standalone_proof", + native_model.getattr("dynamic_contract")?, + ), + ], + )?; + call_with( + &imported(py, HAND_BUILT_MODULE, "hand_built_compiled_model")?, + &[ + ("model_input", model_input.clone()), + ("deps", native_model.getattr("deps")?), + ("analysis", analysis), + ("macro_deps", macro_deps), + ], + ) +} + +/// The columns a parse-only native query inference reads from a model whose analysis failed. +fn fallback_columns<'py>( + context: &AnalysisContext<'py>, + analysis_query_sql: &Bound<'py, PyAny>, + placeholders: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = analysis_query_sql.py(); + let query: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativeColumnQuery")?, + &[ + ("sql", analysis_query_sql.clone()), + ("mode", PyString::new(py, "parse").into_any()), + ("placeholders", placeholders.clone()), + ], + )?; + call_with( + &imported(py, INFER_ROWS_MODULE, "infer_native_query_column_rows")?, + &[ + ("queries", PyTuple::new(py, [query])?.into_any()), + ("profile", context.profile.clone()), + ("column_nullability_by_table", context.nullability.clone()), + ], + )? + .get_item(0)? + .get_item(1) +} + +/// `_project_binding_diagnostics`: every analysed model's located binding diagnostics. +pub(crate) fn project_binding_diagnostics<'py>( + models: &Bound<'py, PyTuple>, + analyses: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + if analyses.is_none() { + return Ok(diagnostics); + } + let positions: Bound<'py, PyAny> = analyses.getattr("positions")?; + for model in models.as_any().try_iter()? { + let model: Bound<'py, PyAny> = model?; + if positions.contains(model.getattr("name")?)? { + for diagnostic in model.getattr("binding_diagnostics")?.try_iter()? { + diagnostics.push(diagnostic?); + } + } + } + Ok(diagnostics) +} + +/// `_binding_rows`: each analysed model's raw binding rows by model name. +pub(crate) fn binding_rows<'py>(analyses: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = analyses.py(); + let rows_by_name: Bound<'py, PyDict> = PyDict::new(py); + if analyses.is_none() { + return Ok(rows_by_name); + } + let session: Bound<'py, PyAny> = analyses.getattr("session")?; + if session.is_none() { + return Ok(rows_by_name); + } + let rows: Bound<'py, PyAny> = session.call_method0("binding_rows")?; + for item in analyses + .getattr("positions")? + .call_method0("items")? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + rows_by_name.set_item(item.get_item(0)?, rows.get_item(item.get_item(1)?)?)?; + } + Ok(rows_by_name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs new file mode 100644 index 0000000000..376a3f1339 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -0,0 +1,426 @@ +//! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; + +use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; + +use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly_analysis::{ + AnalysisContext, NATIVE_STAGES_MODULE, analysis_context, and_not, +}; +use crate::bindings::_helpers::compile_driver::assembly_checks::{ + compiled_sql_scenarios, test_model_query_overrides, +}; +use crate::bindings::_helpers::compile_driver::assembly_models::{ + binding_rows, compiled_models, project_binding_diagnostics, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; +const SCOPE_DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.scope"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +pub(crate) const ADAPTER_NAME_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; +const SCENARIO_CONFIG_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const SQL_TESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.sql_tests"; +const RECOVERY_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.recovery"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SOURCE_SLOT: &str = "source"; +const SEED_SLOT: &str = "seed"; +const AUDIT_SLOT: &str = "audit"; + +/// One native resource kind: its inputs, recorded facts, slot and the owner's methods. +struct ResourceKind<'a> { + inputs: &'a str, + facts: &'a str, + slot: Option<&'a str>, + record: &'a str, + compiled: &'a str, +} + +const SOURCES: ResourceKind<'static> = ResourceKind { + inputs: "source_inputs", + facts: "source_namespaces", + slot: Some(SOURCE_SLOT), + record: "record_source_namespaces", + compiled: "compiled_source", +}; +const SEEDS: ResourceKind<'static> = ResourceKind { + inputs: "seed_inputs", + facts: "seed_namespaces", + slot: Some(SEED_SLOT), + record: "record_seed_namespaces", + compiled: "compiled_seed", +}; +const FUNCTIONS: ResourceKind<'static> = ResourceKind { + inputs: "sql_function_inputs", + facts: "function_deps", + slot: None, + record: "record_function_deps", + compiled: "compiled_function", +}; +const AUDITS: ResourceKind<'static> = ResourceKind { + inputs: "audit_inputs", + facts: "audit_scope_deps", + slot: Some(AUDIT_SLOT), + record: "record_audit_scope_deps", + compiled: "compiled_audit", +}; + +/// Convert attached compile inputs into the planner-ready project view. +pub(crate) fn assembled_project<'py>( + py: Python<'py>, + request: &AssemblyRequest, +) -> PyResult> { + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); + let context: AnalysisContext<'py> = analysis_context(py, request)?; + let native_session: Bound<'py, PyAny> = if context.analyses.is_none() { + py.None().into_bound(py) + } else { + context.analyses.getattr("session")? + }; + let native: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_STAGES_MODULE, "native_project_facts")?, + &[ + ("inputs", inputs.clone()), + ("dialect", context.profile.getattr("sql_analysis_dialect")?), + ("analysis_model_names", names.clone()), + ("analyses", context.analyses.clone()), + ("session", native_session.clone()), + ("column_types_by_table", context.column_types.clone()), + ( + "authoritative_column_types_by_table", + context.complete_schemas.clone(), + ), + ("column_nullability_by_table", context.nullability.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + let resources: Bound<'py, PyAny> = native.getattr("resources")?; + let scope_index: Bound<'py, PyAny> = call_with( + &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, + &[("inputs", inputs.clone())], + )?; + call_with( + &imported(py, SCOPE_DIAGNOSTICS_MODULE, "report_scope_index_errors")?, + &[("index", scope_index.clone())], + )?; + let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; + let models: Bound<'py, PyTuple> = + compiled_models(inputs, &context, (&native.getattr("models")?, names))?; + let fields: Bound<'py, PyDict> = project_fields( + inputs, + &context, + (&models, &resources, &scope_index), + &target_values, + )?; + let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; + let binding_results: Bound<'py, PyDict> = binding_rows(&context.analyses)?; + call_with( + &imported(py, RECOVERY_MODULE, "complete_semantic_diagnostics")?, + &[ + ("project", project), + ("profile", context.profile.clone()), + ( + "resource_sql_analysis", + and_not( + &context.sql_analysis_enabled, + &inputs.getattr("no_sql_validation")?, + )?, + ), + ("native_session", native_session), + ("binding_results", binding_results.into_any()), + ], + ) +} + +/// `value.strip()` when `value` is a string with non-blank text, else `None`. +fn str_or_none<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if let Ok(text) = value.downcast::() + && let Some(stripped) = configured_namespace(&text.to_cow()?) + { + return Ok(PyString::new(py, &stripped).into_any()); + } + Ok(py.None().into_bound(py)) +} + +/// The selected target's `field`, or `None` without a selected target. +fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult> { + let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + if target.is_none() { + return Ok(target); + } + target.getattr(field) +} + +/// The target database and schema, with connection fallbacks, resolved as early model templates. +fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let connection: Bound<'py, PyAny> = inputs.getattr("effective_connection")?; + let database: Bound<'py, PyAny> = target_field(inputs, "database")?; + let database: Bound<'py, PyAny> = if database.is_truthy()? { + database + } else { + connection_database_fallback(inputs, &connection)? + }; + let schema: Bound<'py, PyAny> = target_field(inputs, "schema")?; + let schema: Bound<'py, PyAny> = if schema.is_truthy()? { + schema + } else { + str_or_none(&connection.call_method1("get", ("schema",))?)? + }; + call_with( + &imported( + py, + CONTEXT_TEMPLATES_MODULE, + "resolve_early_model_templates", + )?, + &[ + ( + "values", + keywords(py, &[("database", database), ("schema", schema)])?.into_any(), + ), + ("effective_vars", inputs.getattr("effective_vars")?), + ( + "effective_target_name", + inputs.getattr("effective_target_name")?, + ), + ("run_id", inputs.getattr("run_id")?), + ], + ) +} + +/// The connection's database, except under DuckDB, whose database is a file. +fn connection_database_fallback<'py>( + inputs: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", inputs.getattr("project_config")?), + ("local_config", inputs.getattr("local_config")?), + ], + )?; + if adapter_name.eq(imported(py, ADAPTER_TYPES_MODULE, "BuiltinAdapter")?.getattr("DUCKDB")?)? { + return Ok(py.None().into_bound(py)); + } + str_or_none(&connection.call_method1("get", ("database",))?) +} + +/// The compiled project's fields, in the order Python's assembly built them. +fn project_fields<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + (models, resources, scope_index): ( + &Bound<'py, PyTuple>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + target_values: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let discovered: Bound<'py, PyAny> = inputs.getattr("discovered_inputs")?; + let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ( + "binding_catalog", + context.profile.getattr("binding_catalog")?, + ), + ("run_id", inputs.getattr("run_id")?), + ( + "effective_target_name", + inputs.getattr("effective_target_name")?, + ), + ( + "effective_connection", + inputs.getattr("effective_connection")?, + ), + ("effective_vars", inputs.getattr("effective_vars")?), + ( + "effective_target_database", + str_or_none(&target_values.call_method1("get", ("database",))?)?, + ), + ( + "effective_target_schema", + str_or_none(&target_values.call_method1("get", ("schema",))?)?, + ), + ( + "sql_analysis_dialect", + context.profile.getattr("sql_analysis_dialect")?, + ), + ("sql_lexical_syntax", inputs.getattr("sql_lexical_syntax")?), + ("compile_cache_dir", inputs.getattr("compile_cache_dir")?), + ("settings", inputs.getattr("effective_settings")?), + ( + "scenario", + call_with( + &imported( + py, + SCENARIO_CONFIG_MODULE, + "resolve_effective_scenario_config", + )?, + &[ + ("project_config", project_config.clone()), + ("local_config", inputs.getattr("local_config")?), + ], + )?, + ), + ("models", models.clone().into_any()), + ], + )?; + for (name, kind) in [ + ("sources", &SOURCES), + ("seeds", &SEEDS), + ("functions", &FUNCTIONS), + ("audits", &AUDITS), + ] { + fields.set_item(name, compiled_resources(inputs, resources, kind)?)?; + } + let inputs_owned: pyo3::Py = inputs.clone().unbind(); + let overrides: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + let py: Python<'_> = arguments.py(); + test_model_query_overrides(&arguments.get_item(0)?, inputs_owned.bind(py)) + .map(|overrides| overrides.into_any().unbind()) + }, + )?; + fields.set_item( + "sql_tests", + call_with( + &imported(py, SQL_TESTS_MODULE, "assemble_sql_tests")?, + &[ + ("inputs", inputs.clone()), + ("macro_mock_queries", overrides.into_any()), + ], + )?, + )?; + fields.set_item("sql_scenarios", compiled_sql_scenarios(inputs)?)?; + for name in ["loader_functions", "hook_functions"] { + fields.set_item(name, discovered.getattr(name)?)?; + } + fields.set_item( + "enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + )?; + for name in ["sql_hook_files", "materialization_files"] { + fields.set_item(name, discovered.getattr(name)?)?; + } + for name in ["public_enums", "public_constants", "loaded_macros"] { + fields.set_item(name, inputs.getattr(name)?)?; + } + let mut diagnostics: Vec> = inputs + .getattr("diagnostics")? + .try_iter()? + .collect::>()?; + diagnostics.extend(project_binding_diagnostics(models, &context.analyses)?); + fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; + fields.set_item( + "external_sql_reference_resolver", + inputs.getattr("external_sql_reference_resolver")?, + )?; + fields.set_item("scope_index", scope_index)?; + Ok(fields) +} + +/// Each resource's view over the compile's native project, raising assembly's error there. +fn compiled_resources<'py>( + inputs: &Bound<'py, PyAny>, + resources: &Bound<'py, PyAny>, + kind: &ResourceKind<'_>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let count: usize = inputs.getattr(kind.inputs)?.len()?; + if count == 0 { + return Ok(PyTuple::empty(py)); + } + let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let facts: Bound<'py, PyAny> = resources.getattr(kind.facts)?; + owner.call_method1( + kind.record, + (PyList::new( + py, + facts.try_iter()?.collect::>>()?, + )?,), + )?; + let mut compiled: Vec> = Vec::with_capacity(count); + for index in 0..count { + if let Some(slot) = kind.slot { + native_resource(resources, (slot, &facts), index)?; + } + compiled.push(owner.call_method1(kind.compiled, (index,))?); + } + PyTuple::new(py, compiled) +} + +/// Resource `index` of `slot`, raising the error Python's assembly raises there. +fn native_resource<'py>( + resources: &Bound<'py, PyAny>, + (slot, items): (&str, &Bound<'py, PyAny>), + index: usize, +) -> PyResult> { + let failure: Bound<'py, PyAny> = resources.getattr("failure")?; + if !failure.is_none() + && failure.getattr("slot")?.eq(slot)? + && failure.getattr("index")?.eq(index)? + { + return Err(PyErr::from_value(failure.getattr("error")?)); + } + items.get_item(index) +} + +/// The native owner's project view when it holds every model; else a `CompiledProject`. +fn project_view<'py>( + inputs: &Bound<'py, PyAny>, + fields: &Bound<'py, PyDict>, + models: &Bound<'py, PyTuple>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let native_model: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeCompiledModel")?; + let mut all_native: bool = true; + for model in models.as_any().try_iter()? { + if !model?.is_instance(&native_model)? { + all_native = false; + break; + } + } + if !owner.is_none() && all_native { + let view_fields: Bound<'py, PyDict> = fields.copy()?; + view_fields.set_item("unmatched_literal_sql_relations", PyTuple::empty(py))?; + view_fields.set_item("native_session", py.None())?; + return owner.call_method1("project_view", (view_fields,)); + } + let sql_expansions: Bound<'py, PyDict> = PyDict::new(py); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let expansion: Bound<'py, PyAny> = model_input.getattr("sql_expansion")?; + if !expansion.is_none() { + sql_expansions.set_item( + model_input.getattr("model_file")?.getattr("file_path")?, + expansion, + )?; + } + } + let arguments: Bound<'py, PyDict> = fields.copy()?; + arguments.set_item("sql_expansions", sql_expansions)?; + imported(py, COMPILE_MODELS_MODULE, "CompiledProject")?.call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs new file mode 100644 index 0000000000..94aa6db407 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -0,0 +1,541 @@ +//! Project discovery: configuration, declaration files and imported project Python, validated. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; +use crate::bindings::_helpers::discovery::declaration_records::{ + discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, + discover_macro_files, discover_model_schema_files, discover_seed_files, + discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, +}; +use crate::bindings::_helpers::discovery::model_records::{ + ModelFileSelection, discovered_model_files, +}; +use crate::bindings::_helpers::discovery::sql_test_records::{ + discover_scenario_files, discover_test_files, +}; +use crate::bindings::_helpers::discovery::yaml_records::{ + discover_schema_files, discover_source_files, +}; + +const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const PROJECT_YAML_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.project"; +const TARGET_CONNECTIONS_MODULE: &str = + "sqlbuild.compiler.discovery._helpers.validation.target_connections"; +const PYTHON_PATHS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.python_paths"; +const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const EVENT_EXPORTERS_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; +const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; +const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; +const LOADERS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.integrations.loaders"; +const SNAPSHOT_MODULE: &str = "sqlbuild.compiler.discovery.classes.directory_snapshot"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; + +/// Declaration file collections counted into the declaration-parse lifecycle, in field order. +const DECLARATION_COLLECTIONS: [&str; 14] = [ + "source_files", + "model_files", + "enum_files", + "constant_files", + "model_schema_files", + "sql_function_files", + "sql_hook_files", + "python_function_files", + "schema_files", + "seed_files", + "test_files", + "scenario_files", + "audit_files", + "macro_files", +]; + +/// A native discoverer that takes only `project_dir`. +type FileDiscoverer = for<'py> fn(&Bound<'py, PyAny>) -> PyResult>; + +/// Early discoverers by result field; `None` is the Python function-file discoverer (uses `ast`). +const FILE_DISCOVERERS: [(&str, Option); 8] = [ + ( + "enum_files", + Some(|dir| Ok(discover_enum_files(dir, None, false)?.into_any())), + ), + ( + "constant_files", + Some(|dir| Ok(discover_constant_files(dir, None, false)?.into_any())), + ), + ( + "model_schema_files", + Some(|dir| Ok(discover_model_schema_files(dir, None)?.into_any())), + ), + ( + "sql_function_files", + Some(|dir| Ok(discover_sql_function_files(dir, None)?.into_any())), + ), + ( + "sql_hook_files", + Some(|dir| Ok(discover_sql_hook_files(dir, None)?.into_any())), + ), + ("python_function_files", None), + ( + "schema_files", + Some(|dir| Ok(discover_schema_files(dir)?.into_any())), + ), + ( + "seed_files", + Some(|dir| Ok(discover_seed_files(dir)?.into_any())), + ), +]; + +/// Discoverers that run after the schema files, in their original order. +const LATE_FILE_DISCOVERERS: [(&str, FileDiscoverer); 6] = [ + ("test_files", |dir| { + Ok(discover_test_files(dir, None, None)?.into_any()) + }), + ("scenario_files", |dir| { + Ok(discover_scenario_files(dir, None)?.into_any()) + }), + ("audit_files", |dir| { + Ok(discover_audit_files(dir, None)?.into_any()) + }), + ("macro_files", |dir| { + Ok(discover_macro_files(dir, false)?.into_any()) + }), + ("adapter_file", discover_adapter_file), + ("native_session", retained_discovery_session), +]; + +/// Discoverers of imported project Python that take `project_dir` and `providers`. +const PYTHON_DISCOVERERS: [(&str, &str, &str); 5] = [ + ( + "python_nodes", + FILESYSTEM_MODULE, + "discover_python_node_functions", + ), + ( + "materialization_files", + FILESYSTEM_MODULE, + "discover_materialization_files", + ), + ( + "hook_functions", + FILESYSTEM_MODULE, + "discover_hook_functions", + ), + ( + "event_exporters", + FILESYSTEM_MODULE, + "discover_event_exporter_functions", + ), + ( + "command_output_sinks", + SINKS_MODULE, + "discover_command_output_sink_functions", + ), +]; + +/// Python node groups whose relative paths count as imported project Python. +const PYTHON_NODE_GROUPS: [&str; 5] = ["loaders", "tasks", "assets", "checks", "audit_factories"]; + +/// `OperationLifecycle(operation_kind="project", operation_name=name)`. +fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", "project".into_pyobject(py)?.into_any()), + ("operation_name", name.into_pyobject(py)?.into_any()), + ], + ) +} + +/// The keyword arguments of one discovery, kept for the frontier stage. +struct DiscoveryRequest { + project_dir: Py, + sql_analysis_enabled_override: Py, + extract_output_column_locations: bool, +} + +/// Load all raw project inputs from disk before semantic resolution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn discover_project_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_project_inputs", + options, + ( + &["project_dir"], + &[ + "sql_analysis_enabled_override", + "extract_output_column_locations", + ], + ), + )?; + let request: DiscoveryRequest = DiscoveryRequest { + project_dir: arguments.value("project_dir")?.unbind(), + sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), + extract_output_column_locations: !arguments.passed("extract_output_column_locations")? + || arguments.flag("extract_output_column_locations")?, + }; + within(&lifecycle(py, "project_discovery")?, |_| { + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + let py: Python<'_> = arguments.py(); + within(&lifecycle(py, "discovery_project_assembly")?, |_| { + assembled_inputs(py, &request) + }) + .map(Bound::unbind) + }, + )?; + call_with( + &imported(py, FRONTIER_MODULE, "compile_frontier")?, + &[ + ( + "until", + imported(py, FRONTIER_TYPES_MODULE, "CompilerStage")? + .getattr("DISCOVERED_PROJECT_INPUTS")?, + ), + ("python_stage", stage.into_any()), + ], + ) + }) +} + +/// `_assemble_discovered_project_inputs`: configuration, its checks, discovery and validation. +fn assembled_inputs<'py>( + py: Python<'py>, + request: &DiscoveryRequest, +) -> PyResult> { + let project_dir: &Bound<'py, PyAny> = request.project_dir.bind(py); + let directory: [(&str, Bound<'py, PyAny>); 1] = [("project_dir", project_dir.clone())]; + let project_config: Bound<'py, PyAny> = call_with( + &imported(py, PROJECT_YAML_MODULE, "load_project_config")?, + &directory, + )?; + let local_config: Bound<'py, PyAny> = call_with( + &imported(py, PROJECT_YAML_MODULE, "load_local_config")?, + &directory, + )?; + let configured: [(&str, Bound<'py, PyAny>); 3] = [ + ("project_dir", project_dir.clone()), + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ]; + for (module, name) in [ + (PROJECT_YAML_MODULE, "validate_local_sql_analysis_policy"), + (PROJECT_YAML_MODULE, "validate_target_warehouses_adapter"), + (TARGET_CONNECTIONS_MODULE, "validate_target_connections"), + ] { + call_with(&imported(py, module, name)?, &configured)?; + } + call_with( + &imported(py, PYTHON_PATHS_MODULE, "validate_project_python_paths")?, + &directory, + )?; + let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); + let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { + sql_analysis_override.clone() + } else if local_config + .getattr("setting_overrides")? + .contains(imported( + py, + DISCOVERY_CONSTANTS_MODULE, + "SQL_ANALYSIS_SETTING_KEY", + )?)? + { + local_config.getattr("settings")?.getattr("sql_analysis")? + } else { + project_config + .getattr("settings")? + .getattr("sql_analysis")? + }; + let discovered: Bound<'py, PyAny> = discovered_inputs( + py, + (project_dir, &project_config, &local_config), + ( + sql_analysis_enabled.is_truthy()?, + request.extract_output_column_locations, + ), + )?; + imported(py, VALIDATION_MODULE, "validate_discovered_inputs")?.call1((&discovered,))?; + imported( + py, + EVENT_EXPORTERS_MODULE, + "configure_discovered_event_exporters", + )? + .call1((&discovered,))?; + Ok(discovered) +} + +/// Discover all project files and functions into one inputs bundle. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_discovered_project_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_discovered_project_inputs", + options, + ( + &[ + "project_dir", + "project_config", + "local_config", + "sql_analysis_enabled", + ], + &["extract_output_column_locations"], + ), + )?; + discovered_inputs( + py, + ( + &arguments.value("project_dir")?, + &arguments.value("project_config")?, + &arguments.value("local_config")?, + ), + ( + arguments.flag("sql_analysis_enabled")?, + !arguments.passed("extract_output_column_locations")? + || arguments.flag("extract_output_column_locations")?, + ), + ) +} + +/// `build_discovered_project_inputs` for a loaded configuration. +fn discovered_inputs<'py>( + py: Python<'py>, + (project_dir, project_config, local_config): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + flags: (bool, bool), +) -> PyResult> { + let declarations: Bound<'py, PyDict> = within( + &lifecycle(py, "discovery_declaration_parse")?, + |declaration_lifecycle| { + let declarations: Bound<'py, PyDict> = declaration_files(project_dir, flags)?; + let mut item_count: usize = 0; + for field in DECLARATION_COLLECTIONS { + item_count += declarations.as_any().get_item(field)?.len()?; + } + item_count += usize::from(!declarations.as_any().get_item("adapter_file")?.is_none()); + completed(declaration_lifecycle, item_count)?; + Ok(declarations) + }, + )?; + let python: Bound<'py, PyDict> = within( + &lifecycle(py, "discovery_python_import")?, + |python_lifecycle| { + let python: Bound<'py, PyDict> = imported_python(project_dir)?; + completed(python_lifecycle, imported_python_paths(&python)?)?; + Ok(python) + }, + )?; + project_inputs( + (project_dir, project_config, local_config), + &declarations, + &python, + ) +} + +/// Report a lifecycle's completion with its item count. +fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { + let py: Python<'_> = lifecycle.py(); + let metadata: Bound<'_, PyDict> = keywords( + py, + &[("item_count", item_count.into_pyobject(py)?.into_any())], + )?; + call_with( + &lifecycle.getattr("completed")?, + &[("metadata", metadata.into_any())], + )?; + Ok(()) +} + +/// `_discover_declaration_files`: every declaration file under one directory snapshot. +fn declaration_files<'py>( + project_dir: &Bound<'py, PyAny>, + (sql_analysis_enabled, extract_output_column_locations): (bool, bool), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let directory: [(&str, Bound<'py, PyAny>); 1] = [("project_dir", project_dir.clone())]; + let snapshot: Bound<'py, PyAny> = call_with( + &imported(py, SNAPSHOT_MODULE, "DirectorySnapshot")?.getattr("scope")?, + &directory, + )?; + within(&snapshot, |_| { + let files: Bound<'py, PyDict> = PyDict::new(py); + files.set_item("source_files", discover_source_files(project_dir, None)?)?; + files.set_item( + "model_files", + discovered_model_files( + project_dir, + (sql_analysis_enabled, extract_output_column_locations), + ModelFileSelection { + selected_model_names: None, + on_fault: None, + }, + )?, + )?; + for (field, discover) in FILE_DISCOVERERS { + let discovered: Bound<'py, PyAny> = match discover { + Some(discover) => discover(project_dir)?, + None => call_with( + &imported(py, FILESYSTEM_MODULE, "discover_python_function_files")?, + &directory, + )?, + }; + files.set_item(field, discovered)?; + } + for (field, discover) in LATE_FILE_DISCOVERERS { + files.set_item(field, discover(project_dir)?)?; + } + Ok(files) + }) +} + +/// Providers, then every kind of imported project Python that may use them. +fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let providers: Bound<'py, PyAny> = call_with( + &imported(py, FILESYSTEM_MODULE, "discover_provider_classes")?, + &[("project_dir", project_dir.clone())], + )?; + let python: Bound<'py, PyDict> = PyDict::new(py); + python.set_item("providers", &providers)?; + for (field, module, name) in PYTHON_DISCOVERERS { + python.set_item( + field, + call_with( + &imported(py, module, name)?, + &[ + ("project_dir", project_dir.clone()), + ("providers", providers.clone()), + ], + )?, + )?; + } + Ok(python) +} + +/// How many distinct project Python files the imported discoveries came from. +fn imported_python_paths(python: &Bound<'_, PyDict>) -> PyResult { + let py: Python<'_> = python.py(); + let python: &Bound<'_, PyAny> = python.as_any(); + let paths: Bound<'_, PySet> = PySet::empty(py)?; + let mut groups: Vec> = vec![python.get_item("providers")?]; + let nodes: Bound<'_, PyAny> = python.get_item("python_nodes")?; + for group in PYTHON_NODE_GROUPS { + groups.push(nodes.getattr(group)?); + } + for field in [ + "materialization_files", + "hook_functions", + "event_exporters", + "command_output_sinks", + ] { + groups.push(python.get_item(field)?); + } + for group in groups { + for item in group.try_iter()? { + paths.add(item?.getattr("relative_path")?)?; + } + } + Ok(paths.len()) +} + +/// The `DiscoveredProjectInputs` of the discovered declarations and imported Python. +fn project_inputs<'py>( + (project_dir, project_config, local_config): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + declarations: &Bound<'py, PyDict>, + python: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let declarations: &Bound<'py, PyAny> = declarations.as_any(); + let python: &Bound<'py, PyAny> = python.as_any(); + let nodes: Bound<'py, PyAny> = python.get_item("python_nodes")?; + let tuple_of = |group: &str| -> PyResult> { + Ok(PyTuple::new( + py, + nodes + .getattr(group)? + .try_iter()? + .collect::>>()?, + )? + .into_any()) + }; + let integration_loaders: Bound<'py, PyAny> = + imported(py, LOADERS_MODULE, "build_integration_loader_functions")? + .call1((declarations.get_item("source_files")?,))?; + let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?.add(integration_loaders)?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("project_dir", project_dir.clone()), + ]; + for field in [ + "model_files", + "enum_files", + "constant_files", + "model_schema_files", + "sql_function_files", + "sql_hook_files", + "python_function_files", + "schema_files", + "source_files", + "seed_files", + "test_files", + "scenario_files", + "audit_files", + "macro_files", + ] { + fields.push((field, declarations.get_item(field)?)); + } + fields.push(( + "materialization_files", + python.get_item("materialization_files")?, + )); + fields.push(("loader_functions", loader_functions)); + fields.push(("task_functions", tuple_of("tasks")?)); + fields.push(("asset_functions", tuple_of("assets")?)); + fields.push(("check_functions", tuple_of("checks")?)); + fields.push(("audit_factories", tuple_of("audit_factories")?)); + for field in [ + "hook_functions", + "event_exporters", + "command_output_sinks", + "providers", + ] { + fields.push((field, python.get_item(field)?)); + } + fields.push(("adapter_file", declarations.get_item("adapter_file")?)); + fields.push(("native_session", declarations.get_item("native_session")?)); + call_with( + &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredProjectInputs")?, + &fields, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_project_inputs, module)?)?; + module.add_function(wrap_pyfunction!(build_discovered_project_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs new file mode 100644 index 0000000000..fc76e5b89e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -0,0 +1,450 @@ +//! The first attached compile input snapshot: target, vars, macros, scopes, models and resources. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; + +const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; +const ATTACHMENT_MODULE: &str = "sqlbuild.compiler.compile._helpers.attachment.core"; +const DECLARATION_SCOPE_MODULE: &str = + "sqlbuild.compiler.compile._helpers.attachment.declaration_scope"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const ATTACH_CHECKS_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_checks"; +const ATTACH_RESOURCES_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_resources"; +const INPUTS_VIEW_MODULE: &str = "sqlbuild.compiler.compiled_project.main._compile_inputs_view"; +const INPUTS_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.inputs_with"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const NATIVE_MODULE: &str = "sqlbuild._native"; + +const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; +const OPTIONAL: [&str; 8] = [ + "selected_target", + "cli_vars", + "run_id", + "no_sql_validation", + "defer_model_sql_validation", + "external_sql_reference_resolver", + "resolved_connection", + "no_cache", +]; + +/// The effective target, vars, settings and macro context every later stage shares. +struct TargetContext<'py> { + target_name: Bound<'py, PyAny>, + target: Bound<'py, PyAny>, + compile_cache_dir: Bound<'py, PyAny>, + vars: Bound<'py, PyAny>, + settings: Bound<'py, PyAny>, + macro_context: Bound<'py, PyAny>, +} + +/// The attached models with the native project that owns them and their declarations. +struct ModelBuild<'py> { + inputs: Bound<'py, PyAny>, + native_project: Bound<'py, PyAny>, + enums: Bound<'py, PyAny>, + constants: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, +} + +/// Seeds, functions and sources, and `(tests, scenarios, audits, diagnostics)` as attached. +struct AttachedResources<'py> { + seeds: Bound<'py, PyAny>, + functions: Bound<'py, PyAny>, + sources: Bound<'py, PyAny>, + checks: Bound<'py, PyAny>, +} + +/// Attach discovered metadata into the first compile input snapshot. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_compile_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_compile_inputs", options, (&REQUIRED, &OPTIONAL))?; + let collector: Bound<'py, PyAny> = + imported(py, COLLECTOR_MODULE, "collect_compile_diagnostics")?.call0()?; + let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + within(&collector, |violations| { + Ok((compile_inputs(py, &arguments)?, violations.clone())) + })?; + let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + if !collected.is_truthy()? { + return Ok(built); + } + let diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + built + .getattr("diagnostics")? + .try_iter()? + .chain(collected.try_iter()?) + .collect::>>()?, + )?; + call_with( + &imported(py, INPUTS_WITH_MODULE, "inputs_with")?, + &[("inputs", built), ("diagnostics", diagnostics.into_any())], + ) +} + +/// The undecorated `build_compile_inputs`. +fn compile_inputs<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { + let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; + let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; + let run_id: Bound<'py, PyAny> = call_with( + &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, + &[("selected_run_id", arguments.value("run_id")?)], + )?; + let loaded_macros: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "load_project_macros")? + .call1((discovered.getattr("macro_files")?,))?; + let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; + let declaration_scope: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATION_SCOPE_MODULE, "build_declaration_scope")?, + &[ + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros), + ("sql_lexical_syntax", syntax.clone()), + ("compile_cache_dir", target.compile_cache_dir.clone()), + ], + )?; + let value_renderer: Bound<'py, PyAny> = adapter_context.getattr("value_renderer")?; + let model_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "ModelInputBuildContext")?, + &[ + ("effective_vars", target.vars.clone()), + ("effective_settings", target.settings.clone()), + ("target_config", target.target.clone()), + ("effective_target_name", target.target_name.clone()), + ("run_id", run_id.clone()), + ("macro_context", target.macro_context.clone()), + ("loaded_macros", declaration_scope.getattr("loaded_macros")?), + ("value_renderer", value_renderer.clone()), + ( + "collection_rendering", + adapter_context.getattr("collection_rendering")?, + ), + ("sql_lexical_syntax", syntax.clone()), + ( + "declaration_resolver", + declaration_scope.getattr("resolver")?, + ), + ], + )?; + let models: ModelBuild<'py> = model_build(arguments, &discovered, &model_context)?; + let attached: AttachedResources<'py> = attached_resources( + arguments, + &adapter_context, + (&discovered, &declaration_scope), + &models, + )?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("discovered_inputs", discovered.clone()), + ("sql_lexical_syntax", syntax), + ("run_id", run_id), + ("effective_target_name", target.target_name.clone()), + ("effective_target", target.target.clone()), + ("compile_cache_dir", target.compile_cache_dir.clone()), + ( + "effective_connection", + effective_connection(arguments, &discovered, &target)?, + ), + ("effective_settings", target.settings.clone()), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ("effective_vars", target.vars.clone()), + ("macro_context", target.macro_context.clone()), + ("loaded_macros", declaration_scope.getattr("loaded_macros")?), + ("public_enums", models.enums.clone()), + ("public_constants", models.constants.clone()), + ("model_inputs", models.inputs.clone()), + ("seed_inputs", attached.seeds), + ("source_inputs", attached.sources), + ("sql_function_inputs", attached.functions), + ("test_inputs", attached.checks.get_item(0)?), + ("scenario_inputs", attached.checks.get_item(1)?), + ("audit_inputs", attached.checks.get_item(2)?), + ("diagnostics", attached.checks.get_item(3)?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ("scope_index", declaration_scope.getattr("index")?), + ("declaration_scope", declaration_scope), + ], + )?; + call_with( + &imported(py, INPUTS_VIEW_MODULE, "compile_inputs_view")?, + &[ + ("fields", fields.into_any()), + ("native_project", models.native_project), + ], + ) +} + +/// The selected target with the macro call store, and the effective vars, settings and macro context. +fn target_context<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + adapter_context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let selected: Bound<'py, PyAny> = call_with( + &imported( + py, + CALL_STORE_MODULE, + "target_context_with_macro_call_store", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", arguments.value("selected_target")?), + ("no_cache", arguments.value_or_false("no_cache")?), + ], + )?; + let target_name: Bound<'py, PyAny> = selected.get_item(0)?; + let target: Bound<'py, PyAny> = selected.get_item(1)?; + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { + PyDict::new(py).into_any() + } else { + cli_vars + }; + let vars: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_effective_vars")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("target_config", target.clone()), + ("cli_vars", cli_vars), + ], + )?; + let settings: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_effective_settings")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ], + )?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config), + ], + )?; + let macro_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ("adapter_name", adapter_name), + ("sql_analysis_enabled", settings.getattr("sql_analysis")?), + ("target_name", target_name.clone()), + ("vars", vars.clone()), + ( + "_value_renderer", + adapter_context.getattr("value_renderer")?, + ), + ( + "_collection_rendering", + adapter_context.getattr("collection_rendering")?, + ), + ( + "_enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + Ok(TargetContext { + target_name, + target, + compile_cache_dir: selected.get_item(2)?, + vars, + settings, + macro_context, + }) +} + +/// `_build_models_with_declarations`: attach every model with the public declaration indexes. +fn model_build<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let native: Bound<'py, PyModule> = py.import(NATIVE_MODULE)?; + let declarations: Bound<'py, PyAny> = native + .getattr("public_declaration_indexes")? + .call1((discovered,))?; + let enums: Bound<'py, PyAny> = declarations.get_item(0)?; + let constants: Bound<'py, PyAny> = declarations.get_item(1)?; + let schemas: Bound<'py, PyAny> = native + .getattr("public_model_schema_index")? + .call1((discovered,))?; + let native_project: Bound<'py, PyAny> = native.getattr("NativeCompiledProject")?.call0()?; + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let model_context: Bound<'py, PyAny> = replace.call( + (context,), + Some(&keywords( + py, + &[ + ("public_enums", enums.clone()), + ("public_constants", constants.clone()), + ("public_model_schemas", schemas), + ], + )?), + )?; + let inputs: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_model_inputs")?, + &[ + ("discovered_inputs", discovered.clone()), + ("context", model_context), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ( + "defer_model_sql_validation", + arguments.value_or_false("defer_model_sql_validation")?, + ), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ("native_project", native_project.clone()), + ], + )?; + let context: Bound<'py, PyAny> = replace.call( + (context,), + Some(&keywords( + py, + &[ + ("public_enums", enums.clone()), + ("public_constants", constants.clone()), + ], + )?), + )?; + Ok(ModelBuild { + inputs, + native_project, + enums, + constants, + context, + }) +} + +/// The seeds, functions, sources and checks attached to the models. +fn attached_resources<'py>( + arguments: &KeywordArguments<'py>, + adapter_context: &Bound<'py, PyAny>, + (discovered, declaration_scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + models: &ModelBuild<'py>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let context: &Bound<'py, PyAny> = &models.context; + let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let resources: Bound<'py, PyAny> = call_with( + &imported(py, ATTACH_RESOURCES_MODULE, "attach_resources")?, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings.clone(), + context.getattr("target_config")?, + context.getattr("macro_context")?.getattr("adapter_name")?, + ], + )? + .into_any(), + ), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ( + "python_functions_inherit_default_namespace", + adapter_context.getattr("python_functions_inherit_default_namespace")?, + ), + ], + )?; + let sources: Bound<'py, PyAny> = resources.get_item(2)?; + let checks: Bound<'py, PyAny> = call_with( + &imported(py, ATTACH_CHECKS_MODULE, "attach_checks")?, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings, + adapter_context.getattr("value_renderer")?, + context.getattr("sql_lexical_syntax")?, + ], + )? + .into_any(), + ), + ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ("source_inputs", sources.clone()), + ], + )?; + Ok(AttachedResources { + seeds: resources.get_item(0)?, + functions: resources.get_item(1)?, + sources, + checks, + }) +} + +/// The resolved connection when given, otherwise the configured one for the target. +fn effective_connection<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + target: &TargetContext<'py>, +) -> PyResult> { + let resolved: Bound<'py, PyAny> = arguments.value("resolved_connection")?; + if !resolved.is_none() { + return Ok(resolved); + } + call_with( + &imported( + discovered.py(), + ATTACHMENT_MODULE, + "build_effective_connection", + )?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_config", target.target.clone()), + ("effective_vars", target.vars.clone()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_compile_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 2aa258368d..5641901225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,6 +1,14 @@ pub(crate) mod analysis; +pub(crate) mod assembly; +pub(crate) mod assembly_analysis; +pub(crate) mod assembly_checks; +pub(crate) mod assembly_models; +pub(crate) mod assembly_project; pub(crate) mod command; +pub(crate) mod discovery; +pub(crate) mod inputs; pub(crate) mod phases; +pub(crate) mod pipeline; pub(crate) mod project; pub(crate) mod python_calls; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index ee36509c2c..a15255bdf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -2,13 +2,13 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyTuple}; +use pyo3::types::{PyBool, PyBytes, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; +use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const DAG_MODULE: &str = "sqlbuild.compiler.dag.main.build"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; const ARTIFACT_BATCH_MODULE: &str = "sqlbuild.cli.commands.classes.native_artifact_batch"; const TARGET_WRITER_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.target_writer"; @@ -120,16 +120,14 @@ pub(crate) fn write_dag_artifact<'py>( &[("discovered_inputs", discovered.clone())], )?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; - let dag_json: Bound<'py, PyAny> = call_with( - &imported(py, DAG_MODULE, "build_dag_json")?, - &[ - ("graph", analysis.getattr("graph")?), - ( - "project_name", - discovered.getattr("project_config")?.getattr("name")?, - ), - ("python_graph", python_graph), - ], + let project_name: String = discovered + .getattr("project_config")? + .getattr("name")? + .extract()?; + let dag_json: String = build_dag_json( + &analysis.getattr("graph")?, + &project_name, + Some(&python_graph), )?; let batch: Bound<'py, PyAny> = call_with( &imported(py, ARTIFACT_BATCH_MODULE, "NativeArtifactBatch")?, @@ -142,7 +140,7 @@ pub(crate) fn write_dag_artifact<'py>( &batch.getattr("queue")?, &[ ("path", path), - ("contents", dag_json.call_method1("encode", ("utf-8",))?), + ("contents", PyBytes::new(py, dag_json.as_bytes()).into_any()), ], )?; batch.call_method0("flush")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs new file mode 100644 index 0000000000..482282183c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -0,0 +1,420 @@ +//! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PySet, PySetMethods}; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const GRAPH_HELPERS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.graph"; +const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; +const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; +const RULES_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; +const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; +const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; +const PLAN_MODULE: &str = "sqlbuild.compiler.pipeline.main._plan_result"; +const GENERIC_DIALECT: &str = "generic"; + +/// The current build's phase timing tracker, when one is recording. +fn timing_tracker(py: Python<'_>) -> PyResult> { + imported(py, TIMING_TRACKER_MODULE, "BuildPhaseTimingTracker")?.call_method0("current") +} + +/// Report `message` through `on_progress` when a callback is set. +fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + if !on_progress.is_none() { + on_progress.call1((message,))?; + } + Ok(()) +} + +/// Compile a project without opening a warehouse planning connection. +#[pyfunction] +#[pyo3(signature = (**options))] +fn compile_project_phase<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "compile_project_phase", + options, + ( + &["discovered_inputs", "adapter"], + &["options", "on_progress"], + ), + )?; + compiled_phase( + ( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + ), + &arguments.value("options")?, + &arguments.value("on_progress")?, + ) +} + +/// `compile_project_phase` with resolved arguments. +fn compiled_phase<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + options: &Bound<'py, PyAny>, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let options: Bound<'py, PyAny> = if options.is_truthy()? { + options.clone() + } else { + imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + }; + let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; + let connection_config: Bound<'py, PyAny> = if configured.is_none() { + call_with( + &imported( + py, + EFFECTIVE_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", options.getattr("selected_target")?), + ("cli_vars", options.getattr("cli_vars")?), + ], + )? + } else { + configured + }; + progress(on_progress, "Compiling project...")?; + let started: f64 = monotonic(py)?; + let compiled: PyResult> = (|| { + let selection: Bound<'py, PyAny> = call_with( + &imported( + py, + ANALYSIS_SELECTION_MODULE, + "resolve_compile_analysis_selection", + )?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.getattr("selected_target")?), + ("no_sql_validation", options.getattr("no_sql_validation")?), + ("cli_vars", options.getattr("cli_vars")?), + ( + "external_sql_reference_resolver", + options.getattr("external_sql_reference_resolver")?, + ), + ("resolved_connection", connection_config.clone()), + ("analysis_selection", selection), + ], + ) + })(); + let compile_seconds: f64 = monotonic(py)? - started; + let tracker: Bound<'py, PyAny> = timing_tracker(py)?; + if !tracker.is_none() { + tracker.setattr("compile_seconds", compile_seconds)?; + } + let project: Bound<'py, PyAny> = compiled?; + reject_compile_errors(&project, &options, on_progress)?; + progress( + on_progress, + &format!("Compiled project. ({compile_seconds:.2}s)"), + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompiledProjectPhaseResult")?, + &[ + ("project", project), + ("connection_config", connection_config), + ( + "compile_seconds", + compile_seconds.into_pyobject(py)?.into_any(), + ), + ], + ) +} + +/// Raise one `CompileInputError` listing every compile error the command must stop on. +fn reject_compile_errors( + project: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, + on_progress: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = project.py(); + let plan_sql_tests: bool = options.getattr("plan_sql_tests")?.is_truthy()?; + let sql_test: Bound<'_, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_TEST")?; + let mut errors: Vec> = Vec::new(); + for diagnostic in project.getattr("diagnostics")?.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? + && (plan_sql_tests || diagnostic.getattr("resource_type")?.ne(&sql_test)?) + { + errors.push(diagnostic); + } + } + let Some(first) = errors.first() else { + return Ok(()); + }; + progress(on_progress, "Project compilation failed.")?; + let mut lines: Vec = Vec::with_capacity(errors.len()); + let helps: Bound<'_, PySet> = PySet::empty(py)?; + for diagnostic in &errors { + let or_one = |name: &str| -> PyResult { + let value: Bound<'_, PyAny> = diagnostic.getattr(name)?; + Ok(if value.is_truthy()? { + value.str()?.to_string() + } else { + "1".to_owned() + }) + }; + lines.push(format!( + "[{}] {}:{}:{}: {}", + diagnostic.getattr("code")?.str()?, + diagnostic.getattr("path")?.str()?, + or_one("line")?, + or_one("column")?, + diagnostic.getattr("message")?.str()?, + )); + helps.add(diagnostic.getattr("help")?)?; + } + let help: Bound<'_, PyAny> = if helps.len() == 1 { + helps.pop().unwrap_or_else(|| py.None().into_bound(py)) + } else { + py.None().into_bound(py) + }; + let error: Bound<'_, PyAny> = imported(py, COMPILE_EXCEPTIONS_MODULE, "CompileInputError")? + .call( + (lines.join("\n"),), + Some(&keywords( + py, + &[("code", first.getattr("code")?), ("help", help)], + )?), + )?; + Err(PyErr::from_value(error)) +} + +/// Run compile inputs, assembly, planning, and manifest generation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_compile_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_compile_pipeline", + options, + (&["discovered_inputs", "adapter"], &["options", "hooks"]), + )?; + let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_SELECTION_MODULE, "apply_unit_test_selectors")?, + &[ + ("options", arguments.value("options")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let hooks: Bound<'py, PyAny> = arguments.value("hooks")?; + let hooks: Bound<'py, PyAny> = if hooks.is_none() { + imported(py, RUNTIME_MODELS_MODULE, "ConnectionHooks")?.call0()? + } else { + hooks + }; + let on_progress: Bound<'py, PyAny> = hooks.getattr("on_progress")?; + let compiled: Bound<'py, PyAny> = + compiled_phase((&discovered, &adapter), &options, &on_progress)?; + let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let graph: Bound<'py, PyAny> = + imported(py, GRAPH_HELPERS_MODULE, "build_project_graph_impl")?.call1((&project,))?; + run_rules_gate( + (&discovered, &adapter), + (&options, &on_progress), + (&project, &graph), + )?; + planned((&discovered, &adapter), (&options, &hooks), &compiled) +} + +/// Run the configured rules and stop the command when any of them finds a fault. +fn run_rules_gate<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (options, on_progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, graph): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<()> { + let py: Python<'py> = discovered.py(); + let rules_error: Bound<'py, PyAny> = imported(py, RULE_EXCEPTIONS_MODULE, "RulesError")?; + let project_dir: Bound<'py, PyAny> = discovered.getattr("project_dir")?; + if project_dir.is_none() { + return Err(PyErr::from_value( + rules_error.call1(("compiler rules require a resolved project directory",))?, + )); + } + let mut config: Bound<'py, PyAny> = call_with( + &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + if options.getattr("no_cache")?.is_truthy()? { + let cache: Bound<'py, PyAny> = call_with( + &imported(py, RULE_MODELS_MODULE, "RulesCacheConfig")?, + &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], + )?; + config = imported(py, "dataclasses", "replace")? + .call((config,), Some(&keywords(py, &[("cache", cache)])?))?; + } + progress(on_progress, "Evaluating configured rules...")?; + let started: f64 = monotonic(py)?; + let dialect: Bound<'py, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = if dialect.is_truthy()? { + dialect + } else { + GENERIC_DIALECT.into_pyobject(py)?.into_any() + }; + let selected_keys: Bound<'py, PyAny> = call_with( + &imported( + py, + ANALYSIS_SELECTION_MODULE, + "resolve_configured_rule_selection", + )?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered.clone()), + ("project", project.clone()), + ], + )?; + let result: Bound<'py, PyAny> = call_with( + &imported(py, RUN_RULES_MODULE, "run_rules")?, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", config), + ("project_dir", project_dir), + ("dialect", dialect), + ("selected_keys", selected_keys), + ("no_sql_analysis", options.getattr("no_sql_validation")?), + ], + )?; + if !on_progress.is_none() { + let elapsed: f64 = monotonic(py)? - started; + let message: Bound<'py, PyAny> = call_with( + &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, + &[ + ("elapsed_seconds", elapsed.into_pyobject(py)?.into_any()), + ("result", result.clone()), + ( + "note_skipped_rules", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + on_progress.call1((message,))?; + } + let findings: Bound<'py, PyAny> = result.getattr("findings")?; + if !findings.is_truthy()? { + return Ok(()); + } + let summary: Bound<'py, PyAny> = call_with( + &imported(py, RULE_MODELS_MODULE, "RulesResult")?, + &[ + ("findings", findings), + ("evaluated_models", result.getattr("evaluated_models")?), + ( + "unevaluated_resources", + result.getattr("unevaluated_resources")?, + ), + ("cache_hits", result.getattr("cache_hits")?), + ("cache_misses", result.getattr("cache_misses")?), + ], + )?; + let rendered: Bound<'py, PyAny> = call_with( + &imported(py, RULES_RESULT_MODULE, "format_result")?, + &[ + ("result", summary), + ( + "json_output", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + Err(PyErr::from_value(rules_error.call1((format!( + "configured compiler rules failed:\n{}", + rendered.str()? + ),))?)) +} + +/// Open the planning connection, plan, and record the compile and planning times. +fn planned<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (options, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + compiled: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let tracker: Bound<'py, PyAny> = timing_tracker(py)?; + let started: f64 = monotonic(py)?; + let mut connection: Option> = None; + let result: PyResult> = (|| { + let opened: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, + &[ + ("adapter", adapter.clone()), + ("connection_config", compiled.getattr("connection_config")?), + ("hooks", hooks.clone()), + ], + )?; + connection = Some(opened.clone()); + call_with( + &imported(py, PLAN_MODULE, "build_pipeline_result")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("connection", opened), + ("project", compiled.getattr("project")?), + ("options", options.clone()), + ("on_progress", hooks.getattr("on_progress")?), + ], + ) + })(); + if let Some(connection) = connection.filter(|connection| !connection.is_none()) { + adapter.call_method1("close", (connection,))?; + } + let planning_seconds: f64 = monotonic(py)? - started; + if !tracker.is_none() { + tracker.setattr("planning_seconds", planning_seconds)?; + } + let result: Bound<'py, PyAny> = result?; + imported(py, "dataclasses", "replace")?.call( + (result,), + Some(&keywords( + py, + &[ + ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "planning_seconds", + planning_seconds.into_pyobject(py)?.into_any(), + ), + ], + )?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_project_phase, module)?)?; + module.add_function(wrap_pyfunction!(run_compile_pipeline, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 499b76e4b3..797e9b18bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -11,8 +11,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; -const BUILD_INPUTS_MODULE: &str = "sqlbuild.compiler.compile.main._build_compile_inputs"; -const ASSEMBLE_MODULE: &str = "sqlbuild.compiler.compile.main._assemble_project"; +const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; +const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_conflicts"; @@ -52,7 +52,7 @@ struct ProjectRequest { } /// `column_lineage_mode`, defaulting to `ColumnLineageMode.FAST` when not passed. -fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { +pub(crate) fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { let mode: Bound<'py, PyAny> = arguments.value("column_lineage_mode")?; if arguments.passed("column_lineage_mode")? { return Ok(mode); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs index 84ca7fe197..22c3e7f7b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; /// `module.name`, imported. pub(crate) fn imported<'py>( @@ -123,6 +123,15 @@ impl<'py> KeywordArguments<'py> { self.value(name)?.is_truthy() } + /// The argument `name`, or `False` when it was not passed. + pub(crate) fn value_or_false(&self, name: &str) -> PyResult> { + let py: Python<'py> = self.values.py(); + Ok(self + .values + .get_item(name)? + .unwrap_or_else(|| PyBool::new(py, false).to_owned().into_any())) + } + /// Whether `name` was passed. pub(crate) fn passed(&self, name: &str) -> PyResult { self.values.contains(name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index fed37dac7a..46cbaf85e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -2,10 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_driver::{analysis, command, project}; +use crate::bindings::_helpers::compile_driver::{ + analysis, assembly, command, discovery, inputs, pipeline, project, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; + assembly::register(module)?; command::register(module)?; - project::register(module) + project::register(module)?; + inputs::register(module)?; + discovery::register(module)?; + pipeline::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index d96a085d1f..c502f0aaef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -220,7 +220,7 @@ impl Conversion { } /// `json.dumps` writes a float key as `float.__repr__`, or as `NaN` / `Infinity`. -fn float_key(key: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn float_key(key: &Bound<'_, PyAny>) -> PyResult { let number: f64 = key.extract::()?; if number.is_nan() { return Ok("NaN".to_owned()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index d500e6fba9..c7f9412a8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; +use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; @@ -57,8 +58,6 @@ pub(crate) struct DiagnosticSettings { loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. python_resources: Vec, - /// The Python locators module: YAML entry lines, MODEL header spans and decorators. - locators: Py, } /// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. @@ -432,16 +431,13 @@ fn function_description<'py>( ("function", name, Some(resource_type)), attached.description.as_deref(), || { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("contents", &contents)?; - arguments.set_item("decorator", PYTHON_UDF_DECORATOR)?; - let (line, column): (usize, usize) = diagnostics - .settings - .locators - .bind(py) - .call_method("decorator_location", (), Some(&arguments))? - .extract()?; - diagnostics.location(&path, (line, column), None) + let version = py.version_info(); + let position: (usize, usize) = decorator_location( + &contents, + PYTHON_UDF_DECORATOR, + (version.major, version.minor), + ); + diagnostics.location(&path, position, None) }, &help, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs new file mode 100644 index 0000000000..a85b7a51d6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -0,0 +1,57 @@ +//! `build_dag_json`: the static DAG artifact of a compiled project graph. + +use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::{PyRef, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; +use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; +use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; + +const SPEC_EXCEPTIONS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; + +fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { + match error { + DagArtifactError::LoaderDestination(message) => py + .import(SPEC_EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("SpecConfigError")) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + DagArtifactError::MissingParameterType(name) => PyKeyError::new_err(name), + DagArtifactError::Unencodable(message) => PyTypeError::new_err(message), + DagArtifactError::Emit(JsonEmitError::NestingTooDeep) => { + PyRecursionError::new_err("maximum recursion depth exceeded") + } + DagArtifactError::Emit(error) => PyValueError::new_err(format!("{error:?}")), + } +} + +/// The DAG JSON of `graph` (a `ProjectGraph`) and the optional discovered Python node graph. +#[pyfunction] +#[pyo3(signature = (*, graph, project_name, python_graph=None))] +pub(crate) fn build_dag_json( + graph: &Bound<'_, PyAny>, + project_name: &str, + python_graph: Option<&Bound<'_, PyAny>>, +) -> PyResult { + compiler_guard(|| { + let py = graph.py(); + let project: DagProjectInput = project_input(&graph.getattr("project")?)?; + let python: Option = python_graph + .filter(|python_graph| !python_graph.is_none()) + .map(python_graph_input) + .transpose()?; + let native: Bound<'_, PyAny> = graph.getattr("native")?; + let native: PyRef<'_, NativeProjectGraph> = native.extract()?; + build(native.graph(), &project, python.as_ref(), project_name) + .map_err(|error| python_error(error, py)) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_dag_json, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs new file mode 100644 index 0000000000..96eb544955 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod artifact; +pub(crate) mod registration; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs new file mode 100644 index 0000000000..29c2baeab9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -0,0 +1,9 @@ +//! Register the DAG artifact builder. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dag_artifact::artifact; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + artifact::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs new file mode 100644 index 0000000000..1cb78168a9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs @@ -0,0 +1,462 @@ +//! Read the compiled project views and the Python node graph into DAG artifact inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::dag_artifact::models::{ + DagAuditInput, DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, + DagModelSchemaInput, DagProjectInput, DagPythonAssetInput, DagPythonGraphInput, + DagPythonNodeInput, DagRelationInput, DagScenarioInput, DagSeedInput, DagSourceEntryInput, + DagSourceInput, DagSqlTestCaseInput, DagSqlTestInput, +}; +use sqlbuild_analysis::dag_artifact::types::DagValue; + +use sqlbuild_analysis::dag_artifact::types::DagColumnLineage; + +use crate::bindings::_helpers::python_values::values::python_value; + +const AUDIT_SEVERITY_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const DECIMAL_KIND: &str = "decimal"; +const ASSET_KIND: &str = "asset"; +const CHECK_KIND: &str = "check"; +const DEFAULT_MATERIALIZATION: &str = "view"; + +fn attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attribute(value, name)?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)?.extract() +} + +/// `str(value.)`. +fn path_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(attribute(value, name)?.str()?.to_string()) +} + +/// The `.value` of an optional enum attribute. +fn enum_value(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let member: Bound<'_, PyAny> = attribute(value, name)?; + if member.is_none() { + return Ok(None); + } + member.getattr("value")?.extract().map(Some) +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + attribute(value, name)?.try_iter()?.collect() +} + +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok((text(key, "resource_type")?, text(key, "name")?)) +} + +fn keys(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(object_key).collect() +} + +fn column(column: &Bound<'_, PyAny>) -> PyResult { + Ok(DagColumnInput { + name: text(column, "name")?, + data_type: optional_text(column, "type")?, + nullable: attribute(column, "nullable")?.extract()?, + description: optional_text(column, "description")?, + meta: dag_value(&attribute(column, "meta")?)?, + }) +} + +fn columns(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(column).collect() +} + +fn named_types(value: &Bound<'_, PyAny>, name: &str) -> PyResult)>> { + items(value, name)? + .iter() + .map(|item| Ok((text(item, "name")?, optional_text(item, "type")?))) + .collect() +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(DagRelationInput { + database: optional_text(value, "database")?, + schema: optional_text(value, "schema")?, + name: optional_text(value, "name")?, + qualified_name: optional_text(value, "qualified_name")?, + logical_database: optional_text(value, "logical_database")?, + logical_schema: optional_text(value, "logical_schema")?, + }) +} + +fn source(source: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(source, "source_entry")?; + Ok(DagSourceInput { + key: object_key(&attribute(source, "key")?)?, + name: text(source, "name")?, + path: path_text(&attribute(source, "source_file")?, "relative_path")?, + entry: DagSourceEntryInput { + name: text(&entry, "name")?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?, + table: optional_text(&entry, "table")?, + description: optional_text(&entry, "description")?, + loader: optional_text(&entry, "loader")?, + meta: dag_value(&attribute(&entry, "meta")?)?, + columns: columns(&entry, "columns")?, + }, + }) +} + +fn loaders(project: &Bound<'_, PyAny>) -> PyResult> { + let loaders: Vec> = items(project, "loader_functions")?; + let names: Bound<'_, PyDict> = PyDict::new(project.py()); + for loader in &loaders { + names.set_item(attribute(loader, "function")?, attribute(loader, "name")?)?; + } + loaders + .iter() + .map(|loader| { + let mut dependency_names: Vec = Vec::new(); + for dependency in items(loader, "depends_on")? { + if let Some(name) = names.get_item(dependency)? + && !name.is_none() + { + dependency_names.push(name.extract()?); + } + } + Ok(DagLoaderInput { + name: text(loader, "name")?, + path: path_text(loader, "relative_path")?, + destination: optional_text(loader, "destination")?, + columns: columns(loader, "columns")?, + dependency_names, + }) + }) + .collect() +} + +fn seed(seed: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(seed, "schema_entry")?; + Ok(DagSeedInput { + key: object_key(&attribute(seed, "key")?)?, + name: text(seed, "name")?, + path: path_text(&attribute(seed, "seed_file")?, "relative_path")?, + destination: relation(&attribute(seed, "destination")?)?, + description: optional_text(&entry, "description")?, + tags: texts(&entry, "tags")?, + meta: dag_value(&attribute(&entry, "meta")?)?, + columns: columns(&entry, "columns")?, + }) +} + +fn function(function: &Bound<'_, PyAny>) -> PyResult { + Ok(DagFunctionInput { + name: text(function, "name")?, + path: path_text(function, "relative_path")?, + destination: relation(&attribute(function, "destination")?)?, + body_sql: text(function, "body_sql")?, + language: enum_value(function, "language")?.unwrap_or_default(), + tags: texts(function, "tags")?, + returns: text(function, "returns")?, + return_columns: named_types(function, "return_columns")?, + arguments: items(function, "arguments")? + .iter() + .map(|argument| Ok((text(argument, "name")?, text(argument, "type")?))) + .collect::>()?, + }) +} + +fn model_schema(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + Ok(Some(DagModelSchemaInput { + description: optional_text(entry, "description")?, + tags: texts(entry, "tags")?, + meta: dag_value(&attribute(entry, "meta")?)?, + columns: columns(entry, "columns")?, + })) +} + +/// `str()` of each configured tag when the `tags` value is a list or tuple. +fn config_tags(values: &Bound<'_, PyAny>) -> PyResult>> { + let tags: Bound<'_, PyAny> = + values.call_method1("get", ("tags", PyTuple::empty(values.py())))?; + if !(tags.is_instance_of::() || tags.is_instance_of::()) { + return Ok(None); + } + tags.try_iter()? + .map(|tag| Ok(tag?.str()?.to_string())) + .collect::>() + .map(Some) +} + +fn model(model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = attribute(model, "config")?.getattr("values")?; + let inferred: Bound<'_, PyAny> = attribute(model, "inferred_columns")?; + Ok(DagModelInput { + key: object_key(&attribute(model, "key")?)?, + name: text(model, "name")?, + path: path_text(model, "relative_path")?, + destination: relation(&attribute(model, "destination")?)?, + authored_sql: text(model, "authored_sql")?, + schema: model_schema(&attribute(model, "schema_entry")?)?, + config_tags: config_tags(&values)?, + materialization: values + .call_method1("get", ("materialized", DEFAULT_MATERIALIZATION))? + .str()? + .to_string(), + inferred_columns: if inferred.is_none() { + None + } else { + Some(named_types(model, "inferred_columns")?) + }, + }) +} + +fn sql_test_case(test: &Bound<'_, PyAny>) -> PyResult> { + let Some(case_name) = optional_text(test, "case_name")? else { + return Ok(None); + }; + let parameter_schema: Vec<(String, String, bool)> = items(test, "parameter_schema")? + .iter() + .map(|parameter| { + Ok(( + text(parameter, "name")?, + enum_value(parameter, "value_type")?.unwrap_or_default(), + attribute(parameter, "nullable")?.extract()?, + )) + }) + .collect::>()?; + let mut parameters: Vec<(String, DagValue)> = Vec::new(); + for pair in items(test, "parameter_values")? { + let (name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let raw: Bound<'_, PyAny> = attribute(&value, "value")?; + let is_decimal: bool = enum_value(&value, "kind")?.as_deref() == Some(DECIMAL_KIND); + let converted: DagValue = if is_decimal { + DagValue::String(raw.str()?.to_string()) + } else { + dag_value(&raw)? + }; + parameters.push((name, converted)); + } + Ok(Some(DagSqlTestCaseInput { + case_name, + block_index: attribute(test, "block_index")?.extract()?, + parent_name: optional_text(test, "parent_name")?, + case_index: attribute(test, "case_index")?.extract()?, + case_fingerprint: optional_text(test, "case_fingerprint")?, + parameter_schema, + parameters, + })) +} + +fn sql_test(test: &Bound<'_, PyAny>) -> PyResult { + let test_file_path: Bound<'_, PyAny> = + attribute(test, "test_file")?.getattr("relative_path")?; + let source_path: Bound<'_, PyAny> = attribute(test, "source_path")?; + let source_path: Bound<'_, PyAny> = if source_path.is_none() { + test_file_path.clone() + } else { + source_path + }; + Ok(DagSqlTestInput { + key: object_key(&attribute(test, "key")?)?, + name: text(test, "name")?, + path: test_file_path.str()?.to_string(), + source_path: source_path.call_method0("as_posix")?.extract()?, + mode: enum_value(test, "mode")?.unwrap_or_default(), + scope_deps: keys(test, "scope_deps")?, + case: sql_test_case(test)?, + }) +} + +fn audit(audit: &Bound<'_, PyAny>) -> PyResult { + Ok(DagAuditInput { + key: object_key(&attribute(audit, "key")?)?, + name: text(audit, "name")?, + path: path_text(&attribute(audit, "audit_file")?, "relative_path")?, + scope_deps: keys(audit, "scope_deps")?, + attached_target_kind: enum_value(audit, "attached_target_kind")?, + attached_target_name: optional_text(audit, "attached_target_name")?, + attached_column_name: optional_text(audit, "attached_column_name")?, + severity: enum_value(audit, "severity")?, + }) +} + +fn scenario(scenario: &Bound<'_, PyAny>) -> PyResult { + Ok(DagScenarioInput { + key: object_key(&attribute(scenario, "key")?)?, + name: text(scenario, "name")?, + path: path_text(&attribute(scenario, "scenario_file")?, "relative_path")?, + expected_model_names: texts(scenario, "expected_model_names")?, + authored_cte_names: items(scenario, "authored_ctes")? + .iter() + .map(|cte| text(cte, "name")) + .collect::>()?, + assertion_names: texts(scenario, "assertion_names")?, + fixture_names: [ + texts(scenario, "source_fixture_names")?, + texts(scenario, "ref_fixture_names")?, + texts(scenario, "seed_fixture_names")?, + texts(scenario, "dbt_ref_fixture_names")?, + ], + }) +} + +/// The compiled project's resources and target defaults. +pub(crate) fn project_input(project: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectInput { + sources: items(project, "sources")? + .iter() + .map(source) + .collect::>()?, + loaders: loaders(project)?, + seeds: items(project, "seeds")? + .iter() + .map(seed) + .collect::>()?, + functions: items(project, "functions")? + .iter() + .map(function) + .collect::>()?, + models: items(project, "models")? + .iter() + .map(model) + .collect::>()?, + sql_tests: items(project, "sql_tests")? + .iter() + .map(sql_test) + .collect::>()?, + audits: items(project, "audits")? + .iter() + .map(audit) + .collect::>()?, + scenarios: items(project, "sql_scenarios")? + .iter() + .map(scenario) + .collect::>()?, + target_database: optional_text(project, "effective_target_database")?, + target_schema: optional_text(project, "effective_target_schema")?, + }) +} + +fn python_asset(node: &Bound<'_, PyAny>) -> PyResult> { + let asset: Bound<'_, PyAny> = attribute(node, "asset")?; + if asset.is_none() { + return Ok(None); + } + let lineage: Bound<'_, PyAny> = attribute(&asset, "column_lineage")?; + let column_lineage: Option = if lineage.is_none() { + None + } else { + let mut entries: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for (column, refs) in lineage.downcast::()?.iter() { + let refs: Vec<(String, String)> = refs + .try_iter()? + .map(|reference| { + let reference: Bound<'_, PyAny> = reference?; + Ok((text(&reference, "node")?, text(&reference, "column")?)) + }) + .collect::>()?; + entries.push((column.extract()?, refs)); + } + Some(entries) + }; + Ok(Some(DagPythonAssetInput { + columns: columns(&asset, "columns")?, + column_lineage, + })) +} + +fn check_severity(node: &Bound<'_, PyAny>) -> PyResult> { + let check: Bound<'_, PyAny> = attribute(node, "check")?; + if check.is_none() { + return Ok(None); + } + let severity: Bound<'_, PyAny> = check.getattr("severity")?.getattr("value")?; + node.py() + .import(AUDIT_SEVERITY_MODULE)? + .getattr("AuditSeverity")? + .call1((severity,))? + .getattr("value")? + .extract() + .map(Some) +} + +fn python_node(node: &Bound<'_, PyAny>) -> PyResult { + let kind: String = enum_value(node, "kind")?.unwrap_or_default(); + let meta: Bound<'_, PyAny> = attribute(node, "meta")?; + Ok(DagPythonNodeInput { + asset: if kind == ASSET_KIND { + python_asset(node)? + } else { + None + }, + check_severity: if kind == CHECK_KIND { + check_severity(node)? + } else { + None + }, + kind, + name: text(node, "name")?, + path: path_text(node, "relative_path")?, + description: optional_text(node, "description")?, + tags: texts(node, "tags")?, + group: optional_text(node, "group")?, + meta: if meta.is_truthy()? { + dag_value(&meta)? + } else { + DagValue::Object(Vec::new()) + }, + sql_deps: items(node, "sql_deps")? + .iter() + .map(|reference| { + Ok(( + enum_value(reference, "kind")?.unwrap_or_default(), + text(reference, "name")?, + )) + }) + .collect::>()?, + }) +} + +/// The discovered Python node graph. +pub(crate) fn python_graph_input(graph: &Bound<'_, PyAny>) -> PyResult { + let mut kinds_by_name: Vec<(String, String)> = Vec::new(); + for (name, node) in attribute(graph, "nodes_by_name")? + .downcast::()? + .iter() + { + kinds_by_name.push(( + name.extract()?, + enum_value(&node, "kind")?.unwrap_or_default(), + )); + } + Ok(DagPythonGraphInput { + nodes: items(graph, "nodes")? + .iter() + .map(python_node) + .collect::>()?, + dependency_edges: items(graph, "dependency_edges")? + .iter() + .map(|edge| Ok((text(edge, "upstream_name")?, text(edge, "downstream_name")?))) + .collect::>()?, + kinds_by_name, + }) +} + +/// A value as `asdict` copies it into the artifact. +fn dag_value(value: &Bound<'_, PyAny>) -> PyResult { + python_value(value, true) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs index 62668f7c36..6317e4be5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs @@ -1,369 +1,38 @@ -//! Native declaration collections as the plain data the Python discovery facade materialises. +//! In-memory declaration file parsing, for the oracle comparing native and Python parsing. use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::parse_declaration_text::parse_declaration_text; -use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ - AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, - DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, - FunctionFile, HookFile, MacroFile, ParsedDeclarationText, SchemaFile, ScopedFile, SeedFile, + CollectionKind, DeclarationFileOptions, ParsedDeclarationText, }; -use sqlbuild_discovery::declaration_files::types::Collection; -use sqlbuild_discovery::models::{DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot}; -use sqlbuild_discovery::tree::models::ProjectTree; -use sqlbuild_sqltext::compiler::models::AuthoredValue; -use std::path::PathBuf; -use std::sync::Arc; +use sqlbuild_discovery::models::FileOutcome; -use crate::bindings::_helpers::discovery::project_files::{ - PyObject, failure_object, object, path_object, read_object, stage_failure_object, tuple_object, -}; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{ - DeclarationDiscoveryRequest, DeclarationTextRequest, NativeDiscoverySession, NativeProjectTree, -}; +use crate::bindings::models::{DeclarationTextRequest, NativeDiscoverySession}; use crate::bindings::types::CompilerDetach; -type Located = (String, usize, usize, usize, usize); - -fn collection_kind(kind: &str) -> PyResult { - CollectionKind::from_name(kind) - .ok_or_else(|| PyValueError::new_err(format!("unknown declaration collection {kind:?}"))) -} - -fn ok_object(py: Python<'_>, mut items: Vec) -> PyResult { - items.insert(0, object(py, "ok")?); - tuple_object(py, items) -} - -fn values_object(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult { - map_to_python(py, values.to_vec()) -} - -/// `{key: value}`, or `{}` when the value is absent, for Python to project and read back. -fn single_value_object( - py: Python<'_>, - key: &str, - value: Option<&AuthoredValue>, -) -> PyResult { - map_to_python( - py, - value - .map(|value| vec![(key.to_owned(), value.clone())]) - .unwrap_or_default(), - ) -} - -fn failure_or_none(py: Python<'_>, failure: Option<&DiscoveryFailure>) -> PyResult { - failure - .map(|failure| failure_object(py, failure.clone())) - .transpose()? - .into_py_any(py) -} - -fn enum_object(py: Python<'_>, file: &EnumFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - values_object(py, &declaration.members)?, - object(py, declaration.scalar_type)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![object(py, &file.contents)?, object(py, declarations)?], - ) -} - -fn constant_object(py: Python<'_>, file: &ConstantFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - single_value_object(py, "value", Some(&declaration.value))?, - object(py, &declaration.explicit_type)?, - object(py, &declaration.render_as)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![ - object(py, &file.contents)?, - object(py, declarations)?, - failure_or_none(py, file.failure.as_ref())?, - ], - ) -} - -fn schema_object(py: Python<'_>, file: &SchemaFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - let locations: Vec = - declaration.column_locations.iter().map(located).collect(); - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - object(py, &declaration.description)?, - object(py, &declaration.extends)?, - single_value_object(py, "columns", declaration.columns.as_ref())?, - object(py, locations)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![ - object(py, &file.contents)?, - object(py, declarations)?, - failure_or_none(py, file.failure.as_ref())?, - ], - ) -} - -fn located((name, span): &(String, LineColumnSpan)) -> Located { - ( - name.clone(), - span.line, - span.column, - span.end_line, - span.end_column, - ) -} - -fn audit_object(py: Python<'_>, file: &AuditFile) -> PyResult { - let blocks: Vec = file - .blocks - .iter() - .map(|block| { - tuple_object( - py, - vec![ - object(py, block.audit_index)?, - values_object(py, &block.header_values)?, - object(py, &block.sql_body)?, - object(py, &block.name)?, - object(py, block.evaluation_mode)?, - object(py, &block.measure_sql)?, - object(py, &block.evidence_sql)?, - ], - ) - }) - .collect::>()?; - ok_object(py, vec![object(py, &file.contents)?, object(py, blocks)?]) -} - -fn hook_object(py: Python<'_>, file: &HookFile) -> PyResult { - ok_object( - py, - vec![ - object(py, &file.contents)?, - values_object(py, &file.header_values)?, - object(py, &file.sql_body)?, - object(py, &file.name)?, - object(py, &file.description)?, - ], - ) -} - -fn function_object(py: Python<'_>, file: &FunctionFile) -> PyResult { - ok_object( - py, - vec![ - object(py, &file.contents)?, - values_object(py, &file.header_values)?, - object(py, &file.body_sql)?, - ], - ) -} - -fn macro_object(py: Python<'_>, file: &MacroFile) -> PyResult { - ok_object(py, vec![object(py, &file.contents)?]) -} +const PARSED_TAG: &str = "ok"; +const FAILED_TAG: &str = "error"; +const UNREADABLE_TAG: &str = "read"; -fn seed_object(py: Python<'_>, _file: &SeedFile) -> PyResult { - ok_object(py, Vec::new()) -} - -fn outcome_object( - py: Python<'_>, - outcome: &FileOutcome, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - match outcome { - FileOutcome::Parsed(value) => parsed(py, value), - FileOutcome::Failed(failure) => failure_object(py, failure.clone()), - FileOutcome::Unreadable(failure) => read_object(py, failure.clone()), - } -} - -/// `(declaration kind, scope kind, ownership root, owning path, declaration root)`. -fn scope_object(py: Python<'_>, tree: &ProjectTree, scope: &FileScope) -> PyResult { - let path = |path: &Option| -> PyResult { - path.as_ref() - .map(|path| path_object(py, tree, path.clone())) - .transpose()? - .into_py_any(py) - }; - tuple_object( - py, - vec![ - object(py, scope.declaration_kind)?, - object(py, scope.scope_kind)?, - path(&scope.ownership_root)?, - path(&scope.owning_path)?, - path(&scope.declaration_root)?, - ], - ) -} - -/// Each file as `(relative_path, scope, payload)`, or the collection's failure payload. -fn collection_object( - py: Python<'_>, - tree: &ProjectTree, - collection: &Collection, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - let files = match collection { - Ok(files) => files, - Err(CollectionFailure::Stage(failure)) => { - return stage_failure_object(py, tree, failure.clone()); - } - Err(CollectionFailure::Layout(failure)) => return failure_object(py, failure.clone()), - }; - files - .iter() - .map(|scoped| scoped_file_object(py, tree, scoped, parsed)) - .collect::>>()? - .into_py_any(py) -} - -/// `(relative_path, scope, payload)` of one file. -fn scoped_file_object( - py: Python<'_>, - tree: &ProjectTree, - scoped: &ScopedFile, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - let scope: PyObject = scoped - .scope - .as_ref() - .map(|scope| scope_object(py, tree, scope)) - .transpose()? - .into_py_any(py)?; - tuple_object( - py, - vec![ - path_object(py, tree, scoped.file.relative_path.clone())?, - scope, - outcome_object(py, &scoped.file.outcome, parsed)?, - ], - ) -} - -fn declaration_collection_object( - py: Python<'_>, - tree: &ProjectTree, - collection: &DeclarationCollection, -) -> PyResult { - match collection { - DeclarationCollection::Enums(files) => collection_object(py, tree, files, enum_object), - DeclarationCollection::Constants(files) => { - collection_object(py, tree, files, constant_object) - } - DeclarationCollection::ModelSchemas(files) => { - collection_object(py, tree, files, schema_object) - } - DeclarationCollection::SqlFunctions(files) => { - collection_object(py, tree, files, function_object) - } - DeclarationCollection::SqlHooks(files) => collection_object(py, tree, files, hook_object), - DeclarationCollection::Audits(files) => collection_object(py, tree, files, audit_object), - DeclarationCollection::Seeds(files) => collection_object(py, tree, files, seed_object), - DeclarationCollection::Macros(files) => collection_object(py, tree, files, macro_object), - } -} - -fn options( - (function_keys, audit_keys, hook_keys): (Vec, Vec, Vec), -) -> DeclarationFileOptions { - DeclarationFileOptions { - function_keys, - audit_keys, - hook_keys, - } -} - -#[pymethods] -impl NativeDiscoverySession { - #[new] - fn new(request: DeclarationDiscoveryRequest) -> PyResult { - Ok(Self { - inner: DiscoverySession::new( - ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }, - options((request.function_keys, request.audit_keys, request.hook_keys)), - ), - }) - } - - /// Discover the collection `kind` once per session: one payload per file, or its failure. - #[pyo3(signature = (kind, tree, isolate_kind=false))] - fn collection( - &self, - py: Python<'_>, - kind: &str, - tree: &NativeProjectTree, - isolate_kind: bool, - ) -> PyResult { - let request = CollectionRequest { - kind: collection_kind(kind)?, - isolate_kind, - }; - let collection: Arc = py - .compiler_detach(|| { - Ok(retained_collection(&self.inner, request, || { - discover_declaration_collection(&self.inner, &tree.inner, request) - })) - }) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - declaration_collection_object(py, &tree.inner, &collection) - } -} - -/// Parse in-memory contents as discovery parses a `kind` declaration file named `file_path`. +/// Whether a `kind` declaration file with these contents parses (`ok`), fails or is unreadable. #[pyfunction] fn parse_declaration_contents( py: Python<'_>, request: DeclarationTextRequest, names: (String, String), contents: String, -) -> PyResult { +) -> PyResult<&'static str> { let (file_path, hook_name): (String, String) = names; - let kind: CollectionKind = collection_kind(&request.kind)?; - let options: DeclarationFileOptions = - options((request.function_keys, request.audit_keys, request.hook_keys)); + let kind: CollectionKind = CollectionKind::from_name(&request.kind).ok_or_else(|| { + PyValueError::new_err(format!("unknown declaration collection {:?}", request.kind)) + })?; + let options = DeclarationFileOptions { + function_keys: request.function_keys, + audit_keys: request.audit_keys, + hook_keys: request.hook_keys, + }; let parsed: Option> = py .compiler_detach(|| { Ok(parse_declaration_text( @@ -374,20 +43,15 @@ fn parse_declaration_contents( )) }) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let Some(outcome) = parsed else { - return Err(PyValueError::new_err(format!( + match parsed { + Some(FileOutcome::Parsed(_)) => Ok(PARSED_TAG), + Some(FileOutcome::Failed(_)) => Ok(FAILED_TAG), + Some(FileOutcome::Unreadable(_)) => Ok(UNREADABLE_TAG), + None => Err(PyValueError::new_err(format!( "{} files have no authored text to parse", kind.as_str() - ))); - }; - outcome_object(py, &outcome, |py, parsed| match parsed { - ParsedDeclarationText::Enum(file) => enum_object(py, file), - ParsedDeclarationText::Constant(file) => constant_object(py, file), - ParsedDeclarationText::ModelSchema(file) => schema_object(py, file), - ParsedDeclarationText::SqlFunction(file) => function_object(py, file), - ParsedDeclarationText::SqlHook(file) => hook_object(py, file), - ParsedDeclarationText::Audit(file) => audit_object(py, file), - }) + ))), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs new file mode 100644 index 0000000000..0277039350 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -0,0 +1,700 @@ +//! Declaration file collections discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; +use sqlbuild_discovery::declaration_files::models::{ + AuditBlock, AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, + DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, + FunctionFile, HookFile, MacroFile, SchemaFile, ScopedFile, +}; +use sqlbuild_discovery::declaration_files::types::Collection; +use sqlbuild_discovery::models::{FileOutcome, LineColumnSpan}; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::sync::Arc; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, snapshot_memo, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const SCHEMA_COLUMNS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.sql.schema_columns"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const PATHLIB_MODULE: &str = "pathlib"; +const SESSION_MEMO_KEY: &str = "native_discovery_session"; +const MEASUREMENT_MODE: &str = "MEASUREMENT"; + +/// One declaration file being built: its paths and the scope native discovery listed it in. +struct FileContext<'a, 'py> { + tree: &'a ProjectTree, + relative_path: Bound<'py, PyAny>, + file_path: Bound<'py, PyAny>, + scope: Option<&'a FileScope>, +} + +impl<'py> FileContext<'_, 'py> { + fn py(&self) -> Python<'py> { + self.relative_path.py() + } + + /// The fields every declaration file record starts with. + fn fields(&self, contents: &str) -> PyResult> { + keywords( + self.py(), + &[ + ("file_path", self.file_path.clone()), + ("relative_path", self.relative_path.clone()), + ("contents", contents.into_bound_py_any(self.py())?), + ], + ) + } + + /// Add the scope fields of a file native discovery listed in a role. + fn add_scope(&self, fields: &Bound<'py, PyDict>) -> PyResult<()> { + let Some(scope) = self.scope else { + return Ok(()); + }; + let py: Python<'py> = self.py(); + fields.set_item( + "scope_kind", + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.call1((scope.scope_kind,))?, + )?; + for (name, path) in [ + ("ownership_root", &scope.ownership_root), + ("owning_path", &scope.owning_path), + ("declaration_root", &scope.declaration_root), + ] { + let value: Bound<'py, PyAny> = match path { + Some(path) => { + path_class(py)?.call1((path_object(py, self.tree, path.clone())?,))? + } + None => py.None().into_bound(py), + }; + fields.set_item(name, value)?; + } + Ok(()) + } + + fn record(&self, class: &str, fields: &Bound<'py, PyDict>) -> PyResult> { + imported(self.py(), MODELS_MODULE, class)?.call((), Some(fields)) + } +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +fn authored<'py>( + py: Python<'py>, + values: &[(String, AuthoredValue)], +) -> PyResult> { + Ok(map_to_python(py, values.to_vec())? + .into_bound(py) + .downcast_into::()?) +} + +/// `{key: value}` or `{}`, read back as Python reads the projected value. +fn single_value<'py>( + py: Python<'py>, + key: &str, + value: Option<&AuthoredValue>, +) -> PyResult>> { + let values: Vec<(String, AuthoredValue)> = value + .map(|value| vec![(key.to_owned(), value.clone())]) + .unwrap_or_default(); + authored(py, &values)?.get_item(key) +} + +fn enum_file<'py>(context: &FileContext<'_, 'py>, file: &EnumFile) -> PyResult> { + let py: Python<'py> = context.py(); + let member_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumMember")?; + let declaration_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumDeclaration")?; + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let members: Vec> = authored(py, &declaration.members)? + .iter() + .map(|(name, value)| call_with(&member_class, &[("name", name), ("value", value)])) + .collect::>()?; + declarations.push(call_with( + &declaration_class, + &[ + ("name", declaration.name.as_str().into_bound_py_any(py)?), + ("members", PyTuple::new(py, members)?.into_any()), + ( + "scalar_type", + declaration.scalar_type.into_bound_py_any(py)?, + ), + ("relative_path", context.relative_path.clone()), + ], + )?); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredEnumFile", &fields) +} + +fn constant_file<'py>( + context: &FileContext<'_, 'py>, + file: &ConstantFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let value: Bound<'py, PyAny> = single_value(py, "value", Some(&declaration.value))? + .unwrap_or_else(|| py.None().into_bound(py)); + declarations.push(typed_constant_record( + py, + (declaration.name.clone(), value), + ( + declaration.explicit_type.clone(), + declaration.render_as.clone(), + ), + (context.file_path.clone(), context.relative_path.clone()), + )?); + } + if let Some(failure) = &file.failure { + return Err(failure_error(py, failure.clone())); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredConstantFile", &fields) +} + +/// Python's `native_locations`: a repeated name keeps its first position and last value. +pub(crate) fn source_locations<'py>( + py: Python<'py>, + relative_path: &Bound<'py, PyAny>, + rows: &[(String, LineColumnSpan)], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?; + let located: Bound<'py, PyDict> = PyDict::new(py); + for (name, span) in rows { + located.set_item( + name, + call_with( + &class, + &[ + ("path", relative_path.clone()), + ("line", span.line.into_bound_py_any(py)?), + ("column", span.column.into_bound_py_any(py)?), + ("end_line", span.end_line.into_bound_py_any(py)?), + ("end_column", span.end_column.into_bound_py_any(py)?), + ], + )?, + )?; + } + Ok(located) +} + +fn schema_file<'py>( + context: &FileContext<'_, 'py>, + file: &SchemaFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let parse_columns: Bound<'py, PyAny> = + imported(py, SCHEMA_COLUMNS_MODULE, "parse_schema_columns")?; + let declaration_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "ModelSchemaDeclaration")?; + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let raw_columns: Bound<'py, PyAny> = + single_value(py, "columns", declaration.columns.as_ref())? + .unwrap_or_else(|| py.None().into_bound(py)); + let columns: Bound<'py, PyAny> = call_with( + &parse_columns, + &[ + ("raw_columns", raw_columns), + ("file_path", context.file_path.clone()), + ( + "label", + format!("schema '{}'", declaration.name).into_bound_py_any(py)?, + ), + ( + "error_class", + imported(py, DISCOVERY_ERRORS_MODULE, "DeclarationParseError")?, + ), + ( + "column_locations", + source_locations(py, &context.relative_path, &declaration.column_locations)? + .into_any(), + ), + ("require_columns", true.into_bound_py_any(py)?), + ], + )?; + declarations.push(call_with( + &declaration_class, + &[ + ("name", declaration.name.as_str().into_bound_py_any(py)?), + ( + "description", + declaration.description.as_deref().into_bound_py_any(py)?, + ), + ( + "extends", + declaration.extends.as_deref().into_bound_py_any(py)?, + ), + ("columns", columns), + ("relative_path", context.relative_path.clone()), + ], + )?); + } + if let Some(failure) = &file.failure { + return Err(failure_error(py, failure.clone())); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredModelSchemaFile", &fields) +} + +fn function_file<'py>( + context: &FileContext<'_, 'py>, + file: &FunctionFile, +) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item( + "header_values", + authored(context.py(), &file.header_values)?, + )?; + fields.set_item("body_sql", &file.body_sql)?; + context.record("DiscoveredSqlFunctionFile", &fields) +} + +fn hook_file<'py>(context: &FileContext<'_, 'py>, file: &HookFile) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item( + "header_values", + authored(context.py(), &file.header_values)?, + )?; + fields.set_item("sql_body", &file.sql_body)?; + fields.set_item("name", &file.name)?; + fields.set_item("description", &file.description)?; + context.add_scope(&fields)?; + context.record("DiscoveredSqlHookFile", &fields) +} + +fn audit_block<'py>(py: Python<'py>, block: &AuditBlock) -> PyResult> { + let header_values: Bound<'py, PyDict> = authored(py, &block.header_values)?; + let mode: Bound<'py, PyAny> = imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? + .call1((block.evaluation_mode,))?; + let measurement: Bound<'py, PyAny> = + if mode.is(&imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? + .getattr(MEASUREMENT_MODE)?) + { + call_with( + &imported(py, AUDITING_MODELS_MODULE, "MeasurementContract")?, + &[ + ("value_column", header_values.as_any().get_item("value")?), + ( + "sample_count_column", + header_values + .as_any() + .call_method1("get", ("sample_count",))?, + ), + ( + "sample_unit", + header_values + .as_any() + .call_method1("get", ("sample_unit",))?, + ), + ], + )? + } else { + py.None().into_bound(py) + }; + call_with( + &imported(py, MODELS_MODULE, "DiscoveredAuditBlock")?, + &[ + ("audit_index", block.audit_index.into_bound_py_any(py)?), + ("header_values", header_values.into_any()), + ("sql_body", block.sql_body.as_str().into_bound_py_any(py)?), + ("name", block.name.as_deref().into_bound_py_any(py)?), + ("evaluation_mode", mode), + ("measurement_contract", measurement), + ( + "measure_sql", + block.measure_sql.as_deref().into_bound_py_any(py)?, + ), + ( + "evidence_sql", + block.evidence_sql.as_deref().into_bound_py_any(py)?, + ), + ], + ) +} + +fn audit_file<'py>( + context: &FileContext<'_, 'py>, + file: &AuditFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let blocks: Vec> = file + .blocks + .iter() + .map(|block| audit_block(py, block)) + .collect::>()?; + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("blocks", PyTuple::new(py, blocks)?)?; + let declaration_kind: &str = context.scope.map_or("", |scope| scope.declaration_kind); + fields.set_item( + "declaration_kind", + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?.call1((declaration_kind,))?, + )?; + context.add_scope(&fields)?; + context.record("DiscoveredAuditFile", &fields) +} + +fn macro_file<'py>( + context: &FileContext<'_, 'py>, + file: &MacroFile, +) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + context.add_scope(&fields)?; + context.record("DiscoveredMacroFile", &fields) +} + +/// The session of this pass, created and memoised by the first declaration collection. +fn discovery_session<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + if let Some(session) = memo.get_item(SESSION_MEMO_KEY)? + && let Ok(session) = session.downcast_into::() + { + return Ok(session); + } + let created: Bound<'py, NativeDiscoverySession> = Bound::new( + py, + NativeDiscoverySession { + inner: DiscoverySession::new( + project_root(project_dir)?, + DeclarationFileOptions { + function_keys: sorted_keys(py, "SQL_FUNCTION_HEADER_KEYS")?, + audit_keys: sorted_keys(py, "SQL_AUDIT_HEADER_KEYS")?, + hook_keys: sorted_keys(py, "SQL_HOOK_HEADER_KEYS")?, + }, + ), + }, + )?; + memo.set_item(SESSION_MEMO_KEY, &created)?; + Ok(created) +} + +/// The session this discovery pass read its declaration files with, if any. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn retained_discovery_session<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + Ok(snapshot_memo(project_dir)? + .get_item(SESSION_MEMO_KEY)? + .filter(|session| session.is_instance_of::()) + .unwrap_or_else(|| py.None().into_bound(py))) +} + +/// One collection read in the pass's snapshot, with the tree it was read from. +fn collection<'py>( + project_dir: &Bound<'py, PyAny>, + request: CollectionRequest, +) -> PyResult<(Bound<'py, NativeProjectTree>, Arc)> { + in_snapshot(project_dir, || { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let session: Bound<'py, NativeDiscoverySession> = discovery_session(project_dir)?; + let collection: Arc = { + let session: &DiscoverySession = &session.get().inner; + let inner: &ProjectTree = &tree.get().inner; + py.detach(|| { + retained_collection(session, request, || { + discover_declaration_collection(session, inner, request) + }) + }) + }; + if let Some(failure) = collection_failure(&collection) { + return Err(match failure { + CollectionFailure::Stage(failure) => { + stage_error(&tree.get().inner, project_dir, failure.clone()) + } + CollectionFailure::Layout(failure) => failure_error(py, failure.clone()), + }); + } + seed_listings(project_dir, &tree)?; + Ok((tree, collection)) + }) +} + +/// The files of a collection whose failure was already raised. +fn files_of(collection: &Collection) -> &[ScopedFile] { + collection.as_ref().map(Vec::as_slice).unwrap_or_default() +} + +fn failure_of(collection: &Collection) -> Option<&CollectionFailure> { + collection.as_ref().err() +} + +fn collection_failure(collection: &DeclarationCollection) -> Option<&CollectionFailure> { + match collection { + DeclarationCollection::Enums(files) => failure_of(files), + DeclarationCollection::Constants(files) => failure_of(files), + DeclarationCollection::ModelSchemas(files) => failure_of(files), + DeclarationCollection::SqlFunctions(files) => failure_of(files), + DeclarationCollection::SqlHooks(files) => failure_of(files), + DeclarationCollection::Audits(files) => failure_of(files), + DeclarationCollection::Seeds(files) => failure_of(files), + DeclarationCollection::Macros(files) => failure_of(files), + } +} + +/// Build each parsed file of a collection, raising or reporting each failing one. +fn records<'py, T>( + faults: &FileFaults<'py>, + tree: &ProjectTree, + files: &[ScopedFile], + build: fn(&FileContext<'_, 'py>, &T) -> PyResult>, +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let mut built: Vec> = Vec::with_capacity(files.len()); + for scoped in files { + let relative_path: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, tree, scoped.file.relative_path.clone())?,))?; + let file_path: Bound<'py, PyAny> = faults.project_dir.div(&relative_path)?; + let record: PyResult> = match &scoped.file.outcome { + FileOutcome::Parsed(file) => build( + &FileContext { + tree, + relative_path: relative_path.clone(), + file_path, + scope: scoped.scope.as_ref(), + }, + file, + ), + FileOutcome::Failed(failure) => Err(failure_error(py, failure.clone())), + FileOutcome::Unreadable(failure) => Err(read_error(failure.clone(), &file_path)), + }; + built.extend(faults.kept(&relative_path, record)?); + } + record_tuple(py, built) +} + +fn discovered<'py>( + faults: &FileFaults<'py>, + kind: CollectionKind, + isolate_kind: bool, +) -> PyResult> { + let (tree, collection) = collection( + &faults.project_dir, + CollectionRequest { kind, isolate_kind }, + )?; + let tree: &ProjectTree = &tree.get().inner; + match collection.as_ref() { + DeclarationCollection::Enums(files) => records(faults, tree, files_of(files), enum_file), + DeclarationCollection::Constants(files) => { + records(faults, tree, files_of(files), constant_file) + } + DeclarationCollection::ModelSchemas(files) => { + records(faults, tree, files_of(files), schema_file) + } + DeclarationCollection::SqlFunctions(files) => { + records(faults, tree, files_of(files), function_file) + } + DeclarationCollection::SqlHooks(files) => records(faults, tree, files_of(files), hook_file), + DeclarationCollection::Audits(files) => records(faults, tree, files_of(files), audit_file), + DeclarationCollection::Macros(files) => records(faults, tree, files_of(files), macro_file), + DeclarationCollection::Seeds(files) => seed_records(faults, tree, files_of(files)), + } +} + +fn seed_records<'py, T>( + faults: &FileFaults<'py>, + tree: &ProjectTree, + files: &[ScopedFile], +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSeedFile")?; + let seeds: Vec> = files + .iter() + .map(|scoped| { + let relative: Py = path_object(py, tree, scoped.file.relative_path.clone())?; + call_with( + &class, + &[ + ("file_path", faults.project_dir.div(relative.bind(py))?), + ("relative_path", path_class(py)?.call1((relative,))?), + ], + ) + }) + .collect::>()?; + record_tuple(py, seeds) +} + +fn faults<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> FileFaults<'py> { + FileFaults { + project_dir: project_dir.clone(), + on_fault: on_fault.filter(|on_fault| !on_fault.is_none()), + } +} + +/// Discover global and scoped enum declaration files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] +pub(crate) fn discover_enum_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Enums, + isolate_declaration_kind, + ) +} + +/// Discover global and scoped constant declaration files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] +pub(crate) fn discover_constant_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Constants, + isolate_declaration_kind, + ) +} + +/// Discover project-wide and scoped reusable model schemas. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_model_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::ModelSchemas, + false, + ) +} + +/// Discover SQL function files under functions/sql/. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_sql_function_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlFunctions, + false, + ) +} + +/// Discover named SQL lifecycle hooks in their project-wide and scoped roles. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_sql_hook_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlHooks, + false, + ) +} + +/// Discover generic and singular audit SQL files in their project-wide and scoped roles. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_audit_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Audits, + false, + ) +} + +/// Discover global and scoped project macro Python files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] +pub(crate) fn discover_macro_files<'py>( + project_dir: &Bound<'py, PyAny>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, None), + CollectionKind::Macros, + isolate_declaration_kind, + ) +} + +/// Discover seed CSV files under seeds/. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_seed_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + discovered(&faults(project_dir, None), CollectionKind::Seeds, false) +} + +/// Detect a project-level adapter.py without importing it. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_adapter_file<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; + if !file_path.call_method0("is_file")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + call_with( + &imported(py, MODELS_MODULE, "DiscoveredAdapterFile")?, + &[ + ("file_path", file_path), + ("relative_path", path_class(py)?.call1(("adapter.py",))?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; + module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs new file mode 100644 index 0000000000..8105fa7314 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs @@ -0,0 +1,62 @@ +//! Report and skip a failing file when the caller tolerates file faults. + +use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyTuple; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const FAULT_CLASS: &str = "DiscoveryFileFault"; +const PROJECT_DIR_MARKER: &str = "."; + +/// The files one discovery builds, with the caller's optional `on_fault` reporter. +pub(crate) struct FileFaults<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) on_fault: Option>, +} + +impl<'py> FileFaults<'py> { + /// Whether `on_fault` takes this error instead of the caller. + fn tolerated(&self, error: &PyErr) -> bool { + let py: Python<'py> = self.project_dir.py(); + self.on_fault.is_some() + && (error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py)) + } + + /// The built record; a tolerated failure for `relative_path` is reported and skipped. + pub(crate) fn kept( + &self, + relative_path: &Bound<'py, PyAny>, + built: PyResult>, + ) -> PyResult>> { + let error: PyErr = match built { + Ok(record) => return Ok(Some(record)), + Err(error) => error, + }; + let Some(on_fault) = self.on_fault.as_ref().filter(|_| self.tolerated(&error)) else { + return Err(error); + }; + let py: Python<'py> = self.project_dir.py(); + let message: Bound<'py, PyAny> = error + .value(py) + .str()? + .call_method1("replace", (self.project_dir.str()?, PROJECT_DIR_MARKER))?; + on_fault.call1((call_with( + &imported(py, DISCOVERY_MODELS_MODULE, FAULT_CLASS)?, + &[("path", relative_path.clone()), ("message", message)], + )?,))?; + Ok(None) + } +} + +/// The built records as the tuple a discoverer returns. +pub(crate) fn record_tuple<'py>( + py: Python<'py>, + records: Vec>, +) -> PyResult> { + PyTuple::new(py, records) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index f177fcbbee..93c2ccccc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -160,8 +160,7 @@ pub(crate) fn model_header_declarations<'py>( } /// `typed_constant_declaration` for a public constant file entry. -#[pyfunction] -fn typed_constant_record<'py>( +pub(crate) fn typed_constant_record<'py>( py: Python<'py>, constant: (String, Bound<'py, PyAny>), options: (Option, Option), @@ -193,6 +192,5 @@ fn typed_constant_record<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_header_declarations, module)?)?; - module.add_function(wrap_pyfunction!(typed_constant_record, module)?) + module.add_function(wrap_pyfunction!(model_header_declarations, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs new file mode 100644 index 0000000000..9b77e9553a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs @@ -0,0 +1,84 @@ +//! The named declaration groups of a valid declaration layout, memoised per snapshot. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; +use sqlbuild_discovery::declarations::models::{DeclarationGroup, DeclarationLayout}; +use sqlbuild_discovery::models::{DiscoveryFailure, StageFailure}; + +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{discovery_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{project_tree, seed_listings, snapshot_memo}; +use crate::bindings::models::NativeProjectTree; + +const GROUPS_MEMO_KEY: &str = "native_declaration_groups"; + +/// The rows to memoise: `(root, directory)` pairs, or the failure's `(kind, message, help)`. +fn memoised<'py>( + py: Python<'py>, + tree: &Bound<'py, NativeProjectTree>, + groups: Result, DiscoveryFailure>, +) -> PyResult> { + match groups { + Ok(groups) => { + let rows: Vec> = groups + .into_iter() + .map(|group| { + PyTuple::new( + py, + [ + path_object(py, &tree.get().inner, group.root)?, + path_object(py, &tree.get().inner, group.directory)?, + ], + ) + }) + .collect::>()?; + Ok(PyList::new(py, rows)?.into_any()) + } + Err(failure) => Ok(PyTuple::new( + py, + [ + failure.kind.as_str().into_bound_py_any(py)?, + failure.message.into_bound_py_any(py)?, + failure.help.into_bound_py_any(py)?, + ], + )? + .into_any()), + } +} + +/// The `(canonical root, group directory)` pairs of a valid named declaration layout. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn declaration_groups<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + let groups: Bound<'py, PyAny> = match memo.get_item(GROUPS_MEMO_KEY)? { + Some(groups) => groups, + None => { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let layout: Result = { + let inner = &tree.get().inner; + py.detach(|| declaration_layout(inner, None)) + }; + let layout: DeclarationLayout = + layout.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + seed_listings(project_dir, &tree)?; + let groups: Bound<'py, PyAny> = memoised(py, &tree, layout.named_groups)?; + memo.set_item(GROUPS_MEMO_KEY, &groups)?; + groups + } + }; + if let Ok(rows) = groups.downcast::() { + return Ok(rows.clone()); + } + let (kind, message, help): (String, String, Option) = groups.extract()?; + Err(discovery_error(py, &kind, message, help)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declaration_groups, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 99bd00951b..444fa2041c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,12 +1,21 @@ pub(crate) mod declaration_files; +pub(crate) mod declaration_records; +pub(crate) mod faults; pub(crate) mod globbing; pub(crate) mod header_declarations; pub(crate) mod key_checks; +pub(crate) mod layout_groups; pub(crate) mod locations; pub(crate) mod model_file_record; +pub(crate) mod model_records; pub(crate) mod project_facts; pub(crate) mod project_files; pub(crate) mod project_validation; +pub(crate) mod raised; pub(crate) mod registration; +pub(crate) mod snapshot; +pub(crate) mod sorted_keys; pub(crate) mod sql_test_blocks; +pub(crate) mod sql_test_records; pub(crate) mod yaml_entries; +pub(crate) mod yaml_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs new file mode 100644 index 0000000000..aff9e21786 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -0,0 +1,144 @@ +//! SQL model files discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::models::NativeProjectTree; + +const PATHLIB_MODULE: &str = "pathlib"; +const STEM_ATTRIBUTE: &str = "stem"; + +/// One discovered model file: its record, or the error building it raises. +type ModelItem<'py> = (Bound<'py, PyAny>, PyResult>); + +/// Discover every model file natively, each as its record or the error it raises when built. +fn model_items<'py>( + project_dir: &Bound<'py, PyAny>, + options: &ModelFileOptions, +) -> PyResult>> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root = project_root(project_dir)?; + let discovered: Result>, StageFailure> = { + let inner = &tree.get().inner; + py.detach(|| discover_models(&root, inner, options)) + }; + let files = + discovered.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let mut items: Vec> = Vec::with_capacity(files.len()); + for file in files { + let relative_path: Bound<'py, PyAny> = + path_class.call1((path_object(py, &tree.get().inner, file.relative_path)?,))?; + let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; + let built: PyResult> = match file.outcome { + FileOutcome::Parsed(model) => model_record( + py, + model, + (file_path, relative_path.clone()), + options.extract_implicit_alias_columns, + )? + .map(|record| record.into_bound(py)), + FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), + FileOutcome::Failed(failure) => Err(failure_error(py, failure)), + }; + items.push((relative_path, built)); + } + seed_listings(project_dir, &tree)?; + Ok(items) + }) +} + +/// Which discovered model files to build, and where failing files are reported. +pub(crate) struct ModelFileSelection<'py> { + pub(crate) selected_model_names: Option>, + pub(crate) on_fault: Option>, +} + +/// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. +pub(crate) fn discovered_model_files<'py>( + project_dir: &Bound<'py, PyAny>, + extraction: (bool, bool), + selection: ModelFileSelection<'py>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; + let options = ModelFileOptions { + supported_keys: sorted_keys(py, "SQL_MODEL_HEADER_KEYS")?, + removed_keys: sorted_keys(py, "REMOVED_SQL_MODEL_HEADER_KEYS")?, + extract_implicit_alias_columns, + extract_output_column_locations, + }; + let items: Vec> = model_items(project_dir, &options)?; + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), + }; + let selected: Option> = selection + .selected_model_names + .filter(|selected| !selected.is_none()); + let mut records: Vec> = Vec::with_capacity(items.len()); + for (relative_path, built) in items { + if let Some(selected) = &selected + && !selected.contains(relative_path.getattr(STEM_ATTRIBUTE)?)? + { + continue; + } + records.extend(faults.kept(&relative_path, built)?); + } + record_tuple(py, records) +} + +/// Discover the (selected) model files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn discover_model_files<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_model_files", + options, + ( + &[ + "project_dir", + "extract_implicit_alias_columns", + "extract_output_column_locations", + ], + &["selected_model_names", "on_fault"], + ), + )?; + discovered_model_files( + &arguments.value("project_dir")?, + ( + arguments + .value("extract_implicit_alias_columns")? + .extract()?, + arguments + .value("extract_output_column_locations")? + .extract()?, + ), + ModelFileSelection { + selected_model_names: Some(arguments.value("selected_model_names")?), + on_fault: Some(arguments.value("on_fault")?), + }, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_model_files, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index be9bc78a0d..5d823b4ecd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -1,193 +1,35 @@ //! Native discovery results as the plain data the Python discovery facade materialises. -use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; +use pyo3::exceptions::PyException; use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; -use sqlbuild_discovery::declarations::models::{ - DeclarationFileFact, DeclarationGroup, DeclarationKind, DeclarationLayout, -}; -use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; -use sqlbuild_discovery::models::{ - DiscoveredFile, DiscoveryFailure, FailureKind, FileOutcome, LineColumnSpan, ProjectRoot, - ReadFailure, StageFailure, -}; -use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; -use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; -use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; -use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; -use sqlbuild_discovery::sql_tests::models::{ - DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, -}; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; -use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; -use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; -use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use std::collections::HashSet; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Mutex; -use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::discovery::declaration_records::source_locations; use crate::bindings::_helpers::discovery::header_declarations::{ HeaderDeclarations, model_header_declarations, }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileParts, NativeModelFile}; +use crate::bindings::_helpers::discovery::raised::failure_error; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{ - ModelDiscoveryRequest, ModelTextRequest, NativeProjectTree, SqlTestDiscoveryRequest, - SqlTestTextRequest, YamlDiscoveryRequest, -}; +use crate::bindings::models::{ModelTextRequest, NativeProjectTree}; use crate::bindings::types::CompilerDetach; pub(crate) type PyObject = Py; -const PATHLIB_MODULE: &str = "pathlib"; -const PATH_CLASS: &str = "Path"; const STEM_ATTRIBUTE: &str = "stem"; const ENUMS_KEY: &str = "enums"; const CONSTANTS_KEY: &str = "constants"; -const RAISED_TAG: &str = "raised"; -type Located = (String, usize, usize, usize, usize); -type Listing = (PyObject, Vec<(PyObject, bool, bool)>); - -fn located(locations: Vec<(String, LineColumnSpan)>) -> Vec { - locations - .into_iter() - .map(|(name, span)| (name, span.line, span.column, span.end_line, span.end_column)) - .collect() -} - -pub(crate) fn tuple_object<'py>(py: Python<'py>, items: Vec) -> PyResult { - Ok(PyTuple::new(py, items)?.unbind().into_any()) -} - -pub(crate) fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { - value.into_py_any(py) -} - -pub(crate) fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { - tuple_object( - py, - vec![ - object(py, "error")?, - object(py, failure.kind.as_str())?, - object(py, failure.message)?, - object(py, failure.help)?, - ], - ) -} - -/// `("read", "os", errno, message, winerror)` or `("read", "decode", bytes, start, end, reason)`. -pub(crate) fn read_object(py: Python<'_>, failure: ReadFailure) -> PyResult { - match failure { - ReadFailure::Io { - errno, - winerror, - message, - } => tuple_object( - py, - vec![ - object(py, "read")?, - object(py, "os")?, - object(py, errno)?, - object(py, message)?, - object(py, winerror)?, - ], - ), - ReadFailure::Decode { - bytes, - start, - end, - reason, - } => tuple_object( - py, - vec![ - object(py, "read")?, - object(py, "decode")?, - PyBytes::new(py, &bytes).unbind().into_any(), - object(py, start)?, - object(py, end)?, - object(py, reason)?, - ], - ), - } -} - -/// A collection's failure payload; an unlistable directory is `("unlistable", path, read)`. -pub(crate) fn stage_failure_object( - py: Python<'_>, - tree: &ProjectTree, - failure: StageFailure, -) -> PyResult { - match failure { - StageFailure::Unlistable { - relative_path, - error, - } => tuple_object( - py, - vec![ - object(py, "unlistable")?, - path_object(py, tree, relative_path)?, - read_object(py, error)?, - ], - ), - StageFailure::Internal(reason) => Err(PyRuntimeError::new_err(reason)), - } -} - -fn outcome_object( - py: Python<'_>, - outcome: FileOutcome, - parsed: impl FnOnce(Python<'_>, T) -> PyResult, -) -> PyResult { - match outcome { - FileOutcome::Parsed(value) => parsed(py, value), - FileOutcome::Unreadable(failure) => read_object(py, failure), - FileOutcome::Failed(failure) => failure_object(py, failure), - } -} - -/// Each discovered file as `(relative_path, payload)`, or the collection's failure payload. -fn files_object( - py: Python<'_>, - tree: &ProjectTree, - discovered: Result>, StageFailure>, - parsed: impl Fn(Python<'_>, T) -> PyResult, -) -> PyResult { - match discovered { - Ok(files) => files - .into_iter() - .map(|file| { - Ok(( - path_object(py, tree, file.relative_path)?, - outcome_object(py, file.outcome, &parsed)?, - )) - }) - .collect::>>()? - .into_py_any(py), - Err(failure) => stage_failure_object(py, tree, failure), - } -} - -fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult { - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, model.contents)?, - map_to_python(py, model.header_values)?, - object(py, located(model.header_column_locations))?, - object(py, located(model.output_column_locations))?, - object(py, model.query_sql)?, - ], - ) -} +pub(crate) type Listing = (PyObject, Vec<(PyObject, bool, bool)>); /// A relative path as Python names it: the real, surrogate-escaped name where it is not UTF-8. pub(crate) fn path_object( @@ -196,8 +38,8 @@ pub(crate) fn path_object( relative_path: String, ) -> PyResult { match tree.raw_relative(&relative_path) { - Some(raw) => object(py, raw.into_os_string()), - None => object(py, relative_path), + Some(raw) => raw.into_os_string().into_py_any(py), + None => relative_path.into_py_any(py), } } @@ -206,83 +48,14 @@ fn entry_rows(py: Python<'_>, entries: &[TreeEntry]) -> PyResult object(py, raw)?, - None => object(py, &entry.name)?, + Some(raw) => raw.into_py_any(py)?, + None => entry.name.as_str().into_py_any(py)?, }; Ok((name, entry.is_dir, entry.is_walkable_dir)) }) .collect() } -fn fact_row(py: Python<'_>, tree: &ProjectTree, fact: DeclarationFileFact) -> PyResult { - let owning_path: Option = fact - .owning_path - .map(|path| path_object(py, tree, path)) - .transpose()?; - tuple_object( - py, - vec![ - path_object(py, tree, fact.relative_path)?, - object(py, fact.kind.as_str())?, - object(py, fact.scope_kind.as_str())?, - path_object(py, tree, fact.ownership_root)?, - object(py, owning_path)?, - path_object(py, tree, fact.declaration_root)?, - ], - ) -} - -fn group_row(py: Python<'_>, tree: &ProjectTree, group: DeclarationGroup) -> PyResult { - tuple_object( - py, - vec![ - path_object(py, tree, group.root)?, - path_object(py, tree, group.directory)?, - ], - ) -} - -/// `("ok", rows)` for a valid layout, or the failure Python raises for it. -fn layout_rows( - py: Python<'_>, - tree: &ProjectTree, - outcome: Result, DiscoveryFailure>, - row: fn(Python<'_>, &ProjectTree, T) -> PyResult, -) -> PyResult { - match outcome { - Ok(items) => { - let rows: Vec = items - .into_iter() - .map(|item| row(py, tree, item)) - .collect::>()?; - tuple_object(py, vec![object(py, "ok")?, object(py, rows)?]) - } - Err(failure) => failure_object(py, failure), - } -} - -fn declaration_kind(kind: Option<&str>) -> PyResult> { - match kind { - None => Ok(None), - Some("macro") => Ok(Some(DeclarationKind::Macro)), - Some("enum") => Ok(Some(DeclarationKind::Enum)), - Some("constant") => Ok(Some(DeclarationKind::Constant)), - Some(other) => Err(PyValueError::new_err(format!( - "unknown declaration kind {other:?}" - ))), - } -} - -fn yaml_failure_kind(kind: &str) -> PyResult { - match kind { - "source" => Ok(FailureKind::Source), - "schema" => Ok(FailureKind::Schema), - other => Err(PyValueError::new_err(format!( - "unknown YAML file kind {other:?}" - ))), - } -} - #[pymethods] impl NativeProjectTree { #[new] @@ -292,9 +65,11 @@ impl NativeProjectTree { exported: Mutex::new(HashSet::new()), } } +} - /// The listings read since the last call, for the pass's Python directory snapshot. - fn listings(&self, py: Python<'_>) -> PyResult> { +impl NativeProjectTree { + /// The listings read since the last export, each exported once. + pub(crate) fn exported_listings(&self, py: Python<'_>) -> PyResult> { let Ok(mut exported) = self.exported.lock() else { return Ok(Vec::new()); }; @@ -311,61 +86,13 @@ impl NativeProjectTree { } } -/// Each model file's `(relative path, record or failure payload)`, or the collection's failure. -#[pyfunction] -fn discover_model_files( - py: Python<'_>, - request: ModelDiscoveryRequest, - tree: &NativeProjectTree, - project_dir: Bound<'_, PyAny>, -) -> PyResult { - let root = ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }; - let options = ModelFileOptions { - supported_keys: request.supported_keys, - removed_keys: request.removed_keys, - extract_implicit_alias_columns: request.extract_implicit_alias_columns, - extract_output_column_locations: request.extract_output_column_locations, - }; - let extract_implicit_alias_columns: bool = options.extract_implicit_alias_columns; - let discovered = py - .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let files: Vec> = match discovered { - Ok(files) => files, - Err(failure) => return stage_failure_object(py, &tree.inner, failure), - }; - let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?; - files - .into_iter() - .map(|file| { - let relative_path: Bound<'_, PyAny> = - path_class.call1((path_object(py, &tree.inner, file.relative_path)?,))?; - let item: PyObject = match file.outcome { - FileOutcome::Parsed(model) => model_record( - py, - model, - (project_dir.div(&relative_path)?, relative_path.clone()), - extract_implicit_alias_columns, - )?, - FileOutcome::Unreadable(failure) => read_object(py, failure)?, - FileOutcome::Failed(failure) => failure_object(py, failure)?, - }; - Ok((relative_path.unbind(), item)) - }) - .collect::>>()? - .into_py_any(py) -} - -/// One parsed model file as its native record, or `("raised", error)` for invalid declarations. -fn model_record<'py>( +/// One parsed model file's record, or its header declaration error, raised only if it is built. +pub(crate) fn model_record<'py>( py: Python<'py>, model: DiscoveredModelFile, paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), extract_implicit_alias_columns: bool, -) -> PyResult { +) -> PyResult> { let (file_path, relative_path) = paths; let header_values: Bound<'py, PyDict> = map_to_python(py, model.header_values)? .into_bound(py) @@ -381,12 +108,7 @@ fn model_record<'py>( relative_path.clone(), ) { Ok(declarations) => declarations, - Err(error) if error.is_instance_of::(py) => { - return tuple_object( - py, - vec![object(py, RAISED_TAG)?, error.into_value(py).into_any()], - ); - } + Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), Err(error) => return Err(error), }; let (enum_declarations, constant_declarations) = declarations; @@ -401,16 +123,16 @@ fn model_record<'py>( declarations: (enum_declarations.unbind(), constant_declarations.unbind()), extract_implicit_alias_columns, }); - record.into_py_any(py) + Ok(Ok(record.into_py_any(py)?)) } -/// Parse in-memory SQL model contents into the payload model discovery returns for a file. +/// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. #[pyfunction] fn parse_model_contents( py: Python<'_>, request: ModelTextRequest, contents: String, -) -> PyResult { +) -> PyResult<(PyObject, String, PyObject)> { let options = ModelFileOptions { supported_keys: request.supported_keys, removed_keys: request.removed_keys, @@ -420,248 +142,24 @@ fn parse_model_contents( let parsed = py .compiler_detach(|| Ok(parse_model_text(&request.file_path, contents, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(model) => model_object(py, model), - Err(failure) => failure_object(py, failure), - } -} - -/// The declaration file facts (of one kind, if given) and named groups, or the stage failure. -#[pyfunction] -#[pyo3(signature = (tree, kind=None))] -fn discover_declaration_layout( - py: Python<'_>, - tree: &NativeProjectTree, - kind: Option<&str>, -) -> PyResult { - let kind: Option = declaration_kind(kind)?; - let layout: Result = py - .compiler_detach(|| Ok(declaration_layout(&tree.inner, kind))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match layout { - Ok(layout) => tuple_object( - py, - vec![ - layout_rows(py, &tree.inner, layout.file_facts, fact_row)?, - layout_rows(py, &tree.inner, layout.named_groups, group_row)?, - ], - ), - Err(failure) => stage_failure_object(py, &tree.inner, failure), - } -} - -fn test_block_object(py: Python<'_>, block: SqlTestBlock) -> PyResult { - tuple_object( - py, - vec![ - map_to_python(py, block.header_values)?, - object(py, block.sql_body)?, - object(py, block.body_span)?, - ], - ) -} - -fn test_file_object(py: Python<'_>, file: DiscoveredSqlTestFile) -> PyResult { - let blocks: Vec = file - .blocks - .into_iter() - .map(|block| test_block_object(py, block)) - .collect::>()?; - let failure: Option = file - .failure - .map(|failure| failure_object(py, failure)) - .transpose()?; - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, file.contents)?, - object(py, blocks)?, - object(py, failure)?, - ], - ) -} - -fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult { - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, file.contents)?, - map_to_python(py, file.header_values)?, - object(py, file.sql_body)?, - object(py, file.body_span)?, - ], - ) -} - -/// The root and options of one SQL test or scenario discovery request. -fn sql_test_inputs( - request: SqlTestDiscoveryRequest, -) -> PyResult<(ProjectRoot, SqlTestFileOptions)> { + let model: DiscoveredModelFile = parsed.map_err(|failure| failure_error(py, failure))?; + let path: Bound<'_, PyAny> = py + .import("pathlib")? + .getattr("Path")? + .call1((request.file_path.as_str(),))?; Ok(( - ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }, - SqlTestFileOptions { - test_keys: request.test_keys, - scenario_keys: request.scenario_keys, - }, + map_to_python(py, model.header_values)?, + model.query_sql, + source_locations(py, &path, &model.header_column_locations)? + .into_any() + .unbind(), )) } -/// Discover and split the SQL test files: one payload per file, or the failure payload. -#[pyfunction] -fn discover_sql_test_files( - py: Python<'_>, - request: SqlTestDiscoveryRequest, - tree: &NativeProjectTree, -) -> PyResult { - let (root, options) = sql_test_inputs(request)?; - let discovered = py - .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - files_object(py, &tree.inner, discovered, test_file_object) -} - -/// Discover and header-parse the scenario files: one payload per file, or the failure payload. -#[pyfunction] -fn discover_scenario_files( - py: Python<'_>, - request: SqlTestDiscoveryRequest, - tree: &NativeProjectTree, -) -> PyResult { - let (root, options) = sql_test_inputs(request)?; - let discovered = py - .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - files_object(py, &tree.inner, discovered, scenario_object) -} - -/// The options of one in-memory SQL test or scenario request. -fn text_options(request: &SqlTestTextRequest) -> PyResult { - Ok(SqlTestFileOptions { - test_keys: request.test_keys.clone(), - scenario_keys: request.scenario_keys.clone(), - }) -} - -/// Split in-memory SQL test contents into the payload test discovery returns for a file. -#[pyfunction] -fn parse_sql_test_contents( - py: Python<'_>, - request: SqlTestTextRequest, - contents: String, -) -> PyResult { - let options: SqlTestFileOptions = text_options(&request)?; - let parsed = py - .compiler_detach(|| Ok(parse_sql_test_text(&request.file_path, contents, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(file) => test_file_object(py, file), - Err(failure) => failure_object(py, failure), - } -} - -/// Header-parse in-memory SQL scenario contents into the payload scenario discovery returns. -#[pyfunction] -fn parse_scenario_contents( - py: Python<'_>, - request: SqlTestTextRequest, - contents: String, -) -> PyResult { - let options: SqlTestFileOptions = text_options(&request)?; - let parsed = py - .compiler_detach(|| Ok(parse_scenario_text(&request.file_path, contents, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(file) => scenario_object(py, file), - Err(failure) => failure_object(py, failure), - } -} - -/// `("ok", contents, value)`; a value Python cannot build fails like invalid YAML. -fn loaded_yaml_object( - py: Python<'_>, - (file_path, kind): (&str, FailureKind), - (contents, value): (String, sqlbuild_config::models::ConfigValue), -) -> PyResult { - match config_value_to_python(py, value) { - Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), - Err(error) => failure_object( - py, - DiscoveryFailure::new( - kind, - format!("{file_path} contains invalid YAML: {}", error.value(py)), - ), - ), - } -} - -/// Read and load YAML files: one payload per path, or the collection's failure payload. -#[pyfunction] -fn load_yaml_files( - py: Python<'_>, - request: YamlDiscoveryRequest, - relative_paths: Vec, - tree: &NativeProjectTree, -) -> PyResult { - let kind: FailureKind = yaml_failure_kind(&request.kind)?; - let root = ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }; - let loaded: Result, StageFailure> = py - .compiler_detach(|| Ok(load_yaml(&root, &tree.inner, &relative_paths, kind))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let outcomes: Vec = match loaded { - Ok(outcomes) => outcomes, - Err(failure) => return stage_failure_object(py, &tree.inner, failure), - }; - outcomes - .into_iter() - .zip(&relative_paths) - .map(|(outcome, relative_path)| match outcome { - YamlFileOutcome::Loaded { contents, value } => loaded_yaml_object( - py, - (&root.display_path(relative_path), kind), - (contents, value), - ), - YamlFileOutcome::Failed(failure) => failure_object(py, failure), - YamlFileOutcome::Unreadable(failure) => read_object(py, failure), - }) - .collect::>>()? - .into_py_any(py) -} - -/// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. -#[pyfunction] -fn load_yaml_document( - py: Python<'_>, - file_path: &str, - text: String, - kind: &str, -) -> PyResult { - let kind: FailureKind = yaml_failure_kind(kind)?; - match load_yaml_text(file_path, &text, kind) { - Ok(value) => loaded_yaml_object(py, (file_path, kind), (text, value)), - Err(failure) => failure_object(py, failure), - } -} - /// Whether native discovery reproduces the string semantics of this Python and Unicode version. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_document, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_test_contents, module)?)?; - module.add_function(wrap_pyfunction!(parse_scenario_contents, module)?)?; module.add_class::()?; module.add_class::()?; - module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; - module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs index 7960161131..82bb91c3cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -1,59 +1,74 @@ //! Cross-file validation of one discovery's inputs. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; use crate::bindings::_helpers::discovery::project_facts::{NodeRow, project_facts}; -use crate::bindings::_helpers::discovery::project_files::{ - PyObject, failure_object, object, read_object, tuple_object, -}; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error}; -const CSV_ERROR_TAG: &str = "csv"; -const CSV_DIALECT_TAG: &str = "csv_dialect"; -const UNREADABLE_TAG: &str = "unreadable"; +const CSV_MODULE: &str = "csv"; +const BUILTINS_MODULE: &str = "builtins"; +const PATHLIB_MODULE: &str = "pathlib"; +/// The exceptions a CSV dialect may raise, by class name. +const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; -/// `None` when the inputs are consistent, else the payload of the first error Python raises. +fn raised(result: PyResult>) -> PyErr { + match result { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Raise the first cross-file conflict across the discovered inputs, as Python raises it. #[pyfunction] -fn discovered_inputs_failure( +fn validate_discovered_project_inputs( py: Python<'_>, inputs: Bound<'_, PyAny>, nodes: Vec, -) -> PyResult> { +) -> Result<(), PyErr> { let facts: ProjectFacts = project_facts(py, &inputs, nodes)?; - let failure: ValidationFailure = match validate_project_inputs(&facts) { - Ok(()) => return Ok(None), - Err(failure) => failure, + let Err(failure) = validate_project_inputs(&facts) else { + return Ok(()); }; - let payload: PyObject = match failure { - ValidationFailure::Discovery(failure) => failure_object(py, failure)?, + Err(match failure { + ValidationFailure::Discovery(failure) => failure_error(py, failure), ValidationFailure::Unreadable { absolute_path, failure, - } => tuple_object( - py, - vec![ - object(py, UNREADABLE_TAG)?, - object(py, absolute_path)?, - read_object(py, failure)?, - ], - )?, - ValidationFailure::Csv(message) => { - tuple_object(py, vec![object(py, CSV_ERROR_TAG)?, object(py, message)?])? + } => match py + .import(PATHLIB_MODULE) + .and_then(|module| module.getattr("Path")) + .and_then(|path| path.call1((absolute_path,))) + { + Ok(file_path) => read_error(failure, &file_path), + Err(error) => error, + }, + ValidationFailure::Csv(message) => raised( + py.import(CSV_MODULE) + .and_then(|module| module.getattr("Error")) + .and_then(|class| class.call1((message,))), + ), + ValidationFailure::CsvDialect(error) => { + if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { + return Err(PyKeyError::new_err(error.class.to_string())); + } + raised( + py.import(BUILTINS_MODULE) + .and_then(|module| module.getattr(error.class.as_str())) + .and_then(|class| class.call1((error.message,))), + ) } - ValidationFailure::CsvDialect(error) => tuple_object( - py, - vec![ - object(py, CSV_DIALECT_TAG)?, - object(py, error.class)?, - object(py, error.message)?, - ], - )?, - }; - Ok(Some(payload)) + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discovered_inputs_failure, module)?) + module.add_function(wrap_pyfunction!( + validate_discovered_project_inputs, + module + )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs new file mode 100644 index 0000000000..06b454c4da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -0,0 +1,163 @@ +//! Native discovery failures as the Python exceptions discovery raises for them. + +use pyo3::exceptions::{PyKeyError, PyRuntimeError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::models::{DiscoveryFailure, ReadFailure, StageFailure}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::discovery::project_files::path_object; + +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const BUILTINS_MODULE: &str = "builtins"; +const OS_MODULE: &str = "os"; +const DECODE_ENCODING: &str = "utf-8"; + +/// The exception class of each native failure kind. +const FAILURE_CLASSES: [(&str, &str, &str); 12] = [ + ("model_sql", DISCOVERY_ERRORS_MODULE, "ModelSqlParseError"), + ( + "declaration", + DISCOVERY_ERRORS_MODULE, + "DeclarationParseError", + ), + ("sql_test", DISCOVERY_ERRORS_MODULE, "SqlTestParseError"), + ( + "sql_scenario", + DISCOVERY_ERRORS_MODULE, + "SqlScenarioParseError", + ), + ("schema", DISCOVERY_ERRORS_MODULE, "SchemaParseError"), + ("source", DISCOVERY_ERRORS_MODULE, "SourceParseError"), + ("project_path", DISCOVERY_ERRORS_MODULE, "ProjectPathError"), + ("sql_hook", DISCOVERY_ERRORS_MODULE, "SqlHookParseError"), + ("sql_audit", DISCOVERY_ERRORS_MODULE, "SqlAuditParseError"), + ( + "resource_identity", + IDENTITY_ERRORS_MODULE, + "ResourceIdentityError", + ), + ( + "conflict", + DISCOVERY_ERRORS_MODULE, + "DiscoveryConflictError", + ), + ("seed", DISCOVERY_ERRORS_MODULE, "SeedDiscoveryError"), +]; + +fn raised(result: PyResult>) -> PyErr { + match result { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `error_class(message, help=help)` for the failure kind `kind`. +pub(crate) fn discovery_error( + py: Python<'_>, + kind: &str, + message: String, + help: Option, +) -> PyErr { + let Some((_, module, class)) = FAILURE_CLASSES.iter().find(|(name, _, _)| *name == kind) else { + return PyKeyError::new_err(kind.to_owned()); + }; + raised((|| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + py.import(*module)? + .getattr(*class)? + .call((message,), Some(&keywords)) + })()) +} + +/// Python's `native_failure`: the discovery error a native failure describes. +pub(crate) fn failure_error(py: Python<'_>, failure: DiscoveryFailure) -> PyErr { + discovery_error(py, failure.kind.as_str(), failure.message, failure.help) +} + +/// Python's `native_read_error`: what `read_text` raises for a file native reading could not read. +pub(crate) fn read_error(failure: ReadFailure, file_path: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = file_path.py(); + raised((|| { + let builtins = py.import(BUILTINS_MODULE)?; + match failure { + ReadFailure::Decode { + bytes, + start, + end, + reason, + } => builtins.getattr("UnicodeDecodeError")?.call1(( + DECODE_ENCODING, + PyBytes::new(py, &bytes), + start, + end, + reason, + )), + ReadFailure::Io { + errno, + winerror, + message, + } => { + let os_error = builtins.getattr("OSError")?; + let path: String = file_path.str()?.to_string(); + match (winerror, errno) { + (Some(winerror), _) => os_error.call1((py.None(), message, path, winerror)), + (None, None) => os_error.call1((message,)), + (None, Some(errno)) => { + let reason = py.import(OS_MODULE)?.getattr("strerror")?.call1((errno,))?; + os_error.call1((errno, reason, path)) + } + } + } + } + })()) +} + +/// Python's `native_collection` failure: an unlistable directory's read error, else the bug. +pub(crate) fn stage_error( + tree: &ProjectTree, + project_dir: &Bound<'_, PyAny>, + failure: StageFailure, +) -> PyErr { + match failure { + StageFailure::Unlistable { + relative_path, + error, + } => match path_object(project_dir.py(), tree, relative_path) + .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) + { + Ok(directory) => read_error(error, &directory), + Err(error) => error, + }, + StageFailure::Internal(reason) => PyRuntimeError::new_err(reason), + } +} + +/// The error `read_error` builds for an operating-system read failure, for the platform oracle. +#[pyfunction] +#[pyo3(signature = (errno, winerror, message, *, file_path))] +fn _oracle_read_error<'py>( + errno: Option, + winerror: Option, + message: String, + file_path: &Bound<'py, PyAny>, +) -> Bound<'py, PyAny> { + let failure = ReadFailure::Io { + errno, + winerror, + message, + }; + read_error(failure, file_path) + .into_value(file_path.py()) + .into_bound(file_path.py()) + .into_any() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 3cc8e6b882..248a1c5234 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -3,15 +3,21 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ - declaration_files, globbing, header_declarations, key_checks, locations, project_files, - project_validation, sql_test_blocks, yaml_entries, + declaration_files, declaration_records, globbing, header_declarations, key_checks, + layout_groups, locations, model_records, project_files, project_validation, raised, + sql_test_records, yaml_entries, yaml_records, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project_files::register(module)?; declaration_files::register(module)?; + declaration_records::register(module)?; + layout_groups::register(module)?; + model_records::register(module)?; + sql_test_records::register(module)?; + yaml_records::register(module)?; + raised::register(module)?; locations::register(module)?; - sql_test_blocks::register(module)?; header_declarations::register(module)?; key_checks::register(module)?; globbing::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs new file mode 100644 index 0000000000..33d57d6a5a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs @@ -0,0 +1,156 @@ +//! The Python directory snapshot one discovery pass shares, and the native tree it memoises. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_discovery::models::ProjectRoot; +use sqlbuild_discovery::tree::models::ProjectTree; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::raised::discovery_error; +use crate::bindings::models::NativeProjectTree; + +const SNAPSHOT_MODULE: &str = "sqlbuild.compiler.discovery.classes.directory_snapshot"; +const SNAPSHOT_CLASS: &str = "DirectorySnapshot"; +const DISCOVERY_TYPES_MODULE: &str = "sqlbuild.compiler.discovery.types"; +const SNAPSHOT_ENTRY_CLASS: &str = "DirectorySnapshotEntry"; +const TREE_MEMO_KEY: &str = "native_project_tree"; +const DISPLAY_PROBE: &str = "_"; +const PROJECT_PATH_KIND: &str = "project_path"; +const SURROGATE_ESCAPE_BASE: u32 = 0xDC00; +const ESCAPED_BYTE_FIRST: u16 = 0xDC80; +const ESCAPED_BYTE_LAST: u16 = 0xDCFF; +const UTF16_ENCODING: &str = "utf-16-le"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// Run `body` inside `DirectorySnapshot.scope(project_dir=project_dir)`. +pub(crate) fn in_snapshot<'py, T>( + project_dir: &Bound<'py, PyAny>, + body: impl FnOnce() -> PyResult, +) -> PyResult { + let py: Python<'py> = project_dir.py(); + let scope: Bound<'py, PyAny> = call_with( + &imported(py, SNAPSHOT_MODULE, SNAPSHOT_CLASS)?.getattr("scope")?, + &[("project_dir", project_dir.clone())], + )?; + within(&scope, |_| body()) +} + +/// `DirectorySnapshot.current(project_dir=project_dir).memo`. +pub(crate) fn snapshot_memo<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + current_snapshot(project_dir)? + .getattr("memo")? + .downcast_into::() + .map_err(Into::into) +} + +fn current_snapshot<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(project_dir.py(), SNAPSHOT_MODULE, SNAPSHOT_CLASS)?.getattr("current")?, + &[("project_dir", project_dir.clone())], + ) +} + +/// `str(project_dir / "_")` without its probe: the text printed before a relative path. +pub(crate) fn display_prefix(project_dir: &Bound<'_, PyAny>) -> PyResult { + let probed: String = project_dir.div(DISPLAY_PROBE)?.str()?.to_str()?.to_owned(); + Ok(probed[..probed.len() - DISPLAY_PROBE.len()].to_owned()) +} + +/// The project root native discovery reads, named as Python prints it. +pub(crate) fn project_root(project_dir: &Bound<'_, PyAny>) -> PyResult { + Ok(ProjectRoot { + directory: PathBuf::from(project_dir.str()?.to_str()?), + display_prefix: display_prefix(project_dir)?, + }) +} + +/// The native listings shared by every native call of the pass, created once per snapshot. +pub(crate) fn project_tree<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + if let Some(tree) = memo.get_item(TREE_MEMO_KEY)? + && let Ok(tree) = tree.downcast_into::() + { + return Ok(tree); + } + let directory: Bound<'py, PyString> = project_dir.str()?; + let prefixed: Bound<'py, PyString> = project_dir.div(DISPLAY_PROBE)?.str()?; + for text in [&directory, &prefixed] { + if text.to_str().is_err() { + let mut shown: String = escaped(text)?; + if text.is(&prefixed) { + shown.truncate(shown.len() - DISPLAY_PROBE.len()); + } + return Err(discovery_error( + py, + PROJECT_PATH_KIND, + format!( + "Project directory {shown} is not valid UTF-8; move the project to a \ + directory whose path is valid UTF-8" + ), + None, + )); + } + } + let created: Bound<'py, NativeProjectTree> = Bound::new( + py, + NativeProjectTree { + inner: ProjectTree::new(std::path::Path::new(directory.to_str()?)), + exported: Mutex::new(HashSet::new()), + }, + )?; + memo.set_item(TREE_MEMO_KEY, &created)?; + Ok(created) +} + +/// Share the native walk's new listings with the pass's Python directory snapshot. +pub(crate) fn seed_listings( + project_dir: &Bound<'_, PyAny>, + tree: &Bound<'_, NativeProjectTree>, +) -> PyResult<()> { + let py: Python<'_> = project_dir.py(); + let entry_class: Bound<'_, PyAny> = imported(py, DISCOVERY_TYPES_MODULE, SNAPSHOT_ENTRY_CLASS)?; + let listings: Bound<'_, PyDict> = PyDict::new(py); + for (directory, entries) in tree.get().exported_listings(py)? { + let entries: Vec> = entries + .into_iter() + .map(|(name, is_dir, walkable)| entry_class.call1((name, is_dir, walkable))) + .collect::>()?; + listings.set_item( + project_dir.div(directory.bind(py))?, + PyTuple::new(py, entries)?, + )?; + } + current_snapshot(project_dir)?.call_method1("seed_listings", (listings,))?; + Ok(()) +} + +/// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. +pub(crate) fn escaped(text: &Bound<'_, PyString>) -> PyResult { + let encoded: Vec = text + .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? + .extract()?; + let units = encoded + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)); + Ok(char::decode_utf16(units) + .map(|decoded| match decoded { + Ok(character) => character.to_string(), + Err(error) => escaped_unit(error.unpaired_surrogate()), + }) + .collect()) +} + +fn escaped_unit(unit: u16) -> String { + if cfg!(unix) && (ESCAPED_BYTE_FIRST..=ESCAPED_BYTE_LAST).contains(&unit) { + return format!("\\x{:02x}", u32::from(unit) - SURROGATE_ESCAPE_BASE); + } + format!("\\u{unit:04x}") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs new file mode 100644 index 0000000000..a32dc24efb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs @@ -0,0 +1,19 @@ +//! Header key sets as the sorted lists native discovery options take. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyFrozenSetMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; + +/// `sorted(constants.)`: a header key set, sorted by code point as Python sorts it. +pub(crate) fn sorted_keys(py: Python<'_>, name: &str) -> PyResult> { + let keys: Bound<'_, PyFrozenSet> = imported(py, CONSTANTS_MODULE, name)?.downcast_into()?; + let mut keys: Vec = keys + .iter() + .map(|key| key.extract()) + .collect::>()?; + keys.sort(); + Ok(keys) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 6a128ec24b..5ef774ed56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -1,10 +1,8 @@ //! SQL `TEST(...)` blocks validated natively and built as discovery records. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::sql_tests::main::sql_test_block::sql_test_block; use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_names_failure; use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; @@ -17,7 +15,7 @@ const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; -type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); +pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); fn parse_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( @@ -101,8 +99,7 @@ fn block_record<'py>( } /// Each `(header, sql_body, span)` block of a SQL test file as a `DiscoveredSqlTestBlock`. -#[pyfunction] -fn sql_test_blocks<'py>( +pub(crate) fn sql_test_block_records<'py>( py: Python<'py>, file_path: &str, blocks: Vec>, @@ -120,19 +117,13 @@ fn sql_test_blocks<'py>( } /// `validate_sql_test_names` over each block's `(test_index, name)`. -#[pyfunction] -fn validate_sql_test_block_names( +pub(crate) fn validate_block_names( py: Python<'_>, file_path: &str, - blocks: Vec<(usize, Option)>, + blocks: &[(usize, Option)], ) -> Result<(), PyErr> { - match sql_test_names_failure(file_path, &blocks) { + match sql_test_names_failure(file_path, blocks) { None => Ok(()), Some(message) => Err(parse_error(py, message)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_blocks, module)?)?; - module.add_function(wrap_pyfunction!(validate_sql_test_block_names, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs new file mode 100644 index 0000000000..16002c4b6e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -0,0 +1,317 @@ +//! SQL unit-test and scenario files discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, StageFailure}; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; +use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; +use sqlbuild_discovery::sql_tests::models::{ + DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, +}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::_helpers::discovery::sql_test_blocks::{ + sql_test_block_records, validate_block_names, +}; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::models::NativeProjectTree; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const PATHLIB_MODULE: &str = "pathlib"; +const SCENARIO_KIND: &str = "sql_scenario"; +const DESCRIPTION_KEY: &str = "description"; +const TAGS_KEY: &str = "tags"; + +/// A native discoverer of one kind of SQL test file. +type Discoverer = fn( + &sqlbuild_discovery::models::ProjectRoot, + &ProjectTree, + &SqlTestFileOptions, +) -> Result>, StageFailure>; + +/// Build one parsed file's record from its `(file_path, relative_path)`. +type RecordBuilder<'py, T> = + fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; + +fn options(py: Python<'_>) -> PyResult { + Ok(SqlTestFileOptions { + test_keys: sorted_keys(py, "SQL_TEST_HEADER_KEYS")?, + scenario_keys: sorted_keys(py, "SQL_SCENARIO_HEADER_KEYS")?, + }) +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +/// `Path(constants.)`. +fn ownership_root<'py>(py: Python<'py>, name: &str) -> PyResult> { + path_class(py)?.call1((imported(py, CONSTANTS_MODULE, name)?,)) +} + +/// Discover the files natively in the pass's snapshot: each relative path text and outcome. +fn discovered_files<'py, T: Send>( + project_dir: &Bound<'py, PyAny>, + discover: Discoverer, +) -> PyResult<(Bound<'py, NativeProjectTree>, Vec>)> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root = project_root(project_dir)?; + let options: SqlTestFileOptions = options(py)?; + let discovered: Result>, StageFailure> = { + let inner = &tree.get().inner; + py.detach(|| discover(&root, inner, &options)) + }; + let files = + discovered.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + seed_listings(project_dir, &tree)?; + Ok((tree, files)) + }) +} + +/// Each block as a `DiscoveredSqlTestBlock`, then the split failure, then the name check. +fn test_blocks<'py>( + file_path: &Bound<'py, PyAny>, + blocks: Vec, + failure: Option, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let mut native = Vec::with_capacity(blocks.len()); + for block in blocks { + native.push(( + map_to_python(py, block.header_values)? + .into_bound(py) + .downcast_into::()?, + block.sql_body, + block.body_span.into_bound_py_any(py)?, + )); + } + let records: Vec> = sql_test_block_records(py, &path, native)?; + if let Some(failure) = failure { + return Err(failure_error(py, failure)); + } + let names: Vec<(usize, Option)> = records + .iter() + .map(|block| { + Ok(( + block.getattr("test_index")?.extract()?, + block.getattr("name")?.extract()?, + )) + }) + .collect::>()?; + validate_block_names(py, &path, &names)?; + PyTuple::new(py, records) +} + +fn test_file<'py>( + paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + file: DiscoveredSqlTestFile, +) -> PyResult> { + let (file_path, relative_path) = paths; + let py: Python<'py> = file_path.py(); + call_with( + &imported(py, MODELS_MODULE, "DiscoveredSqlTestFile")?, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", file.contents.into_bound_py_any(py)?), + ( + "blocks", + test_blocks(file_path, file.blocks, file.failure)?.into_any(), + ), + ( + "ownership_root", + ownership_root(py, "SQL_TESTS_OWNERSHIP_ROOT")?, + ), + ], + ) +} + +/// Python's `build_sql_scenario_file`: validate the header and build the record. +fn scenario_file<'py>( + paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + file: DiscoveredScenarioFile, +) -> PyResult> { + let (file_path, relative_path) = paths; + let py: Python<'py> = file_path.py(); + let header_values: Bound<'py, PyDict> = map_to_python(py, file.header_values)? + .into_bound(py) + .downcast_into::()?; + let shown: Bound<'py, PyString> = file_path.str()?; + let failure = |message: String| Err(discovery_error(py, SCENARIO_KIND, message, None)); + if let Some(description) = header_values.get_item(DESCRIPTION_KEY)? + && !description.is_instance_of::() + { + return failure(format!( + "SCENARIO() description in '{shown}' must be a string" + )); + } + if let Some(tags) = header_values.get_item(TAGS_KEY)? { + let tags_are_strings: bool = match tags.downcast::() { + Ok(tags) => tags.iter().all(|tag| tag.is_instance_of::()), + Err(_) => false, + }; + if !tags_are_strings { + return failure(format!( + "SCENARIO() tags in '{shown}' must be a list of strings" + )); + } + } + if file.sql_body.is_empty() { + return failure(format!( + "SQL scenario '{shown}' must define SQL after SCENARIO(...)" + )); + } + call_with( + &imported(py, MODELS_MODULE, "DiscoveredSqlScenarioFile")?, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", file.contents.into_bound_py_any(py)?), + ("header_values", header_values.into_any()), + ("sql_body", file.sql_body.into_bound_py_any(py)?), + ("sql_body_span", file.body_span.into_bound_py_any(py)?), + ("name", file_path.getattr("stem")?), + ( + "ownership_root", + ownership_root(py, "SQL_SCENARIOS_OWNERSHIP_ROOT")?, + ), + ], + ) +} + +/// Build each file, raising or reporting each failing one. +fn records<'py, T>( + faults: &FileFaults<'py>, + files: Vec<(Bound<'py, PyAny>, FileOutcome)>, + build: RecordBuilder<'py, T>, +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let mut built: Vec> = Vec::with_capacity(files.len()); + for (relative_path, outcome) in files { + let file_path: Bound<'py, PyAny> = faults.project_dir.div(&relative_path)?; + let record: PyResult> = match outcome { + FileOutcome::Parsed(file) => build((&file_path, &relative_path), file), + FileOutcome::Failed(failure) => Err(failure_error(py, failure)), + FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), + }; + built.extend(faults.kept(&relative_path, record)?); + } + record_tuple(py, built) +} + +/// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] +pub(crate) fn discover_test_files<'py>( + project_dir: &Bound<'py, PyAny>, + selected_paths: Option>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_tests)?; + let tree: &ProjectTree = &tree.get().inner; + let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); + for file in files { + let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); + if let Some(selected) = &selected_paths + && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? + { + continue; + } + kept.push((path_class(py)?.call1((relative,))?, file.outcome)); + } + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + test_file, + ) +} + +/// Discover scenario files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_scenario_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_scenarios)?; + let tree: &ProjectTree = &tree.get().inner; + let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files + .into_iter() + .map(|file| { + Ok(( + path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, + file.outcome, + )) + }) + .collect::>()?; + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + scenario_file, + ) +} + +/// Parse in-memory SQL test contents into ordered blocks, as test discovery does. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path))] +pub(crate) fn parse_sql_test_file<'py>( + contents: String, + file_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options(py)?; + let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); + let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; + test_blocks(file_path, file.blocks, file.failure) +} + +/// Parse in-memory scenario contents into the record scenario discovery builds for a file. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path, relative_path))] +pub(crate) fn parse_sql_scenario_file<'py>( + contents: String, + file_path: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options(py)?; + let parsed = py.detach(|| parse_scenario_text(&path, contents, &options)); + let file: DiscoveredScenarioFile = parsed.map_err(|failure| failure_error(py, failure))?; + scenario_file((file_path, relative_path), file) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 8418ddb2e6..3d3638e69d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -27,7 +27,7 @@ const SOURCE_PARSE_ERROR: &str = "SourceParseError"; /// The Python classes parsed YAML entries become. #[derive(FromPyObject)] #[pyo3(from_item_all)] -struct YamlClasses<'py> { +pub(crate) struct YamlClasses<'py> { contracts: ContractClasses<'py>, schema_seed_entry: Bound<'py, PyAny>, seed_csv_settings: Bound<'py, PyAny>, @@ -157,7 +157,7 @@ fn seed_entry<'py>( /// The seed entries of one loaded schema YAML document, or the parse error Python raises. #[pyfunction] -fn parse_schema_document<'py>( +pub(crate) fn parse_schema_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, @@ -339,7 +339,7 @@ fn source_entry<'py>( /// The source entries of one loaded sources YAML document, or the parse error Python raises. #[pyfunction] -fn parse_sources_document<'py>( +pub(crate) fn parse_sources_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs new file mode 100644 index 0000000000..cc729fb70c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -0,0 +1,382 @@ +//! Source and schema YAML files discovered natively and built as Python discovery records. + +use pyo3::exceptions::PyUnicodeEncodeError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyList, PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind, ProjectRoot, StageFailure}; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; +use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; +use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; +use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; +use std::path::PathBuf; + +use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; +use crate::bindings::_helpers::discovery::snapshot::{ + escaped, in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::yaml_entries::{ + YamlClasses, parse_schema_document, parse_sources_document, +}; +use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +/// The most YAML bytes one native load reads before its files are built and released. +const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; +const YAML_CLASSES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.yaml_classes"; +const PATHLIB_MODULE: &str = "pathlib"; +const PROJECT_PATH_KIND: &str = "project_path"; +const SCHEMA_KIND: &str = "schema"; + +/// The YAML files one discovery reads, and the kind their failures raise as. +#[derive(Clone, Copy)] +enum YamlKind { + Source, + Schema, +} + +impl YamlKind { + fn failure_kind(self) -> FailureKind { + match self { + Self::Source => FailureKind::Source, + Self::Schema => FailureKind::Schema, + } + } +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +/// `(contents, loaded)` of a loaded file; a value Python cannot build fails like invalid YAML. +fn loaded<'py>( + py: Python<'py>, + (display_path, kind): (&str, FailureKind), + (contents, value): (String, ConfigValue), +) -> PyResult<(String, Bound<'py, PyAny>)> { + match config_value_to_python(py, value) { + Ok(loaded) => Ok((contents, loaded.into_bound(py))), + Err(error) => Err(failure_error( + py, + DiscoveryFailure::new( + kind, + format!("{display_path} contains invalid YAML: {}", error.value(py)), + ), + )), + } +} + +/// One file's `(contents, loaded)`, or the error Python raises for it. +fn outcome_loaded<'py>( + root: &ProjectRoot, + (relative_path, kind): (&str, FailureKind), + outcome: YamlFileOutcome, + file_path: &Bound<'py, PyAny>, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let py: Python<'py> = file_path.py(); + match outcome { + YamlFileOutcome::Loaded { contents, value } => loaded( + py, + (&root.display_path(relative_path), kind), + (contents, value), + ), + YamlFileOutcome::Failed(failure) => Err(failure_error(py, failure)), + YamlFileOutcome::Unreadable(failure) => Err(read_error(failure, file_path)), + } +} + +/// The pass's tree, or a private one for an undecodable project directory (fails on first read). +fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + match project_dir.str()?.to_str() { + Ok(_) => Ok(ListingTree::Shared(project_tree(project_dir)?)), + Err(error) if error.is_instance_of::(project_dir.py()) => { + let directory: PathBuf = project_dir.extract()?; + Ok(ListingTree::Private(ProjectTree::new(&directory))) + } + Err(error) => Err(error), + } +} + +enum ListingTree<'py> { + Shared(Bound<'py, NativeProjectTree>), + Private(ProjectTree), +} + +impl ListingTree<'_> { + fn tree(&self) -> &ProjectTree { + match self { + Self::Shared(tree) => &tree.get().inner, + Self::Private(tree) => tree, + } + } +} + +/// Split the files into batches bounded by size, as Python's `_batches` does. +fn batches(tree: &ProjectTree, paths: Vec, limit: u64) -> Vec> { + let mut batches: Vec> = Vec::new(); + let mut batch: Vec = Vec::new(); + let mut batch_bytes: u64 = 0; + for path in paths { + let size: u64 = + std::fs::metadata(tree.absolute(&path)).map_or(0, |metadata| metadata.len()); + if !batch.is_empty() && batch_bytes + size > limit { + batches.push(std::mem::take(&mut batch)); + batch_bytes = 0; + } + batch.push(path); + batch_bytes += size; + } + if !batch.is_empty() { + batches.push(batch); + } + batches +} + +/// Pass each file's paths and `(contents, loaded)` or error to `on_file`, in size-bounded batches. +fn load_files<'py>( + project_dir: &Bound<'py, PyAny>, + (paths, kind): (Vec, YamlKind), + mut on_file: impl FnMut( + (Bound<'py, PyAny>, Bound<'py, PyAny>), + PyResult<(String, Bound<'py, PyAny>)>, + ) -> PyResult<()>, +) -> PyResult<()> { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root: ProjectRoot = project_root(project_dir)?; + for batch in batches(&tree.get().inner, paths, YAML_BATCH_BYTES) { + let named: Vec = batch + .iter() + .filter(|path| !tree.get().inner.is_undecodable(path)) + .cloned() + .collect(); + let outcomes: Result, StageFailure> = { + let inner = &tree.get().inner; + let failure_kind: FailureKind = kind.failure_kind(); + py.detach(|| load_yaml(&root, inner, &named, failure_kind)) + }; + let mut outcomes = outcomes + .map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))? + .into_iter(); + seed_listings(project_dir, &tree)?; + for path in batch { + let relative: Bound<'py, PyAny> = + path_object(py, &tree.get().inner, path.clone())?.into_bound(py); + let relative_path: Bound<'py, PyAny> = path_class(py)?.call1((&relative,))?; + let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; + let built: PyResult<(String, Bound<'py, PyAny>)> = + if tree.get().inner.is_undecodable(&path) { + Err(unnamed_error(project_dir, &relative)?) + } else { + match outcomes.next() { + Some(outcome) => { + outcome_loaded(&root, (&path, kind.failure_kind()), outcome, &file_path) + } + None => Err(pyo3::exceptions::PyRuntimeError::new_err( + "native YAML loading returned too few files", + )), + } + }; + on_file((relative_path, file_path), built)?; + } + } + Ok(()) +} + +/// The `ProjectPathError` of a file whose name native discovery cannot read. +fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> PyResult { + let shown: String = escaped(&project_dir.div(relative)?.str()?)?; + Ok(discovery_error( + project_dir.py(), + PROJECT_PATH_KIND, + format!("Project path {shown} is not valid UTF-8; rename it so SQLBuild can read it"), + None, + )) +} + +fn yaml_classes(py: Python<'_>) -> PyResult> { + imported(py, YAML_CLASSES_MODULE, "yaml_entry_classes")? + .call0()? + .extract() +} + +/// Discover source YAML files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_source_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = source_yaml_paths(listing.tree()) + .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault, + }; + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSourceFile")?; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Source), + |(relative_path, file_path), built| { + let record = built.and_then(|(contents, loaded)| { + let shown: String = file_path.str()?.to_str()?.to_owned(); + call_with( + &class, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", contents.into_bound_py_any(py)?), + ( + "source_entries", + parse_sources_document(py, loaded, &shown, yaml_classes(py)?)? + .into_any(), + ), + ], + ) + }); + records.extend(faults.kept(&relative_path, record)?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) +} + +/// Discover model schema.yml and seed declaration files natively. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = match schema_yaml_paths(listing.tree()) { + Ok(paths) => paths, + Err(YamlPathFailure::Stage(failure)) => { + return Err(stage_error(listing.tree(), project_dir, failure)); + } + Err(YamlPathFailure::YamlSeedDeclaration(path)) => { + let relative: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; + return Err(discovery_error( + py, + SCHEMA_KIND, + format!( + "Seed declaration file {} must use .yml; .yaml is not supported", + relative.str()? + ), + None, + )); + } + }; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSchemaFile")?; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Schema), + |(relative_path, file_path), built| { + let (contents, loaded) = built?; + let shown: String = file_path.str()?.to_str()?.to_owned(); + records.push(call_with( + &class, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("contents", contents.into_bound_py_any(py)?), + ("model_entries", PyTuple::empty(py).into_any()), + ( + "seed_entries", + parse_schema_document(py, loaded, &shown, yaml_classes(py)?)? + .into_any(), + ), + ], + )?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) +} + +fn yaml_kind(kind: &str) -> PyResult { + match kind { + "source" => Ok(FailureKind::Source), + "schema" => Ok(FailureKind::Schema), + other => Err(pyo3::exceptions::PyValueError::new_err(format!( + "unknown YAML file kind {other:?}" + ))), + } +} + +/// Read and load YAML files: each file's `(contents, loaded)`, or the error it raises. +#[pyfunction] +fn load_yaml_files<'py>( + py: Python<'py>, + request: YamlDiscoveryRequest, + relative_paths: Vec, + tree: &NativeProjectTree, +) -> PyResult> { + let kind: FailureKind = yaml_kind(&request.kind)?; + let root = ProjectRoot { + directory: PathBuf::from(&request.project_dir), + display_prefix: request.display_prefix, + }; + let project_dir: Bound<'py, PyAny> = path_class(py)?.call1((request.project_dir.as_str(),))?; + let outcomes: Vec = py + .detach(|| load_yaml(&root, &tree.inner, &relative_paths, kind)) + .map_err(|failure| stage_error(&tree.inner, &project_dir, failure))?; + let mut rows: Vec> = Vec::with_capacity(outcomes.len()); + for (outcome, relative_path) in outcomes.into_iter().zip(&relative_paths) { + let file_path: Bound<'py, PyAny> = project_dir.div(relative_path)?; + rows.push( + match outcome_loaded(&root, (relative_path, kind), outcome, &file_path) { + Ok(loaded) => loaded.into_bound_py_any(py)?, + Err(error) => error.into_value(py).into_any().into_bound(py), + }, + ); + } + PyList::new(py, rows) +} + +/// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. +#[pyfunction] +fn load_yaml_document<'py>( + py: Python<'py>, + file_path: &str, + text: String, + kind: &str, +) -> PyResult> { + let kind: FailureKind = yaml_kind(kind)?; + let value: ConfigValue = + load_yaml_text(file_path, &text, kind).map_err(|failure| failure_error(py, failure))?; + Ok(loaded(py, (file_path, kind), (text, value))?.1) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_document, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs new file mode 100644 index 0000000000..895fbdc8d4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -0,0 +1,258 @@ +//! Latest fingerprint rows decoded into `Fingerprint` models and indexed into a `FingerprintSet`. + +use std::ffi::CString; + +use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyDateTime, PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::errors::StorageDecodeError; +use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage; +use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; +use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; +use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const DATETIME_MODULE: &str = "datetime"; +const BINASCII_MODULE: &str = "binascii"; +const BASE64_MODULE: &str = "base64"; +const TIMESTAMP_COLUMN: usize = 11; + +/// `FingerprintInputError(message)`. +pub(crate) fn fingerprint_input_error(py: Python<'_>, message: &str) -> PyErr { + match py + .import(EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("FingerprintInputError")) + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The `FingerprintSet` of fetched latest rows, a model winning a node-name collision. +#[pyfunction] +fn fingerprint_set_from_rows<'py>( + py: Python<'py>, + schema: &str, + qualified_name: &str, + rows: &Bound<'py, PyAny>, +) -> PyResult> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let fingerprint_class: Bound<'py, PyAny> = models.getattr("Fingerprint")?; + let from_iso: Bound<'py, PyAny> = py + .import(DATETIME_MODULE)? + .getattr("datetime")? + .getattr("fromisoformat")?; + let mut fingerprints: Vec> = Vec::new(); + for row in rows.try_iter()? { + fingerprints.push(row_fingerprint( + &fingerprint_class, + &from_iso, + &row?, + qualified_name, + )?); + } + indexed_set(&models, schema, &fingerprints) +} + +/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. +#[pyfunction] +fn select_latest_fingerprints<'py>( + py: Python<'py>, + fingerprint_set: &Bound<'py, PyAny>, + node_names: Vec, + filtered_node_types: Vec, +) -> PyResult> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let schema: String = fingerprint_set.getattr("schema")?.extract()?; + if node_names.is_empty() && filtered_node_types.is_empty() { + return indexed_set(&models, &schema, &[]); + } + let by_identity: Bound<'py, PyAny> = fingerprint_set.getattr("fingerprints_by_identity")?; + let latest: Vec> = if by_identity.is_truthy()? { + by_identity + .call_method0("values")? + .try_iter()? + .collect::>()? + } else { + Vec::new() + }; + let identities: Vec<(String, String)> = + latest.iter().map(identity_of).collect::>()?; + let positions: Vec = + selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); + let selected: Vec> = positions + .into_iter() + .map(|position| latest[position].clone()) + .collect(); + indexed_set(&models, &schema, &selected) +} + +fn indexed_set<'py>( + models: &Bound<'py, PyModule>, + schema: &str, + fingerprints: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = models.py(); + let identities: Vec<(String, String)> = fingerprints + .iter() + .map(identity_of) + .collect::>()?; + let index: LatestIndex = latest_index(&borrowed(&identities)); + let by_name: Bound<'py, PyDict> = PyDict::new(py); + for position in index.by_name { + by_name.set_item(&identities[position].1, &fingerprints[position])?; + } + let by_identity: Bound<'py, PyDict> = PyDict::new(py); + for position in index.by_identity { + by_identity.set_item(identities[position].clone(), &fingerprints[position])?; + } + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("schema", schema)?; + kwargs.set_item("fingerprints", by_name)?; + kwargs.set_item("fingerprints_by_identity", by_identity)?; + models.getattr("FingerprintSet")?.call((), Some(&kwargs)) +} + +fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + fingerprint.getattr("node_type")?.extract()?, + fingerprint.getattr("node_name")?.extract()?, + )) +} + +fn borrowed(identities: &[(String, String)]) -> Vec> { + identities + .iter() + .map(|(node_type, node_name)| FingerprintIdentity { + node_type, + node_name, + }) + .collect() +} + +fn row_fingerprint<'py>( + fingerprint_class: &Bound<'py, PyAny>, + from_iso: &Bound<'py, PyAny>, + row: &Bound<'py, PyAny>, + qualified_name: &str, +) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; + let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { + raw_ts + } else { + from_iso.call1((raw_ts.str()?,))? + }; + let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; + let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; + let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; + let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; + let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; + let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; + let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; + let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; + let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("node_type", node_type)?; + kwargs.set_item("node_name", node_name)?; + kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; + kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; + kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; + kwargs.set_item("run_id", row.get_item(5)?.str()?)?; + kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; + kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; + kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; + kwargs.set_item("definition", definition)?; + kwargs.set_item("metadata_json", metadata_json)?; + kwargs.set_item("ts", ts)?; + fingerprint_class.call((), Some(&kwargs)) +} + +fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + if value.is_none() { + return Ok(None); + } + value.str().map(Some) +} + +/// Base64 storage as text, raising Python's errors for non-ASCII or invalid storage. +fn decoded( + storage: &Bound<'_, PyString>, + node_name: &Bound<'_, PyString>, + qualified_name: &str, +) -> PyResult { + let py: Python<'_> = storage.py(); + let text: String = storage.extract()?; + if !text.is_ascii() { + let _ = storage.call_method1("encode", ("ascii",))?; + return Err(PyValueError::new_err( + "non-ASCII fingerprint storage encoded as ASCII", + )); + } + let failure: StorageDecodeError = match decode_storage(&text) { + Ok(decoded) => return Ok(decoded), + Err(failure) => failure, + }; + let cause: PyErr = decode_error(storage, failure)?; + let name: String = node_name.extract()?; + let raised: PyErr = fingerprint_input_error( + py, + &format!( + "Invalid fingerprint definition storage for '{name}' in {qualified_name}: expected \ + base64-encoded UTF-8. This can happen after upgrading from an older sqlbuild \ + version; delete or rebuild {qualified_name} to regenerate fingerprints." + ), + ); + raised.value(py).setattr("__context__", cause.value(py))?; + raised.set_cause(py, Some(cause)); + Err(raised) +} + +/// The interpreter's own decoder error, whose message varies by Python version, else ours. +fn decode_error(storage: &Bound<'_, PyString>, failure: StorageDecodeError) -> PyResult { + let py: Python<'_> = storage.py(); + match failure { + StorageDecodeError::Base64(message) => { + let kwargs: Bound<'_, PyDict> = PyDict::new(py); + kwargs.set_item("validate", true)?; + let decoded = py.import(BASE64_MODULE)?.getattr("b64decode")?.call( + (storage.call_method1("encode", ("ascii",))?,), + Some(&kwargs), + ); + match decoded { + Err(error) => Ok(error), + Ok(_) => Ok(PyErr::from_value( + py.import(BINASCII_MODULE)? + .getattr("Error")? + .call1((message,))?, + )), + } + } + StorageDecodeError::Utf8 { + bytes, + start, + end, + reason, + } => { + if let Err(error) = PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",)) { + return Ok(error); + } + let reason: CString = + CString::new(reason).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(PyErr::from_value( + PyUnicodeDecodeError::new(py, c"utf-8", &bytes, start..end, &reason)?.into_any(), + )) + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fingerprint_set_from_rows, module)?)?; + module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs new file mode 100644 index 0000000000..96da41548e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod latest_rows; +pub(crate) mod registration; +pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs new file mode 100644 index 0000000000..318ebc2d78 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs @@ -0,0 +1,10 @@ +//! Register the native fingerprint storage functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::fingerprints::{latest_rows, storage_sql}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + storage_sql::register(module)?; + latest_rows::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs new file mode 100644 index 0000000000..34ca1125c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs @@ -0,0 +1,116 @@ +//! Fingerprint table SQL: DDL, the latest-row read, its node filter, and batched inserts. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyTuple; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::errors::FingerprintStorageError; +use sqlbuild_analysis::fingerprint_storage::main::create_table_sql::create_table_sql; +use sqlbuild_analysis::fingerprint_storage::main::insert_batch_lengths::insert_batch_lengths; +use sqlbuild_analysis::fingerprint_storage::main::insert_sql::insert_sql; +use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; +use sqlbuild_analysis::fingerprint_storage::main::relevant_read_sql::relevant_read_sql; +use sqlbuild_analysis::fingerprint_storage::models::{RelevantRead, StoredFingerprint}; + +use crate::bindings::_helpers::fingerprints::latest_rows::fingerprint_input_error; + +/// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, given `(string_type, timestamp_type)`. +#[pyfunction] +fn fingerprint_create_table_sql(qualified_name: &str, column_types: (String, String)) -> String { + create_table_sql(qualified_name, &column_types.0, &column_types.1) +} + +/// The windowed latest-row read of the fingerprint table. +#[pyfunction] +fn fingerprint_read_latest_sql(qualified_name: &str) -> String { + read_latest_sql(qualified_name) +} + +/// The latest-row read narrowed to the relevant nodes, or `None` when nothing can match. +#[pyfunction] +#[pyo3(signature = (read_sql, node_names, filtered_node_types))] +fn fingerprint_relevant_read_sql( + read_sql: &str, + node_names: Option>, + filtered_node_types: Vec, +) -> Option { + match relevant_read_sql(read_sql, node_names.as_deref(), &filtered_node_types) { + RelevantRead::Sql(sql) => Some(sql), + RelevantRead::Empty => None, + } +} + +/// One `INSERT` of a single fingerprint. +#[pyfunction] +fn fingerprint_insert_sql( + py: Python<'_>, + qualified_name: &str, + fingerprint: &Bound<'_, PyAny>, +) -> PyResult { + insert_sql(qualified_name, &[stored_fingerprint(fingerprint)?]) + .map_err(|error| storage_error(py, error)) +} + +/// `(batch, insert_sql)` per bounded batch, given `(batch_size, batch_max_bytes)`. +#[pyfunction] +fn fingerprint_insert_batches<'py>( + py: Python<'py>, + qualified_name: &str, + fingerprints: Vec>, + limits: (i64, i64), +) -> PyResult, String)>> { + let stored: Vec = fingerprints + .iter() + .map(stored_fingerprint) + .collect::>()?; + let lengths: Vec = insert_batch_lengths(qualified_name, &stored, limits.0, limits.1) + .map_err(|error| storage_error(py, error))?; + let mut batches: Vec<(Bound<'py, PyTuple>, String)> = Vec::with_capacity(lengths.len()); + let mut start: usize = 0; + for length in lengths { + let end: usize = start + length; + let sql: String = insert_sql(qualified_name, &stored[start..end]) + .map_err(|error| storage_error(py, error))?; + batches.push((PyTuple::new(py, &fingerprints[start..end])?, sql)); + start = end; + } + Ok(batches) +} + +fn stored_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult { + Ok(StoredFingerprint { + node_type: fingerprint.getattr("node_type")?.extract()?, + node_name: fingerprint.getattr("node_name")?.extract()?, + target_database: fingerprint.getattr("target_database")?.extract()?, + target_schema: fingerprint.getattr("target_schema")?.extract()?, + target_name: fingerprint.getattr("target_name")?.extract()?, + run_id: formatted(fingerprint, "run_id")?, + definition_hash: formatted(fingerprint, "definition_hash")?, + version_hash: formatted(fingerprint, "version_hash")?, + schema_fingerprint: formatted(fingerprint, "schema_fingerprint")?, + definition: fingerprint.getattr("definition")?.extract()?, + metadata_json: fingerprint.getattr("metadata_json")?.extract()?, + ts: fingerprint + .getattr("ts")? + .call_method0("isoformat")? + .extract()?, + }) +} + +fn formatted(fingerprint: &Bound<'_, PyAny>, name: &str) -> PyResult { + fingerprint.getattr(name)?.str()?.extract() +} + +fn storage_error(py: Python<'_>, error: FingerprintStorageError) -> PyErr { + fingerprint_input_error(py, &error.to_string()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fingerprint_create_table_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_read_latest_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_relevant_read_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_insert_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_insert_batches, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index b3783d0db2..ee70abfb5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -68,6 +68,11 @@ fn failure(error: SelectorError) -> FailureRow { } impl NativeProjectGraph { + /// The graph this object wraps. + pub(crate) fn graph(&self) -> &ProjectGraph { + &self.graph + } + /// Wrap a graph another native stage built, such as project assembly. pub(crate) fn new(graph: ProjectGraph) -> Self { Self { graph } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs new file mode 100644 index 0000000000..53c7ff78cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -0,0 +1,559 @@ +//! `ProjectColumnLineage`: the project column lineage graph with lazily materialised views. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyType}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; +use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; + +use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; + +/// One compact model's facts and root-star flag. +struct CompactModel { + columns: Py, + has_star: bool, +} + +/// The Python objects one edge record materialises from. +enum EdgeParts { + /// An edge given to the eager constructor, returned as is. + Given(Py), + /// Fact values: source resource type, transform kind and confidence objects. + Facts { + resource_type: Py, + transform: Py, + confidence: Py, + }, +} + +/// Indexed edge records, materialised edges shared between equal records. +struct EdgeRecords { + keys: Vec, + parts: Vec, + index: EdgeIndex, + materialised: Vec>>, +} + +#[pyclass(module = "sqlbuild._native", name = "ProjectColumnLineage")] +pub(crate) struct ProjectColumnLineage { + models: Py, + compact: HashMap, + model_order: Vec, + edges: Option>, + records: Option, + edge_counts: HashMap, +} + +#[pymethods] +impl ProjectColumnLineage { + #[new] + #[pyo3(signature = (*, models, edges))] + fn new(models: &Bound<'_, PyAny>, edges: &Bound<'_, PyAny>) -> PyResult { + let mut keys: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); + for edge in edges.try_iter()? { + let edge: Bound<'_, PyAny> = edge?; + keys.push(EdgeKey { + source_name: edge + .getattr("source")? + .getattr("resource_name")? + .extract()?, + target_model: edge + .getattr("target")? + .getattr("resource_name")? + .extract()?, + ..empty_key() + }); + parts.push(EdgeParts::Given(edge.unbind())); + } + Ok(Self { + model_order: models + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + models: models.clone().unbind(), + compact: HashMap::new(), + edges: Some(edges.clone().unbind()), + records: Some(EdgeRecords::new(keys, parts)), + edge_counts: HashMap::new(), + }) + } + + /// Build indexed project lineage without expanding compact analysis facts. + #[classmethod] + #[pyo3(signature = (*, models, compact_models, model_order))] + fn from_fast_facts( + cls: &Bound<'_, PyType>, + models: &Bound<'_, PyAny>, + compact_models: &Bound<'_, PyDict>, + model_order: Vec, + ) -> PyResult { + let classes: LineageClasses<'_> = LineageClasses::new(cls.py())?; + let mut compact: HashMap = HashMap::new(); + for (name, entry) in compact_models.iter() { + let (columns, has_star): (Bound<'_, PyAny>, bool) = entry.extract()?; + let _ = compact.insert( + name.extract()?, + CompactModel { + columns: columns.unbind(), + has_star, + }, + ); + } + let mut edge_counts: HashMap = HashMap::new(); + for name in &model_order { + if let Some(model) = compact.get(name) { + let count: usize = fact_edge_count(&classes, model.columns.bind(cls.py()))?; + let _ = edge_counts.insert(name.clone(), count); + } + } + for item in models.call_method0("items")?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let _ = edge_counts.insert(name, upstream_count(&model.getattr("columns")?)?); + } + Ok(Self { + models: models.clone().unbind(), + compact, + model_order, + edges: None, + records: None, + edge_counts, + }) + } + + /// Return model lineage, expanding compact model facts on first access. + #[getter] + fn models(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !slf.compact.is_empty() { + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + let models: Bound<'_, PyAny> = slf.models.bind(py).clone(); + for name in &slf.model_order { + let Some(compact) = slf.compact.get(name) else { + continue; + }; + if models.contains(name)? { + continue; + } + let lineage: Bound<'_, PyAny> = + model_from_facts(&classes, name, compact.columns.bind(py), compact.has_star)?; + models.set_item(name, lineage)?; + } + slf.compact.clear(); + } + Ok(slf.models.clone_ref(py)) + } + + /// Return all graph edges, expanding compact edge records on first access. + #[getter] + fn edges(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if let Some(edges) = &slf.edges { + return Ok(edges.clone_ref(py)); + } + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + slf.ensure_records(&classes)?; + let count: usize = slf.records.as_ref().map_or(0, |records| records.keys.len()); + let edges: Vec> = slf.materialised(&classes, 0..count)?; + let edges: Py = PyTuple::new(py, edges)?.into_any().unbind(); + slf.edges = Some(edges.clone_ref(py)); + Ok(edges) + } + + /// Return whether model lineage is available without expanding its columns. + #[pyo3(name = "has_model")] + fn contains_model(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { + if self.models.bind(py).contains(model_name)? { + return Ok(true); + } + Ok(model_name + .extract::() + .is_ok_and(|name| self.compact.contains_key(&name))) + } + + /// Return whether model lineage retains unresolved root-star uncertainty. + fn model_has_star(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { + if let Ok(name) = model_name.extract::() + && let Some(compact) = self.compact.get(&name) + { + return Ok(compact.has_star); + } + let model: Bound<'_, PyAny> = self.models.bind(py).call_method1("get", (model_name,))?; + if model.is_none() { + return Ok(false); + } + model.getattr("has_star")?.extract() + } + + /// Return a target model's direct edge count without expanding edge objects. + fn edge_count_targeting(&self, model_name: &str) -> usize { + match &self.records { + Some(records) => records + .index + .by_target_model + .get(model_name) + .map_or(0, Vec::len), + None => self.edge_counts.get(model_name).copied().unwrap_or(0), + } + } + + /// Return edges whose target model is `model_name`. + fn edges_targeting<'py>( + mut slf: PyRefMut<'py, Self>, + model_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = slf.positions(|index| index.by_target_model.get(model_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return edges whose source resource is `resource_name`. + fn edges_sourced_from<'py>( + mut slf: PyRefMut<'py, Self>, + resource_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = + slf.positions(|index| index.by_source_resource.get(resource_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return direct downstream consumers of `resource_name.column_name`. + #[pyo3(signature = (*, resource_name, column_name))] + fn column_consumers<'py>( + slf: &Bound<'py, Self>, + resource_name: &str, + column_name: &Bound<'py, PyAny>, + ) -> PyResult> { + let sourced: Bound<'py, PyAny> = + slf.call_method1("edges_sourced_from", (resource_name,))?; + let mut consumers: Vec> = Vec::new(); + for edge in sourced.try_iter()? { + let edge: Bound<'py, PyAny> = edge?; + if edge + .getattr("source")? + .getattr("column_name")? + .eq(column_name)? + { + consumers.push(edge); + } + } + PyTuple::new(slf.py(), consumers) + } + + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let models: Bound<'_, PyString> = slf.getattr("models")?.repr()?; + let edges: Bound<'_, PyString> = slf.getattr("edges")?.repr()?; + Ok(format!( + "ProjectColumnLineage(models={models}, edges={edges})" + )) + } + + fn __eq__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !other.is_instance_of::() { + return Ok(py.NotImplemented()); + } + let equal: bool = slf.getattr("models")?.eq(other.getattr("models")?)? + && slf.getattr("edges")?.eq(other.getattr("edges")?)?; + Ok(pyo3::types::PyBool::new(py, equal) + .to_owned() + .into_any() + .unbind()) + } +} + +impl ProjectColumnLineage { + /// Build fast edge records from the compact facts, or the expanded models, in model order. + fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { + if self.records.is_some() { + return Ok(()); + } + let py: Python<'_> = classes.py(); + let mut builder = RecordBuilder::default(); + let models: Bound<'_, PyAny> = self.models.bind(py).clone(); + for name in &self.model_order { + if let Some(compact) = self.compact.get(name) { + builder.push_facts(classes, name, compact.columns.bind(py))?; + continue; + } + let model: Bound<'_, PyAny> = models.call_method1("get", (name,))?; + if model.is_none() { + continue; + } + builder.push_columns(name, &model.getattr("columns")?)?; + } + self.records = Some(EdgeRecords::new(builder.keys, builder.parts)); + Ok(()) + } + + fn positions(&self, select: impl Fn(&EdgeIndex) -> Option<&Vec>) -> Vec { + self.records + .as_ref() + .and_then(|records| select(&records.index)) + .cloned() + .unwrap_or_default() + } + + fn materialised( + &mut self, + classes: &LineageClasses<'_>, + positions: impl IntoIterator, + ) -> PyResult>> { + let Some(records) = self.records.as_mut() else { + return Ok(Vec::new()); + }; + positions + .into_iter() + .map(|position| records.edge(classes, position)) + .collect() + } +} + +impl EdgeRecords { + fn new(keys: Vec, parts: Vec) -> Self { + let index: EdgeIndex = edge_index(&keys); + let materialised: Vec>> = keys.iter().map(|_| None).collect(); + Self { + keys, + parts, + index, + materialised, + } + } + + /// The edge at `position`, built once per distinct fact edge. + fn edge(&mut self, classes: &LineageClasses<'_>, position: usize) -> PyResult> { + let py: Python<'_> = classes.py(); + let (resource_type, transform, confidence) = match &self.parts[position] { + EdgeParts::Given(edge) => return Ok(edge.clone_ref(py)), + EdgeParts::Facts { + resource_type, + transform, + confidence, + } => (resource_type, transform, confidence), + }; + let first: usize = self.index.first_equal[position]; + if let Some(edge) = &self.materialised[first] { + return Ok(edge.clone_ref(py)); + } + let key: &EdgeKey = &self.keys[position]; + let edge: Py = classes + .edge( + classes.qualified(resource_type.bind(py), &key.source_name, &key.source_column)?, + &classes.qualified( + &classes.model_resource, + &key.target_model, + &key.target_column, + )?, + transform.bind(py), + confidence.bind(py), + )? + .unbind(); + self.materialised[first] = Some(edge.clone_ref(py)); + Ok(edge) + } +} + +fn empty_key() -> EdgeKey { + EdgeKey { + source_type: String::new(), + source_name: String::new(), + source_column: String::new(), + target_model: String::new(), + target_column: String::new(), + transform: String::new(), + confidence: String::new(), + } +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +/// Edge records accumulated in model order. +#[derive(Default)] +struct RecordBuilder { + keys: Vec, + parts: Vec, +} + +impl RecordBuilder { + /// Records of compact facts, or of plain column facts, targeting `model_name`. + fn push_facts( + &mut self, + classes: &LineageClasses<'_>, + model_name: &str, + columns: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if !columns.is_instance(&classes.compact_facts)? { + return self.push_columns(model_name, columns); + } + let py: Python<'_> = columns.py(); + let pool: Bound<'_, PyAny> = columns.getattr("string_pool")?; + let remapped: bool = columns.getattr("resource_names")?.is_truthy()? + || columns.getattr("resource_name_indexes")?.is_truthy()?; + let mut members = CodeMembers::default(); + for row in columns.getattr("rows")?.try_iter()? { + let (name_index, transform_code, confidence_code, sources): ( + Bound<'_, PyAny>, + i64, + i64, + Bound<'_, PyAny>, + ) = row?.extract()?; + let output_column: String = pool.get_item(name_index)?.extract()?; + for source in sources.try_iter()? { + let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + source?.extract()?; + let resource_type: Bound<'_, PyAny> = pool.get_item(kind)?; + let resource_name: String = if remapped { + columns.call_method1("resource_name", (name,))?.extract()? + } else { + pool.get_item(name)?.extract()? + }; + let transform: Py = members.transform(columns, transform_code)?; + let confidence: Py = members.confidence(columns, confidence_code)?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: resource_name, + source_column: pool.get_item(column)?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(transform.bind(py))?, + confidence: text(confidence.bind(py))?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform, + confidence, + }); + } + } + Ok(()) + } + + /// Records of columns with `upstream_columns`, `transform_kind` and `confidence`. + fn push_columns(&mut self, model_name: &str, columns: &Bound<'_, PyAny>) -> PyResult<()> { + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let output_column: String = column.getattr("output_column")?.extract()?; + let transform: Bound<'_, PyAny> = column.getattr("transform_kind")?; + let confidence: Bound<'_, PyAny> = column.getattr("confidence")?; + for source in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let resource_type: Bound<'_, PyAny> = source.getattr("resource_type")?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: source.getattr("resource_name")?.extract()?, + source_column: source.getattr("column_name")?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(&transform)?, + confidence: text(&confidence)?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform: transform.clone().unbind(), + confidence: confidence.clone().unbind(), + }); + } + } + Ok(()) + } +} + +/// `facts.transform_kind(code)` and `facts.confidence(code)`, looked up once per code. +#[derive(Default)] +struct CodeMembers { + members: HashMap<(&'static str, i64), Py>, +} + +impl CodeMembers { + fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts, "transform_kind", code) + } + + fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts, "confidence", code) + } + + fn member( + &mut self, + facts: &Bound<'_, PyAny>, + method: &'static str, + code: i64, + ) -> PyResult> { + if let Some(member) = self.members.get(&(method, code)) { + return Ok(member.clone_ref(facts.py())); + } + let member: Py = facts.call_method1(method, (code,))?.unbind(); + let _ = self + .members + .insert((method, code), member.clone_ref(facts.py())); + Ok(member) + } +} + +/// The edge count of compact facts or plain column facts. +fn fact_edge_count(classes: &LineageClasses<'_>, columns: &Bound<'_, PyAny>) -> PyResult { + if !columns.is_instance(&classes.compact_facts)? { + return upstream_count(columns); + } + let mut count: usize = 0; + for row in columns.getattr("rows")?.try_iter()? { + count += row?.get_item(3)?.len()?; + } + Ok(count) +} + +fn upstream_count(columns: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = 0; + for column in columns.try_iter()? { + count += column?.getattr("upstream_columns")?.len()?; + } + Ok(count) +} + +/// `ModelColumnLineage` of compact or plain column facts. +fn model_from_facts<'py>( + classes: &LineageClasses<'py>, + model_name: &str, + columns: &Bound<'py, PyAny>, + has_star: bool, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut lineage_columns: Vec> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let mut sources: Vec> = Vec::new(); + for source in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + sources.push(classes.lineage_source.call1(( + source.getattr("resource_type")?, + source.getattr("resource_name")?, + source.getattr("column_name")?, + ))?); + } + lineage_columns.push(classes.column_lineage.call1(( + column.getattr("output_column")?, + column.getattr("transform_kind")?, + py.None(), + PyTuple::new(py, sources)?, + &classes.unknown_nullability, + column.getattr("confidence")?, + ))?); + } + classes.model(model_name, lineage_columns, has_star) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs deleted file mode 100644 index 469124bee8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Fast column lineage. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::lineage::main::build_fast_lineage::build_fast_lineage; -use sqlbuild_analysis::lineage::models::{ - FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, LineageSource, -}; -use sqlbuild_analysis::semantic_validation::models as validation; - -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::models::ProjectCatalog; -use crate::bindings::types::CompilerDetach; - -type SourceRow = (&'static str, String, String); -type ColumnRow = (String, &'static str, &'static str, Vec); -/// `(dialect, [(resource_type, name, columns)], [(star_expansion, query_sql, column_names)])`. -type LineageRequestInput = ( - Option, - Vec<(String, String, Vec)>, - Vec<(bool, String, Vec)>, -); -/// `(status, columns, has_star, detail)`; see `FastLineageOutcome::into_parts`. -type OutcomeRow = (&'static str, Vec, bool, Option); - -/// Build fast lineage for the requested models on `catalog`'s analysis pool, if any. -#[pyfunction] -fn build_fast_column_lineage( - py: Python<'_>, - catalog: Option>, - request: LineageRequestInput, -) -> PyResult> { - let (dialect, schema, models) = request; - let schema: Vec = schema - .into_iter() - .map(schema_resource) - .collect::>()?; - let models: Vec = models.into_iter().map(lineage_model).collect(); - let request = FastLineageRequest { - dialect, - schema, - models, - }; - let catalog: Option<&validation::ProjectCatalog> = - catalog.as_ref().map(|catalog| &catalog.inner); - let outcomes: Vec = py - .compiler_detach(|| build_fast_lineage(&request, catalog)) - .map_err(compiler_error)?; - Ok(outcomes.into_iter().map(outcome_row).collect()) -} - -fn schema_resource( - (resource_type, name, columns): (String, String, Vec), -) -> PyResult { - let Some(kind) = LineageResourceType::from_value(&resource_type) else { - return Err(PyValueError::new_err(format!( - "unknown resource: {resource_type}" - ))); - }; - Ok(LineageSchemaResource { - resource_type: kind, - name, - columns, - }) -} - -fn lineage_model( - (star_expansion, query_sql, names): (bool, String, Vec), -) -> FastLineageModel { - if star_expansion { - FastLineageModel::StarExpansion { - query_sql, - existing_columns: names, - } - } else { - FastLineageModel::Parse { - query_sql, - inferred_columns: names, - } - } -} - -fn outcome_row(outcome: FastLineageOutcome) -> OutcomeRow { - let (status, columns, has_star, detail) = outcome.into_parts(); - (status, column_rows(columns), has_star, detail) -} - -fn column_rows(columns: Vec) -> Vec { - columns - .into_iter() - .map(|column| { - ( - column.output_column, - column.transform_kind.as_str(), - column.confidence.as_str(), - column - .upstream_columns - .into_iter() - .map(source_row) - .collect(), - ) - }) - .collect() -} - -fn source_row(source: LineageSource) -> SourceRow { - ( - source.resource_type.as_str(), - source.resource_name, - source.column_name, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_fast_column_lineage, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs new file mode 100644 index 0000000000..45a01e499d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -0,0 +1,185 @@ +//! The Python lineage models and enums native lineage builds. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; + +const LINEAGE_MODELS_MODULE: &str = "sqlbuild.compiler.lineage.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const LOGGING_MODULE: &str = "logging"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; + +/// The lineage model classes and enums, imported once per call. +pub(crate) struct LineageClasses<'py> { + pub(crate) qualified_column: Bound<'py, PyAny>, + pub(crate) lineage_source: Bound<'py, PyAny>, + pub(crate) lineage_edge: Bound<'py, PyAny>, + pub(crate) column_lineage: Bound<'py, PyAny>, + pub(crate) model_lineage: Bound<'py, PyAny>, + pub(crate) semantic_use: Bound<'py, PyAny>, + pub(crate) column_fact: Bound<'py, PyAny>, + pub(crate) source_fact: Bound<'py, PyAny>, + pub(crate) compact_facts: Bound<'py, PyAny>, + pub(crate) transform_kind: Bound<'py, PyAny>, + pub(crate) confidence: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) analysis_dialect: Bound<'py, PyAny>, + pub(crate) resource_type: Bound<'py, PyAny>, + pub(crate) model_resource: Bound<'py, PyAny>, + pub(crate) unknown_nullability: Bound<'py, PyAny>, +} + +impl<'py> LineageClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + let models: Bound<'py, PyModule> = py.import(LINEAGE_MODELS_MODULE)?; + let types: Bound<'py, PyModule> = py.import(LINEAGE_TYPES_MODULE)?; + let compile_models: Bound<'py, PyModule> = py.import(COMPILE_MODELS_MODULE)?; + let resource_type: Bound<'py, PyAny> = py + .import(COMPILE_TYPES_MODULE)? + .getattr("CompiledResourceType")?; + let nullability: Bound<'py, PyAny> = types.getattr("InferredNullability")?; + Ok(Self { + qualified_column: models.getattr("QualifiedLineageColumn")?, + lineage_source: models.getattr("ColumnLineageSource")?, + lineage_edge: models.getattr("ColumnLineageEdge")?, + column_lineage: models.getattr("ColumnLineage")?, + model_lineage: models.getattr("ModelColumnLineage")?, + semantic_use: models.getattr("DirectSemanticColumnUse")?, + column_fact: compile_models.getattr("CompiledLineageColumnFact")?, + source_fact: compile_models.getattr("CompiledLineageSourceFact")?, + compact_facts: compile_models.getattr("CompactLineageFacts")?, + transform_kind: types.getattr("ColumnTransformKind")?, + confidence: types.getattr("ColumnLineageConfidence")?, + analysis_dialect: types.getattr("PolyglotAnalysisDialect")?, + model_resource: resource_type.getattr("MODEL")?, + unknown_nullability: nullability.getattr("UNKNOWN")?, + nullability, + resource_type, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.qualified_column.py() + } + + /// `QualifiedLineageColumn(resource_type, resource_name, column_name)`. + pub(crate) fn qualified( + &self, + resource_type: &Bound<'py, PyAny>, + resource_name: &str, + column_name: &str, + ) -> PyResult> { + self.qualified_column + .call1((resource_type, resource_name, column_name)) + } + + /// `ColumnLineageEdge(source, target, transform_kind, confidence)`. + pub(crate) fn edge( + &self, + source: Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, + transform_kind: &Bound<'py, PyAny>, + confidence: &Bound<'py, PyAny>, + ) -> PyResult> { + self.lineage_edge + .call1((source, target, transform_kind, confidence)) + } + + /// `ModelColumnLineage(model_name, columns, has_star)`. + pub(crate) fn model( + &self, + model_name: &str, + columns: Vec>, + has_star: bool, + ) -> PyResult> { + self.model_lineage + .call1((model_name, PyTuple::new(self.py(), columns)?, has_star)) + } + + /// `ColumnLineage` of one native column; `nullability` is the enum member to record. + pub(crate) fn column( + &self, + column: &LineageColumn, + nullability: &Bound<'py, PyAny>, + resource_enum: bool, + ) -> PyResult> { + let mut sources: Vec> = Vec::with_capacity(column.upstream_columns.len()); + for source in &column.upstream_columns { + let resource_type: Bound<'py, PyAny> = if resource_enum { + self.resource_type.call1((source.resource_type.as_str(),))? + } else { + pyo3::types::PyString::new(self.py(), source.resource_type.as_str()).into_any() + }; + sources.push(self.lineage_source.call1(( + resource_type, + &source.resource_name, + &source.column_name, + ))?); + } + self.column_lineage.call1(( + &column.output_column, + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + self.py().None(), + PyTuple::new(self.py(), sources)?, + nullability, + self.confidence.call1((column.confidence.as_str(),))?, + )) + } + + /// `CompiledLineageColumnFact` of one native star-expansion column. + pub(crate) fn column_fact(&self, column: &LineageColumn) -> PyResult> { + let sources: Vec> = column + .upstream_columns + .iter() + .map(|source| self.source_fact(source)) + .collect::>()?; + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("output_column", &column.output_column)?; + fields.set_item("upstream_columns", PyTuple::new(self.py(), sources)?)?; + fields.set_item( + "transform_kind", + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + )?; + fields.set_item( + "confidence", + self.confidence.call1((column.confidence.as_str(),))?, + )?; + self.column_fact.call((), Some(&fields)) + } + + fn source_fact(&self, source: &LineageSource) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("resource_type", source.resource_type.as_str())?; + fields.set_item("resource_name", &source.resource_name)?; + fields.set_item("column_name", &source.column_name)?; + self.source_fact.call((), Some(&fields)) + } +} + +/// A structured debug event on the `sqlbuild.lineage` logger for one model. +pub(crate) fn log_lineage_event( + py: Python<'_>, + message: &str, + model_name: &str, + error: Option<&str>, +) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import(LOGGING_MODULE)? + .getattr("getLogger")? + .call1((LINEAGE_LOGGER,))?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("logger", logger)?; + fields.set_item("message", message)?; + fields.set_item("sqlbuild_model", model_name)?; + fields.set_item("sqlbuild_error", error)?; + let _ = py + .import(DEBUG_EVENT_MODULE)? + .getattr("log_debug_event")? + .call((), Some(&fields))?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index f0aec5490f..cb9ea47c07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,4 +1,7 @@ -pub(crate) mod fast_lineage; +pub(crate) mod column_lineage_graph; +pub(crate) mod lineage_objects; +pub(crate) mod project_lineage; +pub(crate) mod project_schema; pub(crate) mod registration; pub(crate) mod relation_fingerprint; -pub(crate) mod rich_lineage; +pub(crate) mod tree_render; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs new file mode 100644 index 0000000000..93d1b1f820 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -0,0 +1,372 @@ +//! Project column lineage in FAST or RICH mode, and direct semantic uses, from a project view. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage::main::build_fast_lineage::build_fast_lineage; +use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; +use sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses; +use sqlbuild_analysis::lineage::models::{ + DirectSemanticUse, FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, + RichLineageOutcome, RichLineageRequest, +}; +use sqlbuild_analysis::lineage_graph::main::analysis_dialect::analysis_dialect; +use sqlbuild_analysis::lineage_graph::models::AnalysisDialect; +use sqlbuild_analysis::semantic_validation::models as validation; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; +use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_lineage_event}; +use crate::bindings::_helpers::lineage::project_schema::{ + fast_schema, inferred_names, rich_schema, +}; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.lineage.exceptions"; +const FAST_MODE: &str = "fast"; +const RICH_MODE: &str = "rich"; +const GENERIC_DIALECT: &str = "generic"; + +/// The project's column lineage graph in `mode`, or `None` without SQL analysis. +#[pyfunction] +fn build_project_column_lineage<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + mode: &Bound<'py, PyAny>, + model_names: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let mode: String = mode.str()?.extract()?; + if mode != FAST_MODE && mode != RICH_MODE { + return Ok(None); + } + let classes: LineageClasses<'py> = LineageClasses::new(project.py())?; + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(None); + } + let models: Vec> = selected_models(project, model_names)?; + let lineage: Bound<'py, PyAny> = if mode == FAST_MODE { + fast_lineage(&classes, project, dialect, &models)? + } else { + rich_lineage(&classes, project, dialect, &models)? + }; + Ok(Some(lineage.unbind())) +} + +/// Direct non-projection semantic column uses of the requested models. +#[pyfunction] +fn build_direct_semantic_uses<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&str>, + model_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let classes: LineageClasses<'py> = LineageClasses::new(py)?; + let schema = rich_schema(project)?; + let mut models: Vec<(String, String, String)> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + if model_names.contains(&name)? { + models.push(( + name.extract()?, + model.getattr("query_sql")?.extract()?, + model.getattr("authored_sql")?.extract()?, + )); + } + } + let dialect: &str = dialect + .filter(|name| !name.is_empty()) + .unwrap_or(GENERIC_DIALECT); + let uses: Vec = py + .compiler_detach(|| direct_semantic_uses(&schema, &models, dialect)) + .map_err(compiler_error)?; + let objects: Vec> = uses + .into_iter() + .map(|usage| semantic_use(&classes, usage)) + .collect::>()?; + PyTuple::new(py, objects) +} + +fn semantic_use<'py>( + classes: &LineageClasses<'py>, + usage: DirectSemanticUse, +) -> PyResult> { + let source: Bound<'py, PyAny> = classes.qualified( + &classes + .resource_type + .call1((usage.resource_type.as_str(),))?, + &usage.resource_name, + &usage.column, + )?; + classes.semantic_use.call1(( + usage.model, + usage.context, + usage.expression, + source, + classes.confidence.call1((usage.confidence,))?, + usage.line, + usage.offset, + )) +} + +fn selected_models<'py>( + project: &Bound<'py, PyAny>, + model_names: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let mut models: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let selected: bool = match model_names { + None => true, + Some(names) => names.contains(model.getattr("name")?)?, + }; + if selected { + models.push(model); + } + } + Ok(models) +} + +/// One model's FAST lineage work: its compact facts and whether its root star expands. +struct FastModel<'py> { + model: Bound<'py, PyAny>, + name: String, + facts: Option>, + expands_star: bool, +} + +fn fast_lineage<'py>( + classes: &LineageClasses<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut work: Vec> = Vec::with_capacity(models.len()); + let mut requests: Vec = Vec::new(); + for model in models { + let facts: Bound<'py, PyAny> = model.getattr("fast_lineage_columns")?; + let expands_star: bool = model.getattr("fast_lineage_has_star")?.is_truthy()? + && !model.getattr("fast_lineage_star_resolved")?.is_truthy()?; + if facts.is_none() { + requests.push(FastLineageModel::Parse { + query_sql: model.getattr("query_sql")?.extract()?, + inferred_columns: inferred_names(model)?, + }); + } else if expands_star { + requests.push(FastLineageModel::StarExpansion { + query_sql: model.getattr("query_sql")?.extract()?, + existing_columns: output_columns(classes, &facts)?, + }); + } + work.push(FastModel { + model: model.clone(), + name: model.getattr("name")?.extract()?, + facts: (!facts.is_none()).then_some(facts), + expands_star, + }); + } + let outcomes: Vec = if requests.is_empty() { + Vec::new() + } else { + let binding_catalog: Bound<'py, PyAny> = project.getattr("binding_catalog")?; + let catalog: Option> = if binding_catalog.is_none() { + None + } else { + binding_catalog.getattr("native")?.extract()? + }; + let request = FastLineageRequest { + dialect: dialect.map(|name| name.extract()).transpose()?, + schema: fast_schema(project)?, + models: requests, + }; + let native: Option<&validation::ProjectCatalog> = + catalog.as_ref().map(|catalog| &catalog.inner); + py.compiler_detach(|| build_fast_lineage(&request, native)) + .map_err(compiler_error)? + }; + let mut outcomes = outcomes.into_iter(); + let built: Bound<'py, PyDict> = PyDict::new(py); + let compact: Bound<'py, PyDict> = PyDict::new(py); + let mut model_order: Vec = Vec::new(); + for model in work { + if let Some(mut columns) = model.facts { + if model.expands_star { + let star_columns: Vec = outcomes + .next() + .map(|outcome| outcome.into_parts().1) + .unwrap_or_default(); + let mut expanded: Vec> = + columns.try_iter()?.collect::>()?; + for column in &star_columns { + expanded.push(classes.column_fact(column)?); + } + columns = PyTuple::new(py, expanded)?.into_any(); + } + compact.set_item( + &model.name, + (columns, model.model.getattr("fast_lineage_has_star")?), + )?; + model_order.push(model.name); + continue; + } + match outcomes.next() { + Some(FastLineageOutcome::Built { columns, has_star }) => { + let lineage_columns: Vec> = columns + .iter() + .map(|column| classes.column(column, &classes.unknown_nullability, false)) + .collect::>()?; + built.set_item( + &model.name, + classes.model(&model.name, lineage_columns, has_star)?, + )?; + model_order.push(model.name); + } + Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( + py, + "fast column lineage parse failed; falling back", + &model.name, + Some(&detail), + )?, + Some(FastLineageOutcome::UnknownDialect(detail)) => { + return Err(unknown_dialect_error(py, &detail)); + } + _ => {} + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("models", built)?; + fields.set_item("compact_models", compact)?; + fields.set_item("model_order", PyTuple::new(py, model_order)?)?; + py.get_type::() + .call_method("from_fast_facts", (), Some(&fields)) +} + +/// The output column names of compact facts or plain column facts. +fn output_columns(classes: &LineageClasses<'_>, facts: &Bound<'_, PyAny>) -> PyResult> { + if facts.is_instance(&classes.compact_facts)? { + let pool: Bound<'_, PyAny> = facts.getattr("string_pool")?; + return facts + .getattr("rows")? + .try_iter()? + .map(|row| pool.get_item(row?.get_item(0)?)?.extract()) + .collect(); + } + facts + .try_iter()? + .map(|column| column?.getattr("output_column")?.extract()) + .collect() +} + +fn unknown_dialect_error(py: Python<'_>, dialect: &str) -> PyErr { + match py + .import(EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("UnknownLineageDialectError")) + .and_then(|class| class.call1((format!("Unknown dialect: {dialect}"),))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn rich_lineage<'py>( + classes: &LineageClasses<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = classes.py(); + let outcomes: Vec = if models.is_empty() { + Vec::new() + } else { + let request = RichLineageRequest { + dialect: polyglot_dialect(classes, dialect)?, + schema: rich_schema(project)?, + models: models + .iter() + .map(|model| model.getattr("query_sql")?.extract()) + .collect::>()?, + }; + py.compiler_detach(|| build_rich_lineage(&request)) + .map_err(compiler_error)? + }; + let built: Bound<'py, PyDict> = PyDict::new(py); + let mut edges: Vec> = Vec::new(); + for (model, outcome) in models.iter().zip(outcomes) { + let name: String = model.getattr("name")?.extract()?; + match outcome { + RichLineageOutcome::Built { columns, has_star } => { + let mut lineage_columns: Vec> = Vec::with_capacity(columns.len()); + for column in &columns { + let nullability: Bound<'py, PyAny> = + classes.nullability.call1((column.nullability.as_str(),))?; + lineage_columns.push(classes.column(&column.column, &nullability, true)?); + } + built.set_item(&name, classes.model(&name, lineage_columns, has_star)?)?; + } + RichLineageOutcome::Skipped(detail) => log_lineage_event( + py, + "rich column lineage analysis failed; skipping model", + &name, + Some(&detail), + )?, + } + } + for item in built.iter() { + let (target_name, result): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item; + for column in result.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let target: Bound<'py, PyAny> = classes.qualified_column.call1(( + &classes.model_resource, + &target_name, + column.getattr("output_column")?, + ))?; + let transform: Bound<'py, PyAny> = column.getattr("transform_kind")?; + let confidence: Bound<'py, PyAny> = column.getattr("confidence")?; + for upstream in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'py, PyAny> = upstream?.call_method0("as_qualified_column")?; + edges.push(classes.edge(source, &target, &transform, &confidence)?); + } + } + } + let lineage: Bound<'py, PyAny> = py.get_type::().into_any(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("models", built)?; + fields.set_item("edges", PyTuple::new(py, edges)?)?; + lineage.call((), Some(&fields)) +} + +/// Python's `_polyglot_dialect(dialect).value`. +fn polyglot_dialect( + classes: &LineageClasses<'_>, + dialect: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let Some(dialect) = dialect else { + return Ok(GENERIC_DIALECT.to_owned()); + }; + let normalized: String = dialect + .call_method0("strip")? + .call_method0("lower")? + .extract()?; + match analysis_dialect(&normalized) { + AnalysisDialect::Alias(value) => Ok(value.to_owned()), + AnalysisDialect::Named(name) => classes + .analysis_dialect + .call1((name,))? + .getattr("value")? + .extract(), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_project_column_lineage, module)?)?; + module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs new file mode 100644 index 0000000000..43085f139d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs @@ -0,0 +1,112 @@ +//! Project resources and their columns, read from a compiled project view for lineage. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::lineage::models::{ + LineageResourceType, LineageSchemaResource, RichSchemaResource, +}; + +type TypedColumns = Vec<(String, Option)>; + +/// The project's models, then sources, then seeds, each with its typed and schema columns. +struct ProjectColumns { + resource_type: LineageResourceType, + name: String, + typed: TypedColumns, + schema: TypedColumns, +} + +/// Fast lineage's schema: model inferred then schema column names; source and seed columns. +pub(crate) fn fast_schema(project: &Bound<'_, PyAny>) -> PyResult> { + let mut schema: Vec = Vec::new(); + for resource in project_columns(project)? { + let mut columns: Vec = Vec::with_capacity(resource.typed.len()); + for (name, _) in resource.typed.into_iter().chain(resource.schema) { + columns.push(name); + } + schema.push(LineageSchemaResource { + resource_type: resource.resource_type, + name: resource.name, + columns, + }); + } + Ok(schema) +} + +/// Rich lineage's typed schema: inferred or declared types, then model schema columns. +pub(crate) fn rich_schema(project: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_columns(project)? + .into_iter() + .map(|resource| RichSchemaResource { + resource_type: resource.resource_type, + name: resource.name, + assigned: resource.typed, + defaulted: resource.schema, + }) + .collect()) +} + +/// `[name for name, _, _ in model.inferred_rows or ()]`. +pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult> { + Ok(inferred_columns(model)? + .into_iter() + .map(|(name, _)| name) + .collect()) +} + +fn project_columns(project: &Bound<'_, PyAny>) -> PyResult> { + let mut resources: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + resources.push(ProjectColumns { + resource_type: LineageResourceType::Model, + name: model.getattr("name")?.extract()?, + typed: inferred_columns(&model)?, + schema: schema_columns(&model)?, + }); + } + for (attribute, entry, resource_type) in [ + ("sources", "source_entry", LineageResourceType::Source), + ("seeds", "schema_entry", LineageResourceType::Seed), + ] { + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let mut typed: TypedColumns = Vec::new(); + for column in resource.getattr(entry)?.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + typed.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + resources.push(ProjectColumns { + resource_type, + name: resource.getattr("name")?.extract()?, + typed, + schema: Vec::new(), + }); + } + } + Ok(resources) +} + +fn inferred_columns(model: &Bound<'_, PyAny>) -> PyResult { + let rows: Bound<'_, PyAny> = model.getattr("inferred_rows")?; + if !rows.is_truthy()? { + return Ok(Vec::new()); + } + rows.try_iter()? + .map(|row| { + let (name, column_type, _nullability): (String, Option, Bound<'_, PyAny>) = + row?.extract()?; + Ok((name, column_type)) + }) + .collect() +} + +fn schema_columns(model: &Bound<'_, PyAny>) -> PyResult { + let rows: Bound<'_, PyAny> = model.getattr("schema_column_rows")?; + if !rows.is_truthy()? { + return Ok(Vec::new()); + } + rows.try_iter()?.map(|row| row?.extract()).collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index 806db27529..cb84372d9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -1,11 +1,14 @@ -//! Register the native column lineage functions. +//! Register the native column lineage functions and classes. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lineage::{fast_lineage, relation_fingerprint, rich_lineage}; +use crate::bindings::_helpers::lineage::{ + column_lineage_graph, project_lineage, relation_fingerprint, tree_render, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - fast_lineage::register(module)?; + column_lineage_graph::register(module)?; + project_lineage::register(module)?; relation_fingerprint::register(module)?; - rich_lineage::register(module) + tree_render::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs deleted file mode 100644 index 2a9c17084f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Rich column lineage. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; -use sqlbuild_analysis::lineage::models::{ - LineageResourceType, LineageSource, RichLineageColumn, RichLineageOutcome, RichLineageRequest, - RichSchemaResource, -}; - -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::types::CompilerDetach; - -type TypedColumns = Vec<(String, Option)>; -type SourceRow = (&'static str, String, String); -type ColumnRow = ( - String, - &'static str, - &'static str, - &'static str, - Vec, -); -/// `(dialect, [(resource_type, name, assigned, defaulted)], [query_sql])`. -type RichRequestInput = ( - String, - Vec<(String, String, TypedColumns, TypedColumns)>, - Vec, -); -/// `(status, columns, has_star, detail)`: detail is the polyglot error or the deferral kind. -type OutcomeRow = (&'static str, Vec, bool, Option); - -/// Build rich lineage for the requested models' SQL, in request order. -#[pyfunction] -fn build_rich_column_lineage( - py: Python<'_>, - request: RichRequestInput, -) -> PyResult> { - let (dialect, schema, models) = request; - let schema: Vec = schema - .into_iter() - .map(schema_resource) - .collect::>()?; - let request = RichLineageRequest { - dialect, - schema, - models, - }; - let outcomes: Vec = py - .compiler_detach(|| build_rich_lineage(&request)) - .map_err(compiler_error)?; - Ok(outcomes.into_iter().map(outcome_row).collect()) -} - -fn schema_resource( - (resource_type, name, assigned, defaulted): (String, String, TypedColumns, TypedColumns), -) -> PyResult { - let Some(kind) = LineageResourceType::from_value(&resource_type) else { - return Err(PyValueError::new_err(format!( - "unknown resource: {resource_type}" - ))); - }; - Ok(RichSchemaResource { - resource_type: kind, - name, - assigned, - defaulted, - }) -} - -fn outcome_row(outcome: RichLineageOutcome) -> OutcomeRow { - let (status, columns, has_star, detail) = outcome.into_parts(); - ( - status, - columns.into_iter().map(column_row).collect(), - has_star, - detail, - ) -} - -fn column_row(column: RichLineageColumn) -> ColumnRow { - let RichLineageColumn { - column, - nullability, - } = column; - ( - column.output_column, - column.transform_kind.as_str(), - column.confidence.as_str(), - nullability.as_str(), - column - .upstream_columns - .into_iter() - .map(source_row) - .collect(), - ) -} - -fn source_row(source: LineageSource) -> SourceRow { - ( - source.resource_type.as_str(), - source.resource_name, - source.column_name, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_rich_column_lineage, module)?)?; - module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; - Ok(()) -} - -type SemanticUseRow = ( - String, - String, - String, - &'static str, - String, - String, - String, - Option, - Option, -); - -/// Project semantic uses onto authored resources without Python analysis reconstruction. -#[pyfunction] -fn build_direct_semantic_uses( - py: Python<'_>, - schema: Vec<(String, String, TypedColumns, TypedColumns)>, - models: Vec<(String, String, String)>, - dialect: &str, -) -> PyResult> { - let schema = schema - .into_iter() - .map(schema_resource) - .collect::>>()?; - let uses = py - .compiler_detach(|| { - sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses( - &schema, &models, dialect, - ) - }) - .map_err(compiler_error)?; - Ok(uses - .into_iter() - .map(|usage| { - ( - usage.model, - usage.context, - usage.expression, - usage.resource_type.as_str(), - usage.resource_name, - usage.column, - usage.confidence, - usage.line, - usage.offset, - ) - }) - .collect()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs new file mode 100644 index 0000000000..6007889188 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs @@ -0,0 +1,306 @@ +//! Lineage output: box-drawing dependency and column-trace trees, and column JSON payloads. + +use std::collections::HashSet; + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage_graph::main::trace_limit_note::trace_limit_note; +use sqlbuild_analysis::lineage_graph::main::tree_lines::tree_lines; +use sqlbuild_analysis::lineage_graph::models::TreeLine; +use sqlbuild_analysis::lineage_graph::types::TreeVisitor; + +const BUILTINS_MODULE: &str = "builtins"; + +/// Box-drawing dependency tree lines for one focus node, expanding each node once. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_dependency_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_dependency_tree", request)?; + let py: Python<'_> = request.py(); + let mut tree = DependencyTree { + deps: request.get("deps")?, + sort_key: request.get("sort_key")?, + format_node: request.get("format_node")?, + branch_style: request.get("branch_style")?, + already_shown: request.get("already_shown")?, + sorted: py.import(BUILTINS_MODULE)?.getattr("sorted")?, + identities: PyDict::new(py), + lines: Vec::new(), + }; + let focus: Bound<'_, PyAny> = request.get("focus")?; + let seen: Bound<'_, PyAny> = request.get("seen")?; + let roots: Vec> = tree.children(&focus)?; + let mut shown: HashSet = HashSet::new(); + for node in seen.try_iter()? { + let _ = shown.insert(tree.key(&node?)?); + } + tree_lines(roots, shown, &mut tree)?; + Ok(tree.lines) +} + +/// Box-drawing column-trace tree lines plus any truncation note. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_column_trace_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_column_trace_tree", request)?; + let target: Bound<'_, PyAny> = request.get("target")?; + let mut tree = ColumnTraceTree { + deps: request.get("deps")?, + column_id: request.get("column_id")?, + related_column: request.get("related_column")?, + format_related: request.get("format_related")?, + branch_style: request.get("branch_style")?, + already_shown: request.get("already_shown")?, + last_key: String::new(), + lines: Vec::new(), + }; + let seen: HashSet = HashSet::from([text(&tree.column_id.call1((&target,))?)?]); + let roots: Vec> = + tree.sorted_edges(&text(&tree.column_id.call1((&target,))?)?)?; + tree_lines(roots, seen, &mut tree)?; + let mut lines: Vec = tree.lines; + lines.extend(limit_note( + &request.get("total")?, + &request.get("limit")?, + &request.get("note_style")?, + )?); + Ok(lines) +} + +/// The shared truncation note when a column trace is limited. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_column_trace_limit_note(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_column_trace_limit_note", request)?; + limit_note( + &request.get("total")?, + &request.get("limit")?, + &request.get("note_style")?, + ) +} + +/// A qualified lineage column's JSON payload, its resource type rendered by the caller. +#[pyfunction] +#[pyo3(signature = (**request))] +fn serialize_column<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("serialize_column", request)?; + column_payload( + &request.get("column")?, + &request.get("render_resource_type")?, + ) +} + +/// A column lineage edge's JSON payload, resource types rendered by the caller. +#[pyfunction] +#[pyo3(signature = (**request))] +fn serialize_column_edge<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("serialize_column_edge", request)?; + let edge: Bound<'py, PyAny> = request.get("edge")?; + let render: Bound<'py, PyAny> = request.get("render_resource_type")?; + let payload: Bound<'py, PyDict> = PyDict::new(request.py()); + payload.set_item("source", column_payload(&edge.getattr("source")?, &render)?)?; + payload.set_item("target", column_payload(&edge.getattr("target")?, &render)?)?; + payload.set_item("transform", edge.getattr("transform_kind")?.str()?)?; + payload.set_item("confidence", edge.getattr("confidence")?.str()?)?; + Ok(payload) +} + +/// Keyword arguments of one call, each required. +struct Request<'a, 'py> { + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + py: Python<'py>, +} + +impl<'a, 'py> Request<'a, 'py> { + fn new(function: &'static str, arguments: Option<&'a Bound<'py, PyDict>>) -> PyResult { + let py: Python<'py> = match arguments { + Some(arguments) => arguments.py(), + None => { + return Err(PyTypeError::new_err(format!( + "{function}() missing required keyword-only arguments" + ))); + } + }; + Ok(Self { + function, + arguments, + py, + }) + } + + fn py(&self) -> Python<'py> { + self.py + } + + fn get(&self, name: &str) -> PyResult> { + let value: Option> = match self.arguments { + Some(arguments) => arguments.get_item(name)?, + None => None, + }; + value.ok_or_else(|| { + PyTypeError::new_err(format!( + "{}() missing 1 required keyword-only argument: '{name}'", + self.function + )) + }) + } +} + +/// `str(value)`, as an f-string renders a callback's text. +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn limit_note( + total: &Bound<'_, PyAny>, + limit: &Bound<'_, PyAny>, + note_style: &Bound<'_, PyAny>, +) -> PyResult> { + let notes: Vec = trace_limit_note(total.extract()?, limit.extract()?); + if notes.is_empty() { + return Ok(notes); + } + let mut lines: Vec = vec![String::new()]; + for note in notes { + lines.push(note_style.call1((note,))?.extract()?); + } + Ok(lines) +} + +fn column_payload<'py>( + column: &Bound<'py, PyAny>, + render_resource_type: &Bound<'py, PyAny>, +) -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(column.py()); + payload.set_item("resource_type", render_resource_type.call1((column,))?)?; + payload.set_item("resource_name", column.getattr("resource_name")?)?; + payload.set_item("column_name", column.getattr("column_name")?)?; + Ok(payload) +} + +/// Dependency nodes shown once by Python equality, children sorted by the caller's key. +struct DependencyTree<'py> { + deps: Bound<'py, PyAny>, + sort_key: Bound<'py, PyAny>, + format_node: Bound<'py, PyAny>, + branch_style: Bound<'py, PyAny>, + already_shown: Bound<'py, PyAny>, + sorted: Bound<'py, PyAny>, + identities: Bound<'py, PyDict>, + lines: Vec, +} + +impl<'py> TreeVisitor, usize, PyErr> for DependencyTree<'py> { + fn key(&mut self, node: &Bound<'py, PyAny>) -> PyResult { + if let Some(identity) = self.identities.get_item(node)? { + return identity.extract(); + } + let identity: usize = self.identities.len(); + self.identities.set_item(node, identity)?; + Ok(identity) + } + + fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { + let suffix: String = if line.repeated { + text(&self.already_shown.call0()?)? + } else { + String::new() + }; + let branch: String = text( + &self + .branch_style + .call1((format!("{}{}", line.prefix, line.branch),))?, + )?; + let node: String = text(&self.format_node.call1((line.node,))?)?; + self.lines.push(format!("{branch}{node}{suffix}")); + Ok(()) + } + + fn children(&mut self, node: &Bound<'py, PyAny>) -> PyResult>> { + let children: Bound<'py, PyAny> = self + .deps + .call_method1("get", (node, PyTuple::empty(node.py())))?; + let kwargs: Bound<'py, PyDict> = PyDict::new(node.py()); + kwargs.set_item("key", &self.sort_key)?; + let sorted: Bound<'py, PyAny> = self.sorted.call((children,), Some(&kwargs))?; + Ok(sorted.downcast::()?.iter().collect()) + } +} + +/// Column-trace edges shown once per related column id, sorted by that id. +struct ColumnTraceTree<'py> { + deps: Bound<'py, PyAny>, + column_id: Bound<'py, PyAny>, + related_column: Bound<'py, PyAny>, + format_related: Bound<'py, PyAny>, + branch_style: Bound<'py, PyAny>, + already_shown: Bound<'py, PyAny>, + last_key: String, + lines: Vec, +} + +impl<'py> ColumnTraceTree<'py> { + fn related_id(&self, edge: &Bound<'py, PyAny>) -> PyResult { + text( + &self + .column_id + .call1((self.related_column.call1((edge,))?,))?, + ) + } + + /// `deps.get(column_id, ())` sorted by each edge's related column id, stably. + fn sorted_edges(&self, column_id: &str) -> PyResult>> { + let py: Python<'py> = self.deps.py(); + let edges: Bound<'py, PyAny> = self + .deps + .call_method1("get", (PyString::new(py, column_id), PyTuple::empty(py)))?; + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for edge in edges.try_iter()? { + let edge: Bound<'py, PyAny> = edge?; + keyed.push((self.related_id(&edge)?, edge)); + } + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, edge)| edge).collect()) + } +} + +impl<'py> TreeVisitor, String, PyErr> for ColumnTraceTree<'py> { + fn key(&mut self, edge: &Bound<'py, PyAny>) -> PyResult { + self.last_key = self.related_id(edge)?; + Ok(self.last_key.clone()) + } + + fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { + let suffix: String = if line.repeated { + text(&self.already_shown.call0()?)? + } else { + String::new() + }; + let branch: String = text(&self.branch_style.call1((line.branch,))?)?; + let related: String = text(&self.format_related.call1((line.node,))?)?; + self.lines + .push(format!("{}{branch}{related}{suffix}", line.prefix)); + Ok(()) + } + + fn children(&mut self, _edge: &Bound<'py, PyAny>) -> PyResult>> { + let key: String = self.last_key.clone(); + self.sorted_edges(&key) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_dependency_tree, module)?)?; + module.add_function(wrap_pyfunction!(render_column_trace_tree, module)?)?; + module.add_function(wrap_pyfunction!(render_column_trace_limit_note, module)?)?; + module.add_function(wrap_pyfunction!(serialize_column, module)?)?; + module.add_function(wrap_pyfunction!(serialize_column_edge, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs new file mode 100644 index 0000000000..a4bfb862b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -0,0 +1,126 @@ +//! `build_manifest`: the dbt-style manifest bytes, whose encoding error is raised on writing. + +use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyBytes; +use pyo3::{FromPyObject, PyRef, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::manifest::errors::{ManifestEncodeError, ManifestError}; +use sqlbuild_analysis::manifest::main::build_manifest_json::build_manifest_json; +use sqlbuild_analysis::manifest::models::ManifestInput; +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::manifest::rows::{ + ProjectRows, plan_input, project_rows, python_hooks, +}; + +const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; + +/// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeManifest { + contents: Result, ManifestEncodeError>, +} + +#[pymethods] +impl NativeManifest { + /// The bytes `json.dumps(manifest, indent=2) + "\n"` encoded, raising what it raised. + fn contents<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.contents { + Ok(bytes) => Ok(PyBytes::new(py, bytes)), + Err(ManifestEncodeError::Unencodable(message)) => { + Err(PyTypeError::new_err(message.clone())) + } + Err(ManifestEncodeError::Emit(JsonEmitError::NestingTooDeep)) => Err( + PyRecursionError::new_err("maximum recursion depth exceeded"), + ), + Err(ManifestEncodeError::Emit(error)) => { + Err(PyValueError::new_err(format!("{error:?}"))) + } + } + } +} + +fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { + match error { + ManifestError::QueryFingerprint(message) => py + .import(FINGERPRINT_EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("QueryFingerprintError")) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + ManifestError::MissingParameterType(name) => PyKeyError::new_err(name), + } +} + +/// The manifest facts only Python knows: names, the generation stamp and hook identities. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ManifestFacts<'py> { + project_name: String, + adapter_type: String, + /// `datetime.now(tz=UTC).isoformat()`. + generated_at: String, + /// `build_hook_identities(project.hook_functions)`. + hook_identities: Bound<'py, PyAny>, +} + +/// The manifest of `project` and the optional plan. +#[pyfunction] +#[pyo3(signature = (*, project, plan_output, graph, facts))] +pub(crate) fn build_manifest( + project: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + graph: PyRef<'_, NativeProjectGraph>, + facts: ManifestFacts<'_>, +) -> PyResult> { + compiler_guard(|| { + let py: Python<'_> = project.py(); + let ManifestFacts { + project_name, + adapter_type, + generated_at, + hook_identities, + } = facts; + let python_hooks = python_hooks(project, &hook_identities)?; + let ProjectRows { + run_id, + sql_analysis_dialect, + models, + sources, + seeds, + macros, + macro_declarations, + sql_hooks, + } = project_rows(project)?; + let input: ManifestInput = ManifestInput { + project_name, + adapter_type, + run_id, + generated_at, + sql_analysis_dialect, + models, + sources, + seeds, + macros, + macro_declarations, + sql_hooks, + python_hooks, + plan: plan_input(plan_output)?, + }; + let graph: &ProjectGraph = graph.graph(); + let contents = py + .detach(|| build_manifest_json(&input, graph)) + .map_err(|error| python_error(error, py))? + .map(String::into_bytes); + Py::new(py, NativeManifest { contents }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_manifest, module)?)?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs new file mode 100644 index 0000000000..c50b252ca8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod builder; +pub(crate) mod registration; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs new file mode 100644 index 0000000000..f2c51d9e32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs @@ -0,0 +1,9 @@ +//! Register the manifest builder. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::manifest::builder; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + builder::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs new file mode 100644 index 0000000000..5d7f5d9752 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs @@ -0,0 +1,484 @@ +//! Read the compiled project, plan output and hook identities into manifest inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::manifest::models::{ + ManifestAuditPlanInput, ManifestColumnInput, ManifestConfigInput, ManifestDynamicColumnInput, + ManifestHookInput, ManifestMacroDeclarationInput, ManifestMacroInput, ManifestModelInput, + ManifestModelSchemaInput, ManifestPathInput, ManifestPlanInput, ManifestPythonHookInput, + ManifestRelationInput, ManifestSeedInput, ManifestSourceInput, ManifestSqlHookInput, + ManifestSqlTestPlanInput, +}; +use sqlbuild_analysis::python_values::models::PythonValue; + +use crate::bindings::_helpers::python_values::values::python_value; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DECIMAL_KIND: &str = "decimal"; +const MACRO_KIND: &str = "macro"; +const DEFAULT_MATERIALIZATION: &str = "view"; + +fn attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attribute(value, name)?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)?.extract() +} + +/// The value as `json.dumps` encodes it. +fn json(value: &Bound<'_, PyAny>) -> PyResult { + python_value(value, false) +} + +fn enum_value(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let member: Bound<'_, PyAny> = attribute(value, name)?; + if member.is_none() { + return Ok(None); + } + member.getattr("value")?.extract().map(Some) +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + attribute(value, name)?.try_iter()?.collect() +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)? + .iter() + .map(PyAnyMethods::extract) + .collect() +} + +/// `relative_path.as_posix()` of an optional path attribute. +fn optional_posix(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let path: Bound<'_, PyAny> = attribute(value, name)?; + if path.is_none() { + return Ok(None); + } + path.call_method0("as_posix")?.extract().map(Some) +} + +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(( + attribute(key, "resource_type")?.str()?.to_string(), + text(key, "name")?, + )) +} + +fn keys(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(object_key).collect() +} + +/// `dict(value)` when `truthy_only` is unset or the value is truthy, as JSON. +fn dict_copy(value: &Bound<'_, PyAny>, truthy_only: bool) -> PyResult> { + if value.is_none() || (truthy_only && !value.is_truthy()?) { + return Ok(None); + } + let copied: Bound<'_, PyAny> = value + .py() + .import("builtins")? + .getattr("dict")? + .call1((value,))?; + json(&copied).map(Some) +} + +fn path(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestPathInput { + text: value.str()?.to_string(), + parent_parts: attribute(&attribute(value, "parent")?, "parts")?.extract()?, + stem: text(value, "stem")?, + }) +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestRelationInput { + database: optional_text(value, "database")?, + schema: optional_text(value, "schema")?, + name: optional_text(value, "name")?, + qualified_name: optional_text(value, "qualified_name")?, + }) +} + +fn column(column: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestColumnInput { + name: text(column, "name")?, + description: optional_text(column, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(column, "meta")?, true)?, + data_type: optional_text(column, "type")?, + }) +} + +fn columns(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(column).collect() +} + +fn hook( + entry: &Bound<'_, PyAny>, + classes: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let (sql_class, python_class) = classes; + if entry.is_instance(sql_class)? { + return Ok(ManifestHookInput::Sql { + statement: text(entry, "statement")?, + name: optional_text(entry, "name")?, + relative_path: optional_posix(entry, "relative_path")?, + definition_sql: optional_text(entry, "definition_sql")?, + kwargs: json(&attribute(entry, "kwargs")?)?, + description: optional_text(entry, "description")?, + }); + } + if entry.is_instance(python_class)? { + return Ok(ManifestHookInput::Python { + name: text(entry, "name")?, + kwargs: json(&attribute(entry, "kwargs")?)?, + }); + } + Ok(ManifestHookInput::Other) +} + +/// The entries when the hooks value is a list or tuple. +fn hooks(value: &Bound<'_, PyAny>) -> PyResult>> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + let module: Bound<'_, PyAny> = value.py().import(DISCOVERY_MODELS_MODULE)?.into_any(); + let sql_class: Bound<'_, PyAny> = module.getattr("SqlHookEntry")?; + let python_class: Bound<'_, PyAny> = module.getattr("PythonHookEntry")?; + value + .try_iter()? + .map(|entry| hook(&entry?, (&sql_class, &python_class))) + .collect::>() + .map(Some) +} + +fn tags(value: &Bound<'_, PyAny>) -> PyResult>> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + value + .try_iter()? + .map(|tag| Ok(tag?.str()?.to_string())) + .collect::>() + .map(Some) +} + +fn config_value<'py>( + values: &Bound<'py, PyAny>, + key: &str, + default: Bound<'py, PyAny>, +) -> PyResult> { + values.call_method1("get", (key, default)) +} + +fn config(values: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = values.py(); + let get = |key: &str| values.call_method1("get", (key,)); + let materialized: Bound<'_, PyAny> = config_value( + values, + "materialized", + PyString::new(py, DEFAULT_MATERIALIZATION).into_any(), + )?; + Ok(ManifestConfigInput { + enabled: json(&config_value( + values, + "enabled", + PyBool::new(py, true).to_owned().into_any(), + )?)?, + alias: json(&get("alias")?)?, + schema: json(&get("schema")?)?, + database: json(&get("database")?)?, + tags: tags(&get("tags")?)?, + materialized: if materialized.is_none() { + DEFAULT_MATERIALIZATION.to_owned() + } else { + materialized.str()?.to_string() + }, + incremental_strategy: json(&get("incremental_strategy")?)?, + merge_exclude_columns: json(&get("merge_exclude_columns")?)?, + full_refresh: json(&get("full_refresh")?)?, + unique_key: json(&get("unique_key")?)?, + on_schema_change: json(&config_value( + values, + "on_schema_change", + PyString::new(py, "ignore").into_any(), + )?)?, + pre_hooks: hooks(&get("pre_hooks")?)?, + post_hooks: hooks(&get("post_hooks")?)?, + }) +} + +fn dynamic_column(family: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestDynamicColumnInput { + name: json(&attribute(family, "name")?)?, + pivot_column: json(&attribute(family, "pivot_column")?)?, + value_column: json(&attribute(family, "value_column")?)?, + aggregate: json(&attribute(family, "aggregate")?)?, + data_type: json(&attribute(family, "type")?)?, + name_pattern: json(&attribute(family, "name_pattern")?)?, + }) +} + +fn model_schema(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + Ok(Some(ManifestModelSchemaInput { + description: optional_text(entry, "description")?, + meta: dict_copy(&attribute(entry, "meta")?, false)?, + columns: columns(entry, "columns")?, + dynamic_columns: items(entry, "dynamic_columns")? + .iter() + .map(dynamic_column) + .collect::>()?, + })) +} + +fn model(model: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestModelInput { + name: text(model, "name")?, + path: path(&attribute(model, "relative_path")?)?, + destination: relation(&attribute(model, "destination")?)?, + query_sql: text(model, "query_sql")?, + config: config(&attribute(&attribute(model, "config")?, "values")?)?, + schema: model_schema(&attribute(model, "schema_entry")?)?, + deps: keys(model, "deps")?, + }) +} + +fn source(source: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(source, "source_entry")?; + let cursor: Bound<'_, PyAny> = attribute(&entry, "cursor_column")?; + let unique_key: Bound<'_, PyAny> = attribute(&entry, "unique_key")?; + Ok(ManifestSourceInput { + name: text(source, "name")?, + path: attribute(&attribute(source, "source_file")?, "relative_path")? + .str()? + .to_string(), + entry_name: text(&entry, "name")?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?.filter(|text| !text.is_empty()), + table: optional_text(&entry, "table")?, + description: optional_text(&entry, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(&entry, "meta")?, true)?, + loader: optional_text(&entry, "loader")?, + write_strategy: enum_value(&entry, "write_strategy")?, + cursor_column: if cursor.is_none() { + None + } else { + Some(json(&cursor)?) + }, + unique_key: if unique_key.is_truthy()? { + Some(json(&unique_key)?) + } else { + None + }, + columns: columns(&entry, "columns")?, + }) +} + +fn seed(seed: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(seed, "schema_entry")?; + Ok(ManifestSeedInput { + name: text(seed, "name")?, + path: path(&attribute(&attribute(seed, "seed_file")?, "relative_path")?)?, + destination: relation(&attribute(seed, "destination")?)?, + tags: texts(&entry, "tags")?, + description: optional_text(&entry, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(&entry, "meta")?, true)?, + columns: columns(&entry, "columns")?, + }) +} + +fn macros(project: &Bound<'_, PyAny>) -> PyResult> { + let loaded: Bound<'_, PyAny> = attribute(project, "loaded_macros")?; + let mut macros: Vec = Vec::new(); + for (key, loaded_macro) in loaded.downcast::()?.iter() { + macros.push(ManifestMacroInput { + key: key.extract()?, + name: text(&loaded_macro, "name")?, + relative_path: optional_posix(&loaded_macro, "relative_path")?.unwrap_or_default(), + raw_source: text(&loaded_macro, "raw_source")?, + }); + } + Ok(macros) +} + +fn macro_declarations(project: &Bound<'_, PyAny>) -> PyResult> { + let scope_index: Bound<'_, PyAny> = attribute(project, "scope_index")?; + if scope_index.is_none() { + return Ok(Vec::new()); + } + let mut declarations: Vec = Vec::new(); + for record in items(&scope_index, "declarations")? { + let identity: Bound<'_, PyAny> = attribute(&record, "identity")?; + if enum_value(&identity, "kind")?.as_deref() != Some(MACRO_KIND) { + continue; + } + let facts: Bound<'_, PyAny> = attribute(&record, "macro")?; + declarations.push(ManifestMacroDeclarationInput { + name: text(&identity, "name")?, + scope: enum_value(&record, "scope")?.unwrap_or_default(), + owning_path: optional_text(&record, "owning_path")?, + dependencies: if facts.is_none() { + None + } else { + Some( + items(&facts, "dependencies")? + .iter() + .map(|dependency| text(dependency, "name")) + .collect::>()?, + ) + }, + }); + } + Ok(declarations) +} + +fn sql_hook(hook: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestSqlHookInput { + name: text(hook, "name")?, + relative_path: optional_posix(hook, "relative_path")?.unwrap_or_default(), + sql_body: text(hook, "sql_body")?, + description: optional_text(hook, "description")?.filter(|text| !text.is_empty()), + }) +} + +/// Each discovered Python hook's path and identity hashes. +pub(crate) fn python_hooks( + project: &Bound<'_, PyAny>, + identities: &Bound<'_, PyAny>, +) -> PyResult> { + items(project, "hook_functions")? + .iter() + .map(|hook| { + let name: String = text(hook, "name")?; + let identity: Bound<'_, PyAny> = identities.get_item(&name)?; + Ok(ManifestPythonHookInput { + relative_path: optional_posix(hook, "relative_path")?.unwrap_or_default(), + definition_hash: text(&identity, "definition_hash")?, + version_hash: text(&identity, "version_hash")?, + name, + }) + }) + .collect() +} + +fn audit_plan(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestAuditPlanInput { + name: text(entry, "name")?, + resolved_sql: text(entry, "resolved_sql")?, + attached_target_name: optional_text(entry, "attached_target_name")?, + attached_column_name: optional_text(entry, "attached_column_name")?, + scope_deps: keys(entry, "scope_deps")?, + }) +} + +fn sql_test_plan(entry: &Bound<'_, PyAny>) -> PyResult { + let mut parameters: Vec<(String, PythonValue)> = Vec::new(); + for pair in items(entry, "parameter_values")? { + let (name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let raw: Bound<'_, PyAny> = attribute(&value, "value")?; + let converted: PythonValue = if enum_value(&value, "kind")?.as_deref() == Some(DECIMAL_KIND) + { + PythonValue::String(raw.str()?.to_string()) + } else { + json(&raw)? + }; + parameters.push((name, converted)); + } + Ok(ManifestSqlTestPlanInput { + key: object_key(&attribute(entry, "key")?)?, + name: text(entry, "name")?, + chain: items(entry, "chain")? + .iter() + .map(|step| Ok((text(step, "model_name")?, text(step, "resolved_sql")?))) + .collect::>()?, + scope_deps: keys(entry, "scope_deps")?, + source_path: optional_posix(entry, "source_path")?, + block_index: json(&attribute(entry, "block_index")?)?, + parent_name: json(&attribute(entry, "parent_name")?)?, + case_name: optional_text(entry, "case_name")?, + case_index: json(&attribute(entry, "case_index")?)?, + case_fingerprint: optional_text(entry, "case_fingerprint")?, + parameter_schema: items(entry, "parameter_schema")? + .iter() + .map(|parameter| { + Ok(( + text(parameter, "name")?, + enum_value(parameter, "value_type")?.unwrap_or_default(), + json(&attribute(parameter, "nullable")?)?, + )) + }) + .collect::>()?, + parameters, + }) +} + +/// The plan entries the manifest reads. +pub(crate) fn plan_input(plan_output: &Bound<'_, PyAny>) -> PyResult> { + if plan_output.is_none() { + return Ok(None); + } + Ok(Some(ManifestPlanInput { + models: items(plan_output, "model_entries")? + .iter() + .map(|entry| Ok((text(entry, "name")?, text(entry, "resolved_sql")?))) + .collect::>()?, + source_load_names: items(plan_output, "source_load_entries")? + .iter() + .map(|entry| text(entry, "name")) + .collect::>()?, + audits: items(plan_output, "audit_entries")? + .iter() + .map(audit_plan) + .collect::>()?, + sql_tests: items(plan_output, "test_entries")? + .iter() + .map(sql_test_plan) + .collect::>()?, + })) +} + +/// The project's models, sources, seeds, macros and hooks. +pub(crate) struct ProjectRows { + pub(crate) run_id: String, + pub(crate) sql_analysis_dialect: Option, + pub(crate) models: Vec, + pub(crate) sources: Vec, + pub(crate) seeds: Vec, + pub(crate) macros: Vec, + pub(crate) macro_declarations: Vec, + pub(crate) sql_hooks: Vec, +} + +/// Read the compiled project. +pub(crate) fn project_rows(project: &Bound<'_, PyAny>) -> PyResult { + Ok(ProjectRows { + run_id: text(project, "run_id")?, + sql_analysis_dialect: optional_text(project, "sql_analysis_dialect")?, + models: items(project, "models")? + .iter() + .map(model) + .collect::>()?, + sources: items(project, "sources")? + .iter() + .map(source) + .collect::>()?, + seeds: items(project, "seeds")? + .iter() + .map(seed) + .collect::>()?, + macros: macros(project)?, + macro_declarations: macro_declarations(project)?, + sql_hooks: items(project, "sql_hook_files")? + .iter() + .map(sql_hook) + .collect::>()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 70c9e1eb58..8e36750bba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -7,11 +7,15 @@ pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod contracts; +pub(crate) mod dag_artifact; pub(crate) mod discovery; +pub(crate) mod fingerprints; pub(crate) mod graph; pub(crate) mod lineage; +pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod project_assembly; +pub(crate) mod python_values; pub(crate) mod refactor; pub(crate) mod render; pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs new file mode 100644 index 0000000000..4b9235503b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs @@ -0,0 +1 @@ +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs new file mode 100644 index 0000000000..7ec29a1a92 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs @@ -0,0 +1,144 @@ +//! Python values as `json.dumps` would encode them, optionally after `dataclasses.asdict`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, + PyString, PyStringMethods, PyTuple, PyTypeMethods, +}; +use sqlbuild_analysis::python_values::models::{PythonKey, PythonValue}; +use sqlbuild_core::json::models::JsonInteger; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_outputs::json_report::float_key; + +const DATACLASS_FIELDS: &str = "__dataclass_fields__"; +const SURROGATE_KEY: &str = "a JSON artifact mapping key holds a lone surrogate, which the native \ + encoder does not write"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +fn type_name(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.get_type().name()?.to_string()) +} + +/// A `str`, as UTF-16 code units when it holds lone surrogates. +pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { + if let Ok(text) = text.to_str() { + return Ok(PythonValue::String(text.to_owned())); + } + let encoded: Bound<'_, PyBytes> = text + .call_method1("encode", ("utf-16-le", "surrogatepass"))? + .downcast_into::()?; + Ok(PythonValue::Utf16( + encoded + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + )) +} + +fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(signed) = value.extract::() { + return Ok(PythonValue::Integer(JsonInteger::from(signed))); + } + if let Ok(unsigned) = value.extract::() { + return Ok(PythonValue::Integer(JsonInteger::from(unsigned))); + } + let decimal: String = value.str()?.to_str()?.to_owned(); + Ok(JsonInteger::parse(&decimal).map_or(PythonValue::Null, PythonValue::Integer)) +} + +/// A mapping key as `json.dumps` writes it, or its `TypeError` message. +fn key_value(key: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = key.downcast::() { + return text + .to_str() + .map(|text| PythonKey { + text: Ok(text.to_owned()), + is_str: true, + }) + .map_err(|_| NativeCompilerError::new_err(SURROGATE_KEY)); + } + Ok(PythonKey { + text: scalar_key_text(key)?, + is_str: false, + }) +} + +fn scalar_key_text(key: &Bound<'_, PyAny>) -> PyResult> { + if key.is_none() { + return Ok(Ok("null".to_owned())); + } + if let Ok(flag) = key.downcast::() { + return Ok(Ok(if flag.is_true() { "true" } else { "false" }.to_owned())); + } + if key.is_instance_of::() { + return Ok(Ok(key.str()?.to_str()?.to_owned())); + } + if key.is_instance_of::() { + return float_key(key).map(Ok); + } + Ok(Err(format!( + "keys must be str, int, float, bool or None, not {}", + type_name(key)? + ))) +} + +fn dataclass_value(value: &Bound<'_, PyAny>) -> PyResult { + let fields: Bound<'_, PyAny> = value + .py() + .import(DATACLASSES_MODULE)? + .getattr("fields")? + .call1((value,))?; + let mut entries: Vec<(PythonKey, PythonValue)> = Vec::new(); + for field in fields.try_iter()? { + let name: Bound<'_, PyAny> = field?.getattr("name")?; + let item: Bound<'_, PyAny> = value.getattr(name.downcast::()?)?; + entries.push((key_value(&name)?, python_value(&item, true)?)); + } + Ok(PythonValue::Object(entries)) +} + +/// The value `json.dumps` encodes, copied by `asdict` first when `dataclasses` is set. +pub(crate) fn python_value(value: &Bound<'_, PyAny>, dataclasses: bool) -> PyResult { + if value.is_none() { + return Ok(PythonValue::Null); + } + if dataclasses && value.get_type().hasattr(DATACLASS_FIELDS)? { + return dataclass_value(value); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(PythonValue::Array( + value + .try_iter()? + .map(|item| python_value(&item?, dataclasses)) + .collect::>()?, + )); + } + if let Ok(mapping) = value.downcast::() { + return Ok(PythonValue::Object( + mapping + .iter() + .map(|(key, item)| Ok((key_value(&key)?, python_value(&item, dataclasses)?))) + .collect::>()?, + )); + } + if let Ok(flag) = value.downcast::() { + return Ok(PythonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return integer_value(value); + } + if value.is_instance_of::() { + return Ok(PythonValue::Float(value.extract::()?)); + } + if let Ok(text) = value.downcast::() { + return text_value(text); + } + Ok(PythonValue::Unencodable(format!( + "Object of type {} is not JSON serializable", + type_name(value)? + ))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs index 19f416b5bb..432cf39b99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod contract_edits; +pub(crate) mod python_strings; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs new file mode 100644 index 0000000000..9299b29f42 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs @@ -0,0 +1,31 @@ +//! Python string constants that name a renamed model or column, for `sqb rename` and `sqb mv`. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::string_literals::main::string_literal_locations::string_literal_locations; +use sqlbuild_pysource::string_literals::models::StringQuery; + +/// The 1-based `(line, column)` of each flagged string constant; none when `contents` does not parse. +#[pyfunction] +#[pyo3(signature = (contents, names, context))] +fn python_string_locations( + py: Python<'_>, + contents: &str, + names: Vec, + context: Option<&str>, +) -> Vec<(usize, usize)> { + let version = py.version_info(); + string_literal_locations( + contents, + &StringQuery { + names: &names, + context, + }, + (version.major, version.minor), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(python_string_locations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs index fa4ab9dc4a..cc2da0c85a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs @@ -144,5 +144,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(with_model_migration_json, module)?)?; module.add_function(wrap_pyfunction!(commit_refactor_plan_json, module)?)?; crate::bindings::_helpers::refactor::contract_edits::register(module)?; + crate::bindings::_helpers::refactor::python_strings::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs new file mode 100644 index 0000000000..cf4ea638ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -0,0 +1,70 @@ +//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_sqltext::code_scanner::main::code_scanner::code_scanner; +use sqlbuild_sqltext::code_scanner::main::scanner_matching_paren::scanner_matching_paren; +use sqlbuild_sqltext::code_scanner::main::scanner_non_code_end::scanner_non_code_end; +use sqlbuild_sqltext::code_scanner::models::CodeScanner; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; + +/// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SqlCodeScanner { + scanner: CodeScanner, + context: String, +} + +#[pymethods] +impl SqlCodeScanner { + #[new] + #[pyo3(signature = (sql, syntax, context))] + fn new(sql: &str, syntax: Option, context: String) -> Self { + Self { + scanner: code_scanner(sql, syntax.map(Into::into)), + context, + } + } + + /// The end of the comment or quoted text at `index`, or `None` when code starts there. + fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { + scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) + } + + /// The index of the parenthesis closing the one at `open`. + fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { + scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) + } +} + +impl SqlCodeScanner { + /// `CompileInputError(" contains an unclosed ...")`. + fn error(&self, py: Python<'_>, unclosed: Unclosed) -> PyErr { + let construct: &str = match unclosed { + Unclosed::BlockComment => "block comment", + Unclosed::Quote => "quoted string", + Unclosed::Parenthesis => "parenthesis", + }; + let message: String = format!("{} contains an unclosed {construct}", self.context); + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 4cd903b971..52ccc56e39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod authored_values; +pub(crate) mod code_scanner; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; pub(crate) mod model_headers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 4fe4b866e4..d94a7da68f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - declaration_references, model_headers, placeholders, sql_references, static_sql, + code_scanner, declaration_references, model_headers, placeholders, sql_references, static_sql, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_headers::register(module)?; + code_scanner::register(module)?; static_sql::register(module)?; sql_references::register(module)?; declaration_references::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 98e4ac6e68..86bb63b8e9 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,8 +5,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, discovery, graph, lineage, model_config, project_assembly, - refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, type_system, + compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, lineage, manifest, + model_config, project_assembly, refactor, render, rules, scopes, semantic_checks, + sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -51,11 +52,14 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { semantic_checks::registration::register(module)?; contracts::registration::register(module)?; lineage::registration::register(module)?; + fingerprints::registration::register(module)?; sql_test_glue::registration::register(module)?; project_assembly::registration::register(module)?; compile_outputs::registration::register(module)?; graph::registration::register(module)?; compiled_project::registration::register(module)?; compile_driver::registration::register(module)?; + dag_artifact::registration::register(module)?; + manifest::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index d0b0304e8d..4a3925a674 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -98,37 +98,6 @@ pub(crate) struct NativeProjectTree { pub(crate) exported: Mutex>, } -/// One native model discovery request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct ModelDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) supported_keys: Vec, - pub(crate) removed_keys: Vec, - pub(crate) extract_implicit_alias_columns: bool, - pub(crate) extract_output_column_locations: bool, -} - -/// One native SQL test or scenario discovery request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) test_keys: Vec, - pub(crate) scenario_keys: Vec, -} - -/// One in-memory SQL test or scenario parse request; `file_path` is the path messages name. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestTextRequest { - pub(crate) file_path: String, - pub(crate) test_keys: Vec, - pub(crate) scenario_keys: Vec, -} - /// One native YAML file discovery request; `kind` is `source` or `schema`. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] @@ -156,17 +125,6 @@ pub(crate) struct NativeDiscoverySession { pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, } -/// One native declaration discovery session request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct DeclarationDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) function_keys: Vec, - pub(crate) audit_keys: Vec, - pub(crate) hook_keys: Vec, -} - /// One in-memory declaration file parse request; `kind` names the collection it belongs to. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs new file mode 100644 index 0000000000..39b1b3ad13 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs @@ -0,0 +1,23 @@ +//! The compiler's generic comment and quoted-text rules without dialect information. + +use crate::sql_scan::main::comment_end::comment_end; +use crate::sql_scan::main::non_code_end::non_code_end; +use crate::sql_scan::models::{QuotePolicy, Unclosed}; + +/// Python's generic comment or quoted-text end at `index`; backticks never double, a lone `$` is one character. +pub(crate) fn generic_skip_end(sql: &[u8], index: usize) -> Result, Unclosed> { + match sql.get(index) { + Some(b'`') => sql[index + 1..] + .iter() + .position(|byte| *byte == b'`') + .map(|offset| Some(index + 1 + offset + 1)) + .ok_or(Unclosed::Quote), + Some(b'$') => Ok(Some( + non_code_end(sql, index, QuotePolicy::COMPILER)?.unwrap_or(index + 1), + )), + _ => match comment_end(sql, index)? { + Some(end) => Ok(Some(end)), + None => non_code_end(sql, index, QuotePolicy::COMPILER), + }, + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs new file mode 100644 index 0000000000..a54aea47b0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod generic; +pub(crate) mod offsets; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs new file mode 100644 index 0000000000..faee9bec80 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs @@ -0,0 +1,23 @@ +//! Conversions between Python code point offsets and UTF-8 byte offsets. + +use crate::code_scanner::models::CodeScanner; + +/// The byte offset of the code point at `index`, or the text length past the end. +pub(crate) fn byte_offset(scanner: &CodeScanner, index: usize) -> usize { + if scanner.char_bytes.is_empty() { + return index.min(scanner.sql.len()); + } + scanner + .char_bytes + .get(index) + .copied() + .unwrap_or(scanner.sql.len()) +} + +/// The code point offset of a byte offset at a code point boundary. +pub(crate) fn char_offset(scanner: &CodeScanner, byte: usize) -> usize { + if scanner.char_bytes.is_empty() { + return byte; + } + scanner.char_bytes.partition_point(|start| *start < byte) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs new file mode 100644 index 0000000000..4049206d36 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs @@ -0,0 +1,22 @@ +//! Prepare one SQL text for scanning at Python string offsets. + +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::models::LexicalSyntax; + +/// A scanner over `sql`, under `syntax` when given and the generic compiler rules otherwise. +#[must_use] +pub fn code_scanner(sql: &str, syntax: Option) -> CodeScanner { + let char_bytes: Vec = if sql.is_ascii() { + Vec::new() + } else { + sql.char_indices() + .map(|(offset, _)| offset) + .chain([sql.len()]) + .collect() + }; + CodeScanner { + sql: sql.to_owned(), + char_bytes, + syntax, + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs new file mode 100644 index 0000000000..60ffc7f155 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs @@ -0,0 +1,3 @@ +pub mod code_scanner; +pub mod scanner_matching_paren; +pub mod scanner_non_code_end; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs new file mode 100644 index 0000000000..f708777596 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs @@ -0,0 +1,39 @@ +//! The parenthesis closing one opened at a Python string offset. + +use crate::code_scanner::_helpers::generic::generic_skip_end; +use crate::code_scanner::_helpers::offsets::{byte_offset, char_offset}; +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use crate::sql_scan::models::Unclosed; + +/// The code point offset of the parenthesis closing the one at `open`, skipping text and comments. +pub fn scanner_matching_paren(scanner: &CodeScanner, open: usize) -> Result { + let sql: &[u8] = scanner.sql.as_bytes(); + let start: usize = byte_offset(scanner, open); + let close: usize = match &scanner.syntax { + Some(syntax) => dialect_matching_paren(sql, start, syntax)?, + None => generic_matching_paren(sql, start)?, + }; + Ok(char_offset(scanner, close)) +} + +fn generic_matching_paren(sql: &[u8], open: usize) -> Result { + let mut depth: usize = 1; + let mut index: usize = open + 1; + while index < sql.len() { + if let Some(end) = generic_skip_end(sql, index)? { + index = end; + continue; + } + if sql[index] == b'(' { + depth += 1; + } else if sql[index] == b')' { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + index += 1; + } + Err(Unclosed::Parenthesis) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs new file mode 100644 index 0000000000..7de499bf89 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs @@ -0,0 +1,21 @@ +//! The end of the comment or quoted text at one Python string offset. + +use crate::code_scanner::_helpers::generic::generic_skip_end; +use crate::code_scanner::_helpers::offsets::{byte_offset, char_offset}; +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use crate::sql_scan::models::Unclosed; + +/// Where the comment or quoted text at code point `index` ends; a generic lone `$` is one character. +pub fn scanner_non_code_end( + scanner: &CodeScanner, + index: usize, +) -> Result, Unclosed> { + let sql: &[u8] = scanner.sql.as_bytes(); + let start: usize = byte_offset(scanner, index); + let end: Option = match &scanner.syntax { + Some(syntax) => dialect_non_code_end(sql, start, syntax)?, + None => generic_skip_end(sql, start)?, + }; + Ok(end.map(|end| char_offset(scanner, end))) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/models.rs b/crates/sqlbuild-sqltext/src/code_scanner/models.rs new file mode 100644 index 0000000000..90e68df56c --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/models.rs @@ -0,0 +1,13 @@ +//! One SQL text scanned for code at Python string offsets. + +use crate::sql_scan::models::LexicalSyntax; + +/// A SQL text, its code point byte offsets and the dialect rules it is scanned under, if any. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CodeScanner { + pub(crate) sql: String, + /// The byte offset of each code point and then the text length; empty for ASCII text. + pub(crate) char_bytes: Vec, + /// Dialect rules; without them the compiler's generic quoting and comments apply. + pub(crate) syntax: Option, +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests.rs new file mode 100644 index 0000000000..a046ce3699 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_code_scanner.rs"] +mod code_scanner; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs new file mode 100644 index 0000000000..efc9231d16 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs @@ -0,0 +1,11 @@ +//! Lexical syntax shared by the code scanner tests. + +use crate::sql_scan::models::LexicalSyntax; + +pub(crate) fn dialect() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_string(), "#".to_string()], + nested_block_comments: true, + ..LexicalSyntax::default() + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs new file mode 100644 index 0000000000..8c8914d9d1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs @@ -0,0 +1,139 @@ +use crate::code_scanner::main::code_scanner::code_scanner; +use crate::code_scanner::main::scanner_matching_paren::scanner_matching_paren; +use crate::code_scanner::main::scanner_non_code_end::scanner_non_code_end; +use crate::code_scanner::tests::helpers::dialect; +use crate::code_scanner::tests::test_types::{MatchingParenTestCase, NonCodeEndTestCase}; +use crate::sql_scan::models::Unclosed; + +/// Expectations recorded from the Python `sql_analysis` scanners this replaces. +#[test] +fn given_sql_when_finding_non_code_end_then_matches_python_offsets() { + let test_cases = [ + NonCodeEndTestCase { + description: "generic backticks end at the first closing backtick", + sql: "`a``b` x", + syntax: None, + index: 0, + expected_end: Ok(Some(3)), + }, + NonCodeEndTestCase { + description: "dialect backticks double to escape", + sql: "`a``b` x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(Some(6)), + }, + NonCodeEndTestCase { + description: "a generic lone dollar is one character", + sql: "$1 x", + syntax: None, + index: 0, + expected_end: Ok(Some(1)), + }, + NonCodeEndTestCase { + description: "a dialect lone dollar is code", + sql: "$1 x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(None), + }, + NonCodeEndTestCase { + description: "offsets count code points", + sql: "é 'naïve' x", + syntax: None, + index: 2, + expected_end: Ok(Some(9)), + }, + NonCodeEndTestCase { + description: "a dialect line comment prefix", + sql: "é # note\nx", + syntax: Some(dialect()), + index: 2, + expected_end: Ok(Some(9)), + }, + NonCodeEndTestCase { + description: "nested block comments", + sql: "/* a /* b */ c */ x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(Some(17)), + }, + NonCodeEndTestCase { + description: "generic block comments do not nest", + sql: "/* a /* b */ c */ x", + syntax: None, + index: 0, + expected_end: Ok(Some(12)), + }, + NonCodeEndTestCase { + description: "an unclosed dollar quote", + sql: "$tag$ never", + syntax: None, + index: 0, + expected_end: Err(Unclosed::Quote), + }, + NonCodeEndTestCase { + description: "an unclosed block comment", + sql: "/* never", + syntax: Some(dialect()), + index: 0, + expected_end: Err(Unclosed::BlockComment), + }, + NonCodeEndTestCase { + description: "code", + sql: "select 1", + syntax: None, + index: 0, + expected_end: Ok(None), + }, + ]; + for test_case in test_cases { + let scanner = code_scanner(test_case.sql, test_case.syntax); + assert_eq!( + scanner_non_code_end(&scanner, test_case.index), + test_case.expected_end, + "{}", + test_case.description + ); + } +} + +/// Expectations recorded from the Python `find_matching_paren` this replaces. +#[test] +fn given_open_parenthesis_when_matching_then_skips_text_and_comments() { + let test_cases = [ + MatchingParenTestCase { + description: "nested parentheses", + sql: "f(a, (b), c) x", + open: 1, + expected_close: Ok(11), + }, + MatchingParenTestCase { + description: "parentheses in quotes, comments and backticks", + sql: "f(')', `)`, -- )\n /* ) */ $$)$$) x", + open: 1, + expected_close: Ok(31), + }, + MatchingParenTestCase { + description: "code point offsets", + sql: "ééf('é)') x", + open: 3, + expected_close: Ok(8), + }, + MatchingParenTestCase { + description: "an unclosed parenthesis", + sql: "f(a", + open: 1, + expected_close: Err(Unclosed::Parenthesis), + }, + ]; + for test_case in test_cases { + let scanner = code_scanner(test_case.sql, None); + assert_eq!( + scanner_matching_paren(&scanner, test_case.open), + test_case.expected_close, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs new file mode 100644 index 0000000000..20bcc9e52c --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +pub(super) struct NonCodeEndTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) syntax: Option, + pub(super) index: usize, + pub(super) expected_end: Result, Unclosed>, +} + +pub(super) struct MatchingParenTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) open: usize, + pub(super) expected_close: Result, +} diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs index 74de9d7d28..8e5ea36cbe 100644 --- a/crates/sqlbuild-sqltext/src/lib.rs +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod code_scanner; pub mod compiler; mod constants; pub mod sql_references; diff --git a/fensu.toml b/fensu.toml index de395eefcc..a4d7b2e642 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,6 +33,11 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/dag/types.py" +reason = "The native DAG builder owns DAG behavior; this package only declares the node kinds integrations read from the artifact." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/adapters/bigquery/__init__.py" @@ -68,6 +73,11 @@ rule = "FFR309" path = "src/sqlbuild/adapters/sqlserver/__init__.py" reason = "The SQL Server backend intentionally owns behavior through its adapter class rather than function entrypoints." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/lineage/__init__.py" +reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/python-boundary.toml b/python-boundary.toml index 53b2f8ecfa..3bef235801 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 137187 +remaining_lines = 130215 [[stays]] path = "executor" @@ -204,6 +204,14 @@ path = "tasks.py" reason = "public-api" note = "Public facade." +[[handled_native_errors]] +location = "compiler/discovery/_helpers/filesystem/aggregation.py::_discover_declarations" +note = "Tolerant scope discovery records a failing declaration category as a fault; no Python implementation runs." + +[[handled_native_errors]] +location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" +note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." + [[handled_native_errors]] location = "compiler/macro_bridge/_helpers/store_keys.py::macro_store_token" note = "A macro without a stable key is not stored; no Python implementation runs." @@ -223,3 +231,19 @@ note = "An undecodable cached finding is a cache miss; the rules run again nativ [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." + +[[handled_native_errors]] +location = "cli/commands/_helpers/cost/refresh.py::refresh_pending_cost_run" +note = "Best-effort cost refresh keeps its existing catch-all; discovery is now entered through the native driver, and no Python implementation runs instead." + +[[handled_native_errors]] +location = "compiler/scopes/main/load_or_build_scope_index.py::load_or_build_scope_index" +note = "Project input errors from discovery and compile-input stages select the tolerant scope index as before; a native compiler failure is a RuntimeError and is not caught." + +[[handled_native_errors]] +location = "lint/_helpers/fixes.py::_FixSession._relation_keys" +note = "A baseline that does not compile yields no relation keys, as before the compile entry became native; no Python implementation runs instead." + +[[handled_native_errors]] +location = "lint/_helpers/fixes.py::_FixSession._verdict" +note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index b3fe3bae74..e8d00f93b9 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -19,7 +19,6 @@ ) ALLOWED_MACRO_LOAD_PATHS: frozenset[str] = frozenset( { - "src/sqlbuild/compiler/compile/main/_build_compile_inputs.py", "src/sqlbuild/compiler/compile/main/sql_expansion_context.py", "src/sqlbuild/compiler/compile/_helpers/render/macros.py", } @@ -278,7 +277,6 @@ "src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py", "src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py", "src/sqlbuild/compiler/discovery/_helpers/yml/project.py", - "src/sqlbuild/compiler/discovery/main/discover.py", "src/sqlbuild/compiler/discovery/main/runtime_extensions.py", "src/sqlbuild/compiler/discovery/models.py", "src/sqlbuild/provider/_helpers/session.py", diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 75f68074e5..69bc123b1f 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -170,7 +170,7 @@ def source_freshness_sql_ownership(*, module: object, ctx: RuleContext) -> list[ code="XSB062", family=Family.CUSTOM, slug="single-macro-load-site", - message="project macros must be loaded once in build_compile_inputs", + message="project macros must be loaded once by the native compile driver", remediation="Pass loaded_macros down instead of calling load_project_macros again.", ) def single_macro_load_site(*, module: object, ctx: RuleContext) -> list[Fault]: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4ee7605459..6b8d8c4dd1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,10 +1,13 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Hashable, Mapping, Sequence, Set from pathlib import Path, PurePath from typing import Any, Literal, TypedDict, overload +from _typeshed import SupportsRichComparison + from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import CompileAnalysis @@ -13,6 +16,7 @@ from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, CompileAnalysisSelection, CompileAuditInput, CompiledAudit, @@ -52,11 +56,16 @@ from sqlbuild.compiler.compile.types import ( ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, + DiscoveredAdapterFile, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredConstantFile, + DiscoveredEnumFile, DiscoveredHookFunction, DiscoveredLoaderFunction, + DiscoveredMacroFile, DiscoveredMaterializationFile, + DiscoveredModelSchemaFile, DiscoveredProjectInputs, DiscoveredPythonFunctionFile, DiscoveredSchemaFile, @@ -65,13 +74,29 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, DiscoveredSqlTestFile, + DiscoveryFileFault, EnumDeclaration, ModelSchemaDeclaration, TolerantScopeDiscovery, ) +from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.lineage.models import ( + ColumnLineageEdge, + DirectSemanticColumnUse, + ModelColumnLineage, + QualifiedLineageColumn, +) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability -from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.pipeline.models import ( + CompiledProjectPhaseResult, + CompilePipelineOptions, + CompilePipelineResult, + ProjectGraph, +) +from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -102,6 +127,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -782,6 +808,26 @@ def relation_key_row( kind: str, name: str, values: dict[str, object], entry: object ) -> RelationKeyRow: ... def declared_relation_keys(rows: list[RelationKeyRow]) -> dict[str, list[list[str]]]: ... +def build_dag_json( + *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None +) -> str: ... + +class _ManifestFacts(TypedDict): + project_name: str + adapter_type: str + generated_at: str + hook_identities: Mapping[str, PythonNodeIdentity] + +class NativeManifest: + def contents(self) -> bytes: ... + +def build_manifest( + *, + project: CompiledProjectView, + plan_output: PlanOutput | None, + graph: NativeProjectGraph, + facts: _ManifestFacts, +) -> NativeManifest: ... class NativeProjectGraph: @staticmethod @@ -870,7 +916,6 @@ def model_header_failure( class NativeProjectTree: def __init__(self, project_dir: str) -> None: ... - def listings(self) -> list[tuple[str, list[tuple[str, bool, bool]]]]: ... class DiscoveredSqlModelFile: """A discovered SQL model file and its raw contents.""" @@ -941,6 +986,15 @@ def build_compiled_project( resolved_connection: dict[str, object] | None = None, analysis_selection: CompileAnalysisSelection | None = None, ) -> CompiledProjectView: ... +def assemble_project( + *, + inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile | None = None, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + analysis_cache_dir: Path | None = None, + analysis_model_names: frozenset[str] | None = None, +) -> CompiledProjectView: ... def build_project_graph_with_analysis_selection( *, discovered_inputs: DiscoveredProjectInputs, @@ -967,6 +1021,47 @@ def build_project_graph( on_progress: Callable[[str], None] | None = None, no_cache: bool = False, ) -> ProjectGraph: ... +def discover_project_inputs( + *, + project_dir: Path, + sql_analysis_enabled_override: bool | None = None, + extract_output_column_locations: bool = True, +) -> DiscoveredProjectInputs: ... +def build_discovered_project_inputs( + *, + project_dir: Path, + project_config: ProjectConfig, + local_config: LocalConfig, + sql_analysis_enabled: bool, + extract_output_column_locations: bool = True, +) -> DiscoveredProjectInputs: ... +def compile_project_phase( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions | None = None, + on_progress: Callable[[str], None] | None = None, +) -> CompiledProjectPhaseResult: ... +def run_compile_pipeline( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions | None = None, + hooks: ConnectionHooks | None = None, +) -> CompilePipelineResult: ... +def build_compile_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter_context: CompileAdapterContext, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, + run_id: str | None = None, + no_sql_validation: bool = False, + defer_model_sql_validation: bool = False, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + resolved_connection: dict[str, object] | None = None, + no_cache: bool = False, +) -> CompileInputsView: ... def run_compile_command( request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None ) -> int: ... @@ -1006,45 +1101,83 @@ def public_declaration_indexes( def public_model_schema_index( inputs: DiscoveredProjectInputs, ) -> dict[str, ModelSchemaDeclaration]: ... -def discovered_inputs_failure( - inputs: DiscoveredProjectInputs, nodes: list[dict[str, object]] -) -> tuple[object, ...] | None: ... +def validate_discovered_project_inputs( + inputs: DiscoveredProjectInputs, nodes: list[dict[str, object]], / +) -> None: ... def discover_model_files( - request: dict[str, object], tree: NativeProjectTree, project_dir: Path -) -> list[tuple[Path, object]] | tuple[object, ...]: ... -def discover_sql_test_files( - request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... + *, + project_dir: Path, + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, + selected_model_names: Set[str] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, +) -> tuple[DiscoveredSqlModelFile, ...]: ... +def discover_test_files( + *, + project_dir: Path, + selected_paths: Set[Path] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, +) -> tuple[DiscoveredSqlTestFile, ...]: ... def discover_scenario_files( - request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlScenarioFile, ...]: ... +def parse_sql_test_file( + *, contents: str, file_path: Path +) -> tuple[DiscoveredSqlTestBlock, ...]: ... +def parse_sql_scenario_file( + *, contents: str, file_path: Path, relative_path: Path +) -> DiscoveredSqlScenarioFile: ... +def discover_source_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSourceFile, ...]: ... +def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: ... def load_yaml_files( request: dict[str, object], relative_paths: list[str], tree: NativeProjectTree -) -> list[tuple[object, ...]] | tuple[object, ...]: ... -def load_yaml_document(file_path: str, text: str, kind: str) -> tuple[object, ...]: ... - -class NativeDiscoverySession: - def __init__(self, request: dict[str, object]) -> None: ... - def collection( - self, kind: str, tree: NativeProjectTree, isolate_kind: bool = False - ) -> ( - list[ - tuple[ - str, tuple[str, str, str | None, str | None, str | None] | None, tuple[object, ...] - ] - ] - | tuple[object, ...] - ): ... +) -> list[tuple[str, object] | Exception]: ... +def load_yaml_document(file_path: str, text: str, kind: str) -> object: ... +def _oracle_read_error( + errno: int | None, winerror: int | None, message: str, *, file_path: Path +) -> Exception: ... +def discover_enum_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, + isolate_declaration_kind: bool = False, +) -> tuple[DiscoveredEnumFile, ...]: ... +def discover_constant_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, + isolate_declaration_kind: bool = False, +) -> tuple[DiscoveredConstantFile, ...]: ... +def discover_model_schema_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredModelSchemaFile, ...]: ... +def discover_sql_function_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlFunctionFile, ...]: ... +def discover_sql_hook_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlHookFile, ...]: ... +def discover_audit_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredAuditFile, ...]: ... +def discover_macro_files( + *, project_dir: Path, isolate_declaration_kind: bool = False +) -> tuple[DiscoveredMacroFile, ...]: ... +def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: ... +def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: ... +def retained_discovery_session(*, project_dir: Path) -> NativeDiscoverySession | None: ... +def declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: ... + +class NativeDiscoverySession: ... def parse_declaration_contents( request: dict[str, object], names: tuple[str, str], contents: str -) -> tuple[object, ...]: ... -def discover_declaration_layout( - tree: NativeProjectTree, kind: str | None = None -) -> tuple[tuple[object, ...], tuple[object, ...]] | tuple[object, ...]: ... -def parse_model_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... -def parse_sql_test_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... -def parse_scenario_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... +) -> str: ... +def parse_model_contents( + request: dict[str, object], contents: str +) -> tuple[dict[str, object], str, dict[str, SourceLocation]]: ... class NativeDeclarationContexts: def __init__( @@ -1165,21 +1298,9 @@ def unresolved_reference_marker_kind( def normalize_typed_sql_value( raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / ) -> object: ... -def sql_test_blocks( - file_path: str, blocks: list[tuple[dict[str, object], str, tuple[int, int]]], / -) -> list[object]: ... -def validate_sql_test_block_names( - file_path: str, blocks: list[tuple[int, str | None]], / -) -> None: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... -def typed_constant_record( - constant: tuple[str, object], - options: tuple[str | None, str | None], - paths: tuple[Path, Path], - /, -) -> object: ... def deferred_target_segments( template: str, logical: str, context_keys: list[str], variables: list[str], / ) -> list[tuple[bool, str]]: ... @@ -1191,6 +1312,30 @@ def resource_sql_diagnostic_rows(catalog: object, request: object, /) -> list[ob def compiler_query_hash(sql: str, dialect: str | None, /) -> str: ... def function_definition_hash(sql: str, language: str, dialect: str | None, /) -> str: ... def normalize_query_text(sql: str, /) -> str: ... +def fingerprint_create_table_sql(qualified_name: str, column_types: tuple[str, str], /) -> str: ... +def fingerprint_read_latest_sql(qualified_name: str, /) -> str: ... +def fingerprint_relevant_read_sql( + read_sql: str, + node_names: Sequence[str] | None, + filtered_node_types: Sequence[str], + /, +) -> str | None: ... +def fingerprint_insert_sql(qualified_name: str, fingerprint: Fingerprint, /) -> str: ... +def fingerprint_insert_batches( + qualified_name: str, + fingerprints: Sequence[Fingerprint], + limits: tuple[int, int], + /, +) -> list[tuple[tuple[Fingerprint, ...], str]]: ... +def fingerprint_set_from_rows( + schema: str, qualified_name: str, rows: Sequence[Sequence[object]], / +) -> FingerprintSet: ... +def select_latest_fingerprints( + fingerprint_set: FingerprintSet, + node_names: Sequence[str], + filtered_node_types: Sequence[str], + /, +) -> FingerprintSet: ... def glob_name_match(name: str, pattern: str, ignore_case: bool, /) -> bool: ... def lone_surrogate_site(raw_values: dict[str, object], /) -> tuple[str, str, int] | None: ... def environment_template_names(values: list[str | None], /) -> list[str]: ... @@ -1209,6 +1354,9 @@ def contract_model_edit( def contract_source_edit( row: tuple[str, str, str, list[tuple[str, str | None]], list[tuple[str, str]], bool], / ) -> tuple[str | None, str | None, str | None]: ... +def python_string_locations( + contents: str, names: list[str], context: str | None +) -> list[tuple[int, int]]: ... def resource_identity_suggestion( name: str, private_identity: bool, / ) -> tuple[str, str] | None: ... @@ -1295,6 +1443,11 @@ class NativeModelValidator: self, model: tuple[str, str], references: list[tuple[str, str, bool]] ) -> int | NativeConfigError | None: ... +class SqlCodeScanner: + def __init__(self, sql: str, syntax: dict[str, object] | None, context: str) -> None: ... + def non_code_end(self, index: int) -> int | None: ... + def matching_paren(self, open: int) -> int: ... + class SqlReferenceScanner: def __init__(self, syntax: dict[str, object]) -> None: ... def extract( @@ -1621,26 +1774,86 @@ def evaluate_project_model_contracts( def project_promotion_conflicts( settings: tuple[str | None, str, str], project: NativeCompiledProject ) -> list[tuple[str, str, str, str, str]]: ... -def build_fast_column_lineage( - catalog: object | None, - request: tuple[str | None, list[tuple[str, str, list[str]]], list[tuple[bool, str, list[str]]]], +def build_project_column_lineage( + project: CompiledProjectView, + dialect: str | None, + mode: ColumnLineageMode, + model_names: frozenset[str] | None, /, -) -> list[tuple[str, list[tuple[str, str, str, list[tuple[str, str, str]]]], bool, str | None]]: ... +) -> ProjectColumnLineage | None: ... def build_direct_semantic_uses( - schema: list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], - models: list[tuple[str, str, str]], - dialect: str, -) -> list[tuple[str, str, str, str, str, str, str, int | None, int | None]]: ... -def build_rich_column_lineage( - request: tuple[ - str, - list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], - list[str], - ], + project: CompiledProjectView, + dialect: str | None, + model_names: frozenset[str], /, -) -> list[ - tuple[str, list[tuple[str, str, str, str, list[tuple[str, str, str]]]], bool, str | None] -]: ... +) -> tuple[DirectSemanticColumnUse, ...]: ... +def render_dependency_tree[Node: Hashable]( + *, + focus: Node, + deps: dict[Node, list[Node]], + seen: set[Node], + format_node: Callable[[Node], str], + sort_key: Callable[[Node], SupportsRichComparison], + branch_style: Callable[[str], str], + already_shown: Callable[[], str], +) -> list[str]: ... +def render_column_trace_tree[Column, Edge]( + *, + target: Column, + deps: dict[str, list[Edge]], + total: int, + limit: int, + column_id: Callable[[Column], str], + related_column: Callable[[Edge], Column], + format_related: Callable[[Edge], str], + branch_style: Callable[[str], str], + already_shown: Callable[[], str], + note_style: Callable[[str], str], +) -> list[str]: ... +def render_column_trace_limit_note( + *, total: int, limit: int, note_style: Callable[[str], str] +) -> list[str]: ... +def serialize_column( + *, + column: QualifiedLineageColumn, + render_resource_type: Callable[[QualifiedLineageColumn], str], +) -> dict[str, object]: ... +def serialize_column_edge( + *, + edge: ColumnLineageEdge, + render_resource_type: Callable[[QualifiedLineageColumn], str], +) -> dict[str, object]: ... + +class ProjectColumnLineage: + """Project-level column lineage graph with lazily materialized public views.""" + + def __init__( + self, + *, + models: dict[str, ModelColumnLineage], + edges: tuple[ColumnLineageEdge, ...], + ) -> None: ... + @classmethod + def from_fast_facts( + cls, + *, + models: dict[str, ModelColumnLineage], + compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]], + model_order: tuple[str, ...], + ) -> ProjectColumnLineage: ... + @property + def models(self) -> dict[str, ModelColumnLineage]: ... + @property + def edges(self) -> tuple[ColumnLineageEdge, ...]: ... + def has_model(self, model_name: str, /) -> bool: ... + def model_has_star(self, model_name: str, /) -> bool: ... + def edge_count_targeting(self, model_name: str, /) -> int: ... + def edges_targeting(self, model_name: str, /) -> tuple[ColumnLineageEdge, ...]: ... + def edges_sourced_from(self, resource_name: str, /) -> tuple[ColumnLineageEdge, ...]: ... + def column_consumers( + self, *, resource_name: str, column_name: str + ) -> tuple[ColumnLineageEdge, ...]: ... + def relation_lineage_fingerprint( project_dir: Path, prefix: bytes, interrupted_listing_uncacheable: bool, / ) -> str | None: ... @@ -1697,6 +1910,20 @@ def assemble_project_resource_facts( tuple[str, int, str, str, str | None, int | None] | None, ]: ... def check_native_explicit_references(request_json: str, /) -> str: ... +def native_execution_gate_edges( + request_json: str, / +) -> tuple[ + list[tuple[str, tuple[str, str], tuple[str, str], tuple[str, str]]], + list[tuple[str, tuple[str, str], tuple[str, str], str, str | None]], +]: ... +def schema_validation_rows( + catalog: Any, + dialect: str, + requests: list[tuple[str, list[tuple[str, bool]], dict[str, Mapping[str, str]]]], +) -> list[list[tuple[str, str, int | None, int | None, int | None, int | None, str]]]: ... +def python_literal_sql_calls( + source: str, start_line: int, placeholder: str +) -> list[tuple[str, str, int]]: ... def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py index 6bbc59afbd..c9d62648a6 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlbuild._native as _native from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target from sqlbuild.cli.commands.models import ( BuildCommandRequest, @@ -19,7 +20,7 @@ def write_build_compile_target( ) -> None: """Write the compile target artifacts with the optional manifest payload.""" - manifest_payload: dict[str, object] | None = None + manifest_payload: _native.NativeManifest | None = None if request.manifest: manifest_payload = build_manifest( project=pipeline_result.project, diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py index f37e4ee86c..f65660587e 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config @@ -19,7 +20,6 @@ ) from sqlbuild.cli.progress.models import CommandProgressReporters from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py b/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py index bedc7c7dd5..4d2469c00a 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import run_compile_pipeline from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -10,7 +11,6 @@ BuildInvocation, ) from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py index 599639cc66..932fc9b368 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py @@ -6,13 +6,13 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.clone.validation import validate_clone_request from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py b/src/sqlbuild/cli/commands/_helpers/compile/lineage.py index fc1150b4e8..0aaad2bd5b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/lineage.py @@ -2,9 +2,9 @@ from __future__ import annotations +import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -19,16 +19,12 @@ def build_compile_lineage( case CompileLineageMode.NONE: return None case CompileLineageMode.FAST: - return build_project_column_lineage( - project=graph.project, - dialect=dialect, - mode=ColumnLineageMode.FAST, + return _native.build_project_column_lineage( + graph.project, dialect, ColumnLineageMode.FAST, None ) case CompileLineageMode.RICH: - return build_project_column_lineage( - project=graph.project, - dialect=dialect, - mode=ColumnLineageMode.RICH, + return _native.build_project_column_lineage( + graph.project, dialect, ColumnLineageMode.RICH, None ) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 97ef017e1c..08b08badab 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -7,6 +7,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.compile.semantic_notice import ( selected_semantic_coverage, semantic_coverage_notice, @@ -36,7 +37,6 @@ ) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index bc1be4841d..52ae0b17d3 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from collections.abc import Callable from functools import partial @@ -78,7 +77,7 @@ def write_compile_target( target_dir: Path, adapter: BaseAdapter, plan_output: PlanOutput, - manifest: dict[str, object] | None = None, + manifest: _native.NativeManifest | None = None, ) -> WrittenTarget: """Write compiled output files under target_dir.""" @@ -115,7 +114,7 @@ def write_static_compile_target( target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView, - manifest: dict[str, object] | None = None, + manifest: _native.NativeManifest | None = None, planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, ) -> WrittenTarget: """Write offline compiled output files under target_dir, reusing SQL tests planned ahead.""" @@ -166,7 +165,7 @@ def publish_static_compile_target( *, prepared: WrittenTarget, target_dir: Path, - manifest: dict[str, object] | None, + manifest: _native.NativeManifest | None, expected_files: frozenset[str], ) -> WrittenTarget: """Publish staged files with the same unchanged-file and stale-file semantics.""" @@ -583,12 +582,12 @@ def _sql_test_artifact_diagnostics( ) -def _write_manifest(*, target_dir: Path, manifest: dict[str, object]) -> None: +def _write_manifest(*, target_dir: Path, manifest: _native.NativeManifest) -> None: """Write manifest.json.""" manifest_path: Path = target_dir / _MANIFEST_FILE batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=True) - batch.queue(path=manifest_path, contents=(json.dumps(manifest, indent=2) + "\n").encode()) + batch.queue(path=manifest_path, contents=manifest.contents()) batch.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py index b5c37ff3cf..f3c3619242 100644 --- a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py +++ b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py @@ -3,6 +3,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import ( resolve_project_connection_config, @@ -10,7 +11,6 @@ from sqlbuild.cli.commands.constants import COST_HISTORY_SELECTOR, COST_LATEST_SELECTOR from sqlbuild.cli.commands.models import CostCommandRequest from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py index 85cade7924..5bcbb73e0d 100644 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/debug/checks.py @@ -7,6 +7,7 @@ from importlib.metadata import PackageNotFoundError, version from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import ( @@ -39,7 +40,6 @@ LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, diff --git a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py index db4f00b109..fce0894b86 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py @@ -4,9 +4,9 @@ from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffIncompleteError from sqlbuild.cli.commands.models import DiffCommandRequest, DiffInvocation -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.discover_configuration import ( discover_project_configuration, ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py index 1868504ac8..b23675a53f 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py @@ -5,6 +5,7 @@ import sys import time +from sqlbuild._native import build_compiled_project from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -15,7 +16,6 @@ ) from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,7 +35,7 @@ def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileC ) with OperationLifecycle(operation_kind="project", operation_name="project_compile"): status.on_progress("Compiling project...") - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=invocation.discovered_inputs, adapter=adapter_context.adapter, selected_target=( diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 229b43313b..5264a0bf7b 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( @@ -11,7 +12,6 @@ JanitorInvocation, JanitorSettings, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py index 363cc8cbad..5a25c8072b 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py @@ -5,6 +5,7 @@ import json from pathlib import Path +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.lineage.cache import ( read_relation_lineage_cache, @@ -106,9 +107,8 @@ def _prepare_graph( ) if cached is not None: return cached, None, parsed_depth - from sqlbuild._native import build_project_graph + from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=project_dir, @@ -159,17 +159,15 @@ def _semantic_uses( ) -> tuple[DirectSemanticColumnUse, ...]: if not request.include_uses: return () - from sqlbuild.compiler.lineage.main.build_semantic_uses import build_direct_semantic_uses - if not isinstance(graph, ProjectGraph) or adapter is None: raise CliUserError( "semantic uses require a compiled project graph", code="C307", ) - return build_direct_semantic_uses( - project=graph.project, - dialect=adapter.sql_analysis_dialect(), - model_names=_semantic_use_model_names(request=request, graph=graph), + return _native.build_direct_semantic_uses( + graph.project, + adapter.sql_analysis_dialect(), + _semantic_use_model_names(request=request, graph=graph), ) @@ -216,7 +214,7 @@ def _render_lineage( if error.code == UNKNOWN_SELECTOR_ERROR_CODE and isinstance( graph, RelationLineageIndex ): - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs + from sqlbuild._native import discover_project_inputs reject_unit_test_selectors( select=request.select, diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/output.py b/src/sqlbuild/cli/commands/_helpers/lineage/output.py index 1984153b55..b962ec46e1 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/output.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/output.py @@ -4,6 +4,13 @@ import json +from sqlbuild._native import ( + render_column_trace_limit_note, + render_column_trace_tree, + render_dependency_tree, + serialize_column, + serialize_column_edge, +) from sqlbuild.cli.commands.constants import ( BOTH_DIRECTIONS, DOWNSTREAM_DIRECTION, @@ -16,13 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.main.render_column_trace_limit_note import ( - render_column_trace_limit_note, -) -from sqlbuild.compiler.lineage.main.render_column_trace_tree import render_column_trace_tree -from sqlbuild.compiler.lineage.main.render_dependency_tree import render_dependency_tree -from sqlbuild.compiler.lineage.main.serialize_column import serialize_column -from sqlbuild.compiler.lineage.main.serialize_column_edge import serialize_column_edge from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn from sqlbuild.compiler.lineage.types import ColumnTransformKind from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py index 61f8dfe0bb..4e6a948f77 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py @@ -5,6 +5,8 @@ from collections.abc import Iterable from dataclasses import dataclass +import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands.constants import ( BOTH_DIRECTIONS, COLUMN_TARGET_KIND, @@ -31,10 +33,8 @@ from sqlbuild.compiler.graph.main.path_nodes import path_nodes from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, - ProjectColumnLineage, QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -225,10 +225,8 @@ def select_column_target_lineage( and len(candidate_selection.model_names) >= RICH_LINEAGE_STATUS_MODEL_THRESHOLD ), ): - column_lineage: ProjectColumnLineage | None = build_project_column_lineage( - project=graph.project, - mode=mode, - model_names=candidate_selection.model_names, + column_lineage: ProjectColumnLineage | None = _native.build_project_column_lineage( + graph.project, None, mode, candidate_selection.model_names ) if column_lineage is None: raise CliUserError( diff --git a/src/sqlbuild/cli/commands/_helpers/lint/selection.py b/src/sqlbuild/cli/commands/_helpers/lint/selection.py index ef0b456211..63fe30b415 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/selection.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -12,7 +12,6 @@ from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors diff --git a/src/sqlbuild/cli/commands/_helpers/load/invocation.py b/src/sqlbuild/cli/commands/_helpers/load/invocation.py index 64ef7df106..0a522f3140 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/load/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.cli.commands._helpers.load.selection import ( select_load_entries, @@ -15,7 +16,6 @@ from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 9123cc88fc..d331b6c0aa 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -20,7 +21,6 @@ ) from sqlbuild.cli.progress.models import CommandProgressReporters from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py index 41fd7ca62b..b52075934b 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/planning.py @@ -2,11 +2,11 @@ from __future__ import annotations +from sqlbuild._native import run_compile_pipeline from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index 752fc1e08b..b22b1aca8c 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -7,7 +7,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( @@ -17,7 +17,6 @@ ) from sqlbuild.cli.commands.models import RulesCommandRequest from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py index 768ff9be5f..180f99d001 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Protocol, TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -16,7 +17,6 @@ build_command_progress_reporters, ) from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index 6b8b7563a0..f014cde75c 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -37,7 +38,6 @@ from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index cc0f8bebc4..7f4f6f86e9 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -54,7 +55,6 @@ from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline diff --git a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py index 83d9d6b5f3..eedd5ec7d3 100644 --- a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py +++ b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py @@ -12,6 +12,7 @@ from filelock import FileLock, Timeout +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import ( plan_static_sql_tests, @@ -148,7 +149,7 @@ def prepare() -> _PreparedResult: return True def publish( - self, *, target_dir: Path, manifest: dict[str, object] | None + self, *, target_dir: Path, manifest: _native.NativeManifest | None ) -> WrittenTarget | None: if self._planned_tests is not None and self._planning_inputs is not None: project, adapter, _ = self._planning_inputs diff --git a/src/sqlbuild/cli/commands/main/execution/_freshness.py b/src/sqlbuild/cli/commands/main/execution/_freshness.py index 6713ecbef8..93bb69164b 100644 --- a/src/sqlbuild/cli/commands/main/execution/_freshness.py +++ b/src/sqlbuild/cli/commands/main/execution/_freshness.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.observe import ( observe_source_freshness_for_command, @@ -30,7 +30,6 @@ FreshnessCommandResult, ) from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py index 08e25a3c72..545e5315f8 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ b/src/sqlbuild/cli/commands/main/inspection/_contract.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.cli.commands._helpers.contract.progress import ( inspect_contracts_with_progress, write_contract_updates_with_progress, @@ -28,7 +28,6 @@ ) from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.contract_adoption.types import ContractAction -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project_graph import build_project_graph_from_compiled_project from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph diff --git a/src/sqlbuild/cli/commands/main/inspection/_query.py b/src/sqlbuild/cli/commands/main/inspection/_query.py index 807a9eda40..4c5ad455a2 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_query.py +++ b/src/sqlbuild/cli/commands/main/inspection/_query.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import QueryResult @@ -14,7 +15,6 @@ resolve_project_connection_config, ) from sqlbuild.cli.commands.exceptions import CliUserError, QueryExecutionError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, diff --git a/src/sqlbuild/cli/commands/main/project/_dag.py b/src/sqlbuild/cli/commands/main/project/_dag.py index f770c9a6da..2502ffef2c 100644 --- a/src/sqlbuild/cli/commands/main/project/_dag.py +++ b/src/sqlbuild/cli/commands/main/project/_dag.py @@ -5,14 +5,13 @@ import json from pathlib import Path -from sqlbuild._native import build_project_graph +import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.dag.main.build import build_dag_json -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph @@ -42,7 +41,7 @@ def run_dag( ), project_dir=effective_project_dir, ) - graph: ProjectGraph = build_project_graph( + graph: ProjectGraph = _native.build_project_graph( discovered_inputs=discovered_inputs, adapter=adapter, no_sql_validation=no_sql_validation, @@ -55,7 +54,7 @@ def run_dag( python_graph: PythonNodeGraph = build_discovered_python_node_graph( discovered_inputs=discovered_inputs ) - dag_json: str = build_dag_json( + dag_json: str = _native.build_dag_json( graph=graph, project_name=discovered_inputs.project_config.name, python_graph=python_graph, diff --git a/src/sqlbuild/cli/commands/main/project/_format.py b/src/sqlbuild/cli/commands/main/project/_format.py index 1ff842a3de..cb4e5fb99a 100644 --- a/src/sqlbuild/cli/commands/main/project/_format.py +++ b/src/sqlbuild/cli/commands/main/project/_format.py @@ -8,6 +8,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.lint.runs import ( prepare_lint_run, @@ -16,7 +17,6 @@ ) from sqlbuild.cli.commands._helpers.lint.selection import resolve_lint_inputs from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.main.has_fixture_typed_null_candidates import ( has_fixture_typed_null_candidates, diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index 4fde249c68..fd437097ee 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path +from sqlbuild._native import ProjectColumnLineage from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.compile.models import ( @@ -13,7 +14,6 @@ CompilerDiagnostic, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.models import NativeSqlTestArtifact diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py deleted file mode 100644 index 052823009c..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ /dev/null @@ -1,687 +0,0 @@ -"""Assemble planner-ready compiled resource objects from compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows import ( - infer_native_query_column_rows, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeColumnQuery, - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.analysis_session.types import ColumnRow, DiagnosticRow -from sqlbuild.compiler.compile._helpers.analysis.cache import ( - build_analysis_cache_context, -) -from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( - model_placeholders as _model_placeholders, -) -from sqlbuild.compiler.compile._helpers.assembly.hand_built_models import ( - hand_built_compiled_model, -) -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) -from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics -from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors -from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( - analyze_model_sql, - expression_source_shapes, - native_project_facts, -) -from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( - assemble_sql_tests, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import ( - resolve_early_model_templates, -) -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, - get_validated_model_cursor_intrinsics, -) -from sqlbuild.compiler.compile._helpers.render.declarations import build_declaration_scope_resolver -from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros, - find_macro_call_names, -) -from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( - scope_index_with_compile_usages, -) -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompiledObjectKey, - CompiledProject, - CompiledSqlScenario, - CompileModelSqlTestInputPayload, - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, - DeclarationScopeResolver, - HandBuiltAnalysis, - MacroContext, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, - CompileInputsView, - ModelInput, - SourceInput, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.profiling.main._record_cpu import record_compile_cpu_timing -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.project_assembly.constants import AUDIT_SLOT, SEED_SLOT, SOURCE_SLOT -from sqlbuild.compiler.project_assembly.main._native_resource import native_resource -from sqlbuild.compiler.project_assembly.models import ( - NativeModelFacts, - NativeProjectFacts, - NativeProjectResources, -) -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.sql_analysis.constants import ( - NATIVE_DIALECT_ALIASES, - NATIVE_FAILURE_PREFIX, -) -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog -from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) -from sqlbuild.spec.contracts.models import ( - SchemaDynamicColumnFamily, -) - - -@dataclass(frozen=True) -class _DynamicContractAnalysisInputs: - families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] - authoritative_column_types_by_table: dict[str, dict[str, str]] - - -def assemble_compiled_project( - *, - inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile | None = None, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - analysis_cache_dir: Path | None = None, - analysis_model_names: frozenset[str] | None = None, -) -> CompiledProjectView: - """Convert attached compile inputs into the planner-ready project view.""" - - sql_analysis_enabled: bool = ( - inputs.effective_settings.sql_analysis and not skip_column_inference - ) - column_types_by_table: dict[str, dict[str, str]] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - complete_binding_schemas: dict[str, dict[str, str]] - column_types_by_table, column_nullability_by_table, complete_binding_schemas = ( - _native.declared_relations(inputs) - ) - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( - _build_dynamic_families_by_table(inputs) - ) - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - profile = replace( - profile, - semantic_known_functions=known_function_names(inputs.sql_function_inputs), - semantic_known_types=known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - quoted_identifiers_ignore_case=ignores_quoted_case( - connection=inputs.effective_connection, dialect=profile.sql_analysis_dialect - ), - sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( - profile.sql_analysis_dialect or "", profile.sql_analysis_dialect - ), - function_return_types={ - **profile.function_return_types, - **{ - f"__sqlbuild_udf_{function.name}".upper(): function.returns - for function in inputs.sql_function_inputs - if not function.return_columns - }, - }, - ) - binding_catalog: Any = create_binding_catalog( - dialect=profile.sql_analysis_dialect or "generic", - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=profile.semantic_known_functions, - known_types=profile.semantic_known_types, - relations={}, - ) - profile = replace(profile, binding_catalog=binding_catalog) - if sql_analysis_enabled: - expression_sources: tuple[SourceInput, ...] = tuple( - source_input - for source_input in inputs.source_inputs - if source_input.source_entry.expression - ) - expression_shapes: tuple[dict[str, str] | None, ...] = expression_source_shapes( - expressions=tuple( - cast(str, source_input.source_entry.expression) - for source_input in expression_sources - ), - profile=profile, - ) - for source_input, shape in zip(expression_sources, expression_shapes, strict=True): - if shape is not None: - declared_types: dict[str, str] = column_types_by_table.get( - source_input.source_entry.name, {} - ) - declared_shape: dict[str, str] = { - name: declared_types.get(name, column_type) - for name, column_type in shape.items() - } - complete_binding_schemas[source_input.source_entry.name] = declared_shape - column_types_by_table[source_input.source_entry.name] = declared_shape - column_nullability_by_table[source_input.source_entry.name] = { - name: column_nullability_by_table.get(source_input.source_entry.name, {}).get( - name, InferredNullability.UNKNOWN - ) - for name in declared_shape - } - binding_catalog.native.update_relations(complete_binding_schemas) - binding_catalog.schemas.update(complete_binding_schemas) - dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs = ( - _DynamicContractAnalysisInputs( - families_by_table=dynamic_families_by_table, - authoritative_column_types_by_table=complete_binding_schemas, - ) - ) - allow_compact_analysis: bool = column_lineage_mode in { - ColumnLineageMode.FAST, - ColumnLineageMode.RICH, - } - rich_type_inference: bool = column_lineage_mode == ColumnLineageMode.RICH - analysis_cache: AnalysisCacheContext | None = ( - build_analysis_cache_context( - root=analysis_cache_dir, - inference_profile=profile, - allow_compact_analysis=allow_compact_analysis, - rich_type_inference=rich_type_inference, - signature_namespace={ - "known_functions": known_function_names(inputs.sql_function_inputs), - "known_types": known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - "target": inputs.effective_target_name, - "vars": inputs.effective_vars, - }, - ) - if sql_analysis_enabled and analysis_model_names != frozenset() - else None - ) - analyses: NativeModelAnalyses | None = None - if sql_analysis_enabled: - with ( - record_compile_timing("model_analysis_ms"), - record_compile_cpu_timing("model_analysis_cpu_ms"), - ): - analyses = analyze_model_sql( - NativeModelAnalysisRequest( - known_functions=known_function_names(inputs.sql_function_inputs), - known_types=known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - model_inputs=tuple( - model_input - for model_input in inputs.model_inputs - if model_input.sql_validation_enabled - and ( - analysis_model_names is None - or _model_name(model_input) in analysis_model_names - ) - ), - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - inference_profile=profile, - allow_compact_analysis=allow_compact_analysis, - rich_type_inference=rich_type_inference, - analysis_cache=analysis_cache, - complete_binding_schemas=complete_binding_schemas, - dynamic_families_by_table=dynamic_families_by_table, - ) - ) - native_session: Any | None = None if analyses is None else analyses.session - native: NativeProjectFacts = native_project_facts( - inputs=inputs, - dialect=profile.sql_analysis_dialect, - analysis_model_names=analysis_model_names, - analyses=analyses, - session=native_session, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=complete_binding_schemas, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ) - resources: NativeProjectResources = native.resources - scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) - report_scope_index_errors(index=scope_index) - effective_target_values: dict[str, object] = resolve_early_model_templates( - values={ - "database": ( - (inputs.effective_target.database if inputs.effective_target is not None else None) - or _connection_database_fallback(inputs=inputs) - ), - "schema": ( - (inputs.effective_target.schema if inputs.effective_target is not None else None) - or _str_or_none(inputs.effective_connection.get("schema")) - ), - }, - effective_vars=inputs.effective_vars, - effective_target_name=inputs.effective_target_name, - run_id=inputs.run_id, - ) - models: tuple[CompiledModelView, ...] = tuple( - _assemble_compiled_model( - model_input=model_input, - sql_analysis_enabled=( - sql_analysis_enabled - and model_input.sql_validation_enabled - and ( - analysis_model_names is None or _model_name(model_input) in analysis_model_names - ) - ), - sql_validation_enabled=( - analysis_model_names is None or _model_name(model_input) in analysis_model_names - ), - native_model=native.models[index], - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, - inference_profile=profile, - analyses=analyses, - allow_compact_analysis=allow_compact_analysis, - ) - for index, model_input in enumerate(inputs.model_inputs) - ) - fields: dict[str, Any] = dict( - binding_catalog=profile.binding_catalog, - run_id=inputs.run_id, - effective_target_name=inputs.effective_target_name, - effective_connection=inputs.effective_connection, - effective_vars=inputs.effective_vars, - effective_target_database=_str_or_none(effective_target_values.get("database")), - effective_target_schema=_str_or_none(effective_target_values.get("schema")), - sql_analysis_dialect=profile.sql_analysis_dialect, - sql_lexical_syntax=inputs.sql_lexical_syntax, - compile_cache_dir=inputs.compile_cache_dir, - settings=inputs.effective_settings, - scenario=resolve_effective_scenario_config( - project_config=inputs.project_config, - local_config=inputs.local_config, - ), - models=models, - sources=_compiled_sources(inputs=inputs, resources=resources), - seeds=_compiled_seeds(inputs=inputs, resources=resources), - functions=_compiled_functions(inputs=inputs, resources=resources), - audits=_compiled_audits(inputs=inputs, resources=resources), - sql_tests=assemble_sql_tests( - inputs=inputs, - macro_mock_queries=lambda test_input: _build_test_model_query_overrides( - test_input=test_input, model_inputs=inputs.model_inputs, inputs=inputs - ), - ), - sql_scenarios=tuple( - _assemble_compiled_sql_scenario(scenario_input) - for scenario_input in inputs.scenario_inputs - ), - loader_functions=inputs.discovered_inputs.loader_functions, - hook_functions=inputs.discovered_inputs.hook_functions, - enforce_explicit_references=inputs.project_config.references.enforce_explicit, - sql_hook_files=inputs.discovered_inputs.sql_hook_files, - materialization_files=inputs.discovered_inputs.materialization_files, - public_enums=inputs.public_enums, - public_constants=inputs.public_constants, - loaded_macros=inputs.loaded_macros, - diagnostics=( - *inputs.diagnostics, - *_project_binding_diagnostics(models=models, analyses=analyses), - ), - external_sql_reference_resolver=inputs.external_sql_reference_resolver, - scope_index=scope_index, - ) - project: CompiledProjectView = _assembled_project(inputs=inputs, fields=fields) - return complete_semantic_diagnostics( - project=project, - profile=profile, - resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, - native_session=native_session, - binding_results=_binding_rows(analyses), - ) - - -def _assembled_project(*, inputs: CompileInputsView, fields: dict[str, Any]) -> CompiledProjectView: - """The native owner's project view when it holds every model; else a `CompiledProject`.""" - - models: tuple[CompiledModelView, ...] = fields["models"] - if inputs.native_project is not None and all( - isinstance(model, _native.NativeCompiledModel) for model in models - ): - return inputs.native_project.project_view( - {**fields, "unmatched_literal_sql_relations": (), "native_session": None} - ) - return CompiledProject( - **fields, - sql_expansions={ - model_input.model_file.file_path: model_input.sql_expansion - for model_input in inputs.model_inputs - if model_input.sql_expansion is not None - }, - ) - - -def _str_or_none(value: object) -> str | None: - return value.strip() if isinstance(value, str) and value.strip() else None - - -def _connection_database_fallback(*, inputs: CompileInputsView) -> str | None: - adapter_name: str = resolve_effective_adapter_name( - project_config=inputs.project_config, - local_config=inputs.local_config, - ) - if adapter_name == BuiltinAdapter.DUCKDB: - return None - return _str_or_none(inputs.effective_connection.get("database")) - - -def _assemble_compiled_model( - *, - model_input: ModelInput, - sql_analysis_enabled: bool, - sql_validation_enabled: bool = True, - native_model: NativeModelFacts, - column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, - column_types_by_table: dict[str, dict[str, str]] | None = None, - dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, - inference_profile: ExpressionInferenceProfile | None = None, - analyses: NativeModelAnalyses | None = None, - allow_compact_analysis: bool = False, -) -> CompiledModelView: - del column_types_by_table, dynamic_contract_analysis_inputs, allow_compact_analysis - model_name: str = model_input.model_file.file_path.stem - syntax_validated: bool = native_model.syntax_valid - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - analysis_query_sql: str = cursor_intrinsics_analysis_sql( - sql=model_input.query_sql, - cursor_type=model_input.config.values.get("cursor_type"), - ) - placeholders: dict[str, str] | None = _model_placeholders(model_input) - if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: - for hook_name in ("pre_hooks", "post_hooks"): - _native.validate_hook_sql_syntax( - model_input.config.values.get(hook_name), - hook_name, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - session_analysis: tuple[_native.NativeModelAnalysisSession, int] | None = None - fallback: list[ColumnRow] | None = None - if sql_analysis_enabled: - position: int | None = None if analyses is None else analyses.positions.get(model_name) - if analyses is None or position is None: - raise _native.NativeCompilerError( - f"{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no " - f"analysis for model '{model_name}'" - ) - session_analysis = ( - cast(_native.NativeModelAnalysisSession, analyses.session), - position, - ) - if model_name not in analyses.succeeded: - if model_input.sql_validation_enabled and not syntax_validated: - _native.validate_sql_syntax( - analysis_query_sql, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - fallback = infer_native_query_column_rows( - queries=( - NativeColumnQuery( - sql=analysis_query_sql, mode="parse", placeholders=placeholders - ), - ), - profile=profile, - column_nullability_by_table=column_nullability_by_table, - )[0][1] - elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: - _native.validate_sql_syntax( - analysis_query_sql, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - macro_deps: tuple[str, ...] = model_input.macro_deps or find_macro_call_names( - model_input.macro_source_sql - ) - if isinstance(model_input, _native.NativeModelInput): - return model_input.compiled_model( - ( - native_model.deps, - session_analysis, - fallback, - native_model.dynamic_contract, - list(macro_deps), - ) - ) - return hand_built_compiled_model( - model_input=model_input, - deps=native_model.deps, - analysis=HandBuiltAnalysis( - session=session_analysis, - fallback=fallback, - standalone_proof=native_model.dynamic_contract, - ), - macro_deps=macro_deps, - ) - - -def _model_name(model_input: ModelInput) -> str: - return model_input.model_file.file_path.stem - - -def _build_dynamic_families_by_table( - inputs: CompileInputsView, -) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: - return { - model_input.schema_entry.name: model_input.schema_entry.dynamic_columns - for model_input in inputs.model_inputs - if model_input.config.values.get("contract") == ContractPolicy.ENFORCED - and model_input.schema_entry is not None - and model_input.schema_entry.dynamic_columns - } - - -def _project_binding_diagnostics( - *, models: tuple[CompiledModelView, ...], analyses: NativeModelAnalyses | None -) -> tuple[CompilerDiagnostic, ...]: - """Every analysed model's located binding diagnostics, which the project also reports.""" - - if analyses is None: - return () - diagnostics: list[CompilerDiagnostic] = [] - for model in models: - if model.name in analyses.positions: - diagnostics.extend(model.binding_diagnostics) - return tuple(diagnostics) - - -def _binding_rows(analyses: NativeModelAnalyses | None) -> dict[str, list[DiagnosticRow]]: - """Each analysed model's raw binding rows, which semantic type recovery identifies.""" - - if analyses is None or analyses.session is None: - return {} - rows: list[list[DiagnosticRow]] = analyses.session.binding_rows() - return {name: rows[position] for name, position in analyses.positions.items()} - - -def _compiled_sources( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledSourceView, ...]: - """Each source's view over the compile's native project, raising assembly's error there.""" - - if not inputs.source_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_source_namespaces(list(resources.source_namespaces)) - sources: list[CompiledSourceView] = [] - for index in range(len(inputs.source_inputs)): - _ = native_resource( - resources=resources, slot=SOURCE_SLOT, items=resources.source_namespaces, index=index - ) - sources.append(owner.compiled_source(index)) - return tuple(sources) - - -def _compiled_seeds( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledSeedView, ...]: - """Each seed's view over the compile's native project, raising assembly's error at its seed.""" - - if not inputs.seed_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_seed_namespaces(list(resources.seed_namespaces)) - seeds: list[CompiledSeedView] = [] - for index in range(len(inputs.seed_inputs)): - _ = native_resource( - resources=resources, slot=SEED_SLOT, items=resources.seed_namespaces, index=index - ) - seeds.append(owner.compiled_seed(index)) - return tuple(seeds) - - -def _compiled_functions( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledFunctionView, ...]: - """Each function's view over the compile's native project, with its resolved deps.""" - - if not inputs.sql_function_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_function_deps(list(resources.function_deps)) - return tuple(owner.compiled_function(index) for index in range(len(inputs.sql_function_inputs))) - - -def _compiled_audits( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledAuditView, ...]: - """Each audit's view over the compile's native project, raising assembly's error there.""" - - if not inputs.audit_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_audit_scope_deps(list(resources.audit_scope_deps)) - audits: list[CompiledAuditView] = [] - for index in range(len(inputs.audit_inputs)): - _ = native_resource( - resources=resources, slot=AUDIT_SLOT, items=resources.audit_scope_deps, index=index - ) - audits.append(owner.compiled_audit(index)) - return tuple(audits) - - -def _build_test_model_query_overrides( - *, - test_input: CompileSqlTestInput, - model_inputs: tuple[ModelInput, ...], - inputs: CompileInputsView, -) -> dict[str, str]: - """Re-expand each model from its pre-macro macro_source_sql with test macro mocks applied.""" - - if not isinstance(test_input.payload, CompileModelSqlTestInputPayload): - return {} - if not test_input.payload.macro_mocks: - return {} - model_macro_context: MacroContext = inputs.macro_context or MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=inputs.project_config, - local_config=inputs.local_config, - ), - sql_analysis_enabled=inputs.effective_settings.sql_analysis, - target_name=inputs.effective_target_name, - vars=inputs.effective_vars, - ) - macro_context: MacroContext = replace(model_macro_context, _enforce_explicit_references=False) - declaration_resolver: DeclarationScopeResolver = build_declaration_scope_resolver( - discovered_inputs=inputs.discovered_inputs, - scope_index=inputs.scope_index, - loaded_macros=inputs.loaded_macros, - lookup=( - inputs.declaration_scope.resolver.lookup - if inputs.declaration_scope is not None - else None - ), - ) - overrides: dict[str, str] = {} - model_input: ModelInput - for model_input in model_inputs: - model_name: str = model_input.model_file.file_path.stem - macro_source_sql: str = model_input.macro_source_sql or model_input.query_sql - overrides[model_name] = get_validated_model_cursor_intrinsics( - sql=expand_sql_macros( - sql=macro_source_sql, - file_path=model_input.model_file.file_path, - loaded_macros=inputs.loaded_macros, - macro_overrides=test_input.payload.macro_mocks, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - ), - config_values=model_input.config.values, - model_name=model_name, - ) - return overrides - - -def _assemble_compiled_sql_scenario( - scenario_input: CompileSqlScenarioInput, -) -> CompiledSqlScenario: - scenario_name: str = scenario_input.scenario_file.name - return CompiledSqlScenario( - key=CompiledObjectKey( - resource_type=CompiledResourceType.SQL_SCENARIO, - name=scenario_name, - ), - name=scenario_name, - scenario_file=scenario_input.scenario_file, - sql_body=scenario_input.sql_body, - authored_ctes=scenario_input.authored_ctes, - expected_ctes=scenario_input.expected_ctes, - assertion_ctes=scenario_input.assertion_ctes, - source_fixture_names=scenario_input.source_fixture_names, - ref_fixture_names=scenario_input.ref_fixture_names, - seed_fixture_names=scenario_input.seed_fixture_names, - dbt_ref_fixture_names=scenario_input.dbt_ref_fixture_names, - expected_model_names=scenario_input.expected_model_names, - assertion_names=scenario_input.assertion_names, - assertion_target_model_names=scenario_input.assertion_target_model_names, - target_model_names=scenario_input.target_model_names, - source_path=scenario_input.scenario_file.relative_path, - ownership_root=scenario_input.scenario_file.ownership_root, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index 2242fe26a1..9ec47b3b56 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -2,24 +2,17 @@ from __future__ import annotations -from collections.abc import Callable, Iterator +from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from functools import wraps -from typing import cast -import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( CollectedCompileDiagnostics, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompileProjectInputs, CompilerDiagnostic, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with -from sqlbuild.compiler.compiled_project.main.project_with import project_with _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None @@ -38,29 +31,6 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: _ACTIVE_COLLECTOR.reset(token) -def with_collected_compile_diagnostics[**P, R: (CompileInputsView, CompiledProjectView)]( - build: Callable[P, R], -) -> Callable[P, R]: - """Append every diagnostic reported while ``build`` runs to the diagnostics it returns.""" - - @wraps(build) - def collecting(*args: P.args, **kwargs: P.kwargs) -> R: - with collect_compile_diagnostics() as violations: - built: R = build(*args, **kwargs) - if not violations.diagnostics: - return built - diagnostics: tuple[CompilerDiagnostic, ...] = ( - *built.diagnostics, - *violations.diagnostics, - ) - if isinstance(built, CompileProjectInputs | _native.NativeInputsView): - return cast(R, inputs_with(inputs=built, diagnostics=diagnostics)) - project: CompiledProjectView = cast(CompiledProjectView, built) - return cast(R, project_with(project=project, diagnostics=diagnostics)) - - return collecting - - def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index 9e06ffcca5..8c65c7639c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -2,10 +2,9 @@ from __future__ import annotations -import ast import inspect import textwrap -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from itertools import chain from pathlib import Path @@ -47,9 +46,6 @@ _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") _RAISE_KEY: str = "raise" _REPORT_KEY: str = "report" -_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"}) -_SQL_PARAMETER_NAME: str = "sql" -_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"}) _VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" @@ -82,7 +78,7 @@ def explicit_reference_report( _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () ) literals: tuple[tuple[_LiteralSql, ...], ...] = tuple( - tuple(_literal_sql_calls(owner.function)) for owner in owners + _literal_sql_calls(owner.function) for owner in owners ) paths: list[Path] = sorted({owner.relative_path for owner in owners}) path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} @@ -373,68 +369,16 @@ def _loader_owners( return tuple(owners) -def _literal_sql_calls(function: Callable[..., object]) -> Iterator[_LiteralSql]: +def _literal_sql_calls(function: Callable[..., object]) -> tuple[_LiteralSql, ...]: try: lines: list[str] start_line: int lines, start_line = inspect.getsourcelines(function) - tree: ast.Module = ast.parse(textwrap.dedent("".join(lines))) - except (OSError, TypeError, SyntaxError): - return - definition: ast.stmt | None = tree.body[0] if tree.body else None - if not isinstance(definition, ast.FunctionDef | ast.AsyncFunctionDef): - return - parameters: list[str] = [ - argument.arg - for argument in ( - *definition.args.posonlyargs, - *definition.args.args, - *definition.args.kwonlyargs, + except (OSError, TypeError): + return () + return tuple( + _LiteralSql(method=method, sql=sql, line=line) + for method, sql, line in _native.python_literal_sql_calls( + textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER ) - ] - context_name: str | None = next( - (name for name in parameters if name in _CONTEXT_PARAMETER_NAMES), - parameters[0] if parameters else None, ) - if context_name is None: - return - for node in ast.walk(definition): - if not ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr in _SQL_METHOD_NAMES - and isinstance(node.func.value, ast.Name) - and node.func.value.id == context_name - ): - continue - argument: ast.expr | None = _sql_argument(node) - sql: str | None = _literal_text(argument) if argument is not None else None - if sql is not None: - yield _LiteralSql(method=node.func.attr, sql=sql, line=start_line + node.lineno - 1) - - -def _sql_argument(call: ast.Call) -> ast.expr | None: - if call.args: - return call.args[0] - for keyword in call.keywords: - if keyword.arg == _SQL_PARAMETER_NAME: - return keyword.value - if keyword.arg is None and isinstance(keyword.value, ast.Dict): - for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): - if isinstance(key, ast.Constant) and key.value == _SQL_PARAMETER_NAME: - return value - return None - - -def _literal_text(node: ast.expr) -> str | None: - if isinstance(node, ast.Constant) and isinstance(node.value, str): - return node.value - if not isinstance(node, ast.JoinedStr): - return None - parts: list[str] = [] - for index, value in enumerate(node.values): - if isinstance(value, ast.Constant) and isinstance(value.value, str): - parts.append(value.value) - else: - parts.append(f"{_VALUE_PLACEHOLDER}{index}") - return "".join(parts) diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py deleted file mode 100644 index 1c255587cd..0000000000 --- a/src/sqlbuild/compiler/compile/main/_assemble_project.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Assemble planner-ready compiled project objects.""" - -from __future__ import annotations - -from collections.abc import Callable -from functools import partial -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - with_collected_compile_diagnostics, -) -from sqlbuild.compiler.compile._helpers.explicit_references.checks import ( - explicit_reference_report, -) -from sqlbuild.compiler.compile.models import ( - PythonSqlReferenceReport, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main._record_model_analyses import record_model_analyses -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge - - -@with_collected_compile_diagnostics -def assemble_project( - *, - inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile | None = None, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - analysis_cache_dir: Path | None = None, - analysis_model_names: frozenset[str] | None = None, -) -> CompiledProjectView: - """Convert compile inputs into the planner-ready project view.""" - - assemble: Callable[[], CompiledProjectView] = partial( - assemble_compiled_project, - inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, - ) - project: CompiledProjectView = _with_native_project( - project=( - run_with_macro_bridge(stage=assemble) if active_macro_bridge() is None else assemble() - ), - native_project=inputs.native_project, - ) - references: PythonSqlReferenceReport = explicit_reference_report( - project=project, - discovered_inputs=inputs.discovered_inputs, - enforce_explicit=inputs.project_config.references.enforce_explicit, - ) - if not references.diagnostics and not references.unmatched: - return project - return project_with( - project=project, - diagnostics=(*project.diagnostics, *references.diagnostics), - unmatched_literal_sql_relations=references.unmatched, - ) - - -def _with_native_project( - *, project: CompiledProjectView, native_project: _native.NativeCompiledProject | None -) -> CompiledProjectView: - """Hand the compile's native project the assembled analyses and keep it on the project.""" - - if native_project is None: - return project - _ = record_model_analyses(project=native_project, models=project.models) - if project.native_project is native_project: - return project - return project_with(project=project, native_project=native_project) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py deleted file mode 100644 index d5e691c979..0000000000 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Build the first attached compile input snapshot from discovered inputs.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment.core import ( - build_effective_connection, - build_effective_settings, - build_effective_vars, - build_model_inputs, -) -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - with_collected_compile_diagnostics, -) -from sqlbuild.compiler.compile._helpers.macro_bridge.call_store import ( - target_context_with_macro_call_store, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id -from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, - DeclarationResolutionContext, - DeclarationScopeBuild, - LoadedMacro, - MacroContext, - ModelInputBuildContext, - ModelInputScopeBuild, -) -from sqlbuild.compiler.compile.types import ( - AuditInput, - CompileInputsView, - FunctionInput, - ModelInput, - SeedInput, - SourceInput, -) -from sqlbuild.compiler.compiled_project.main._attach_checks import attach_checks -from sqlbuild.compiler.compiled_project.main._attach_resources import attach_resources -from sqlbuild.compiler.compiled_project.main._compile_inputs_view import compile_inputs_view -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - EnumDeclaration, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig - - -@with_collected_compile_diagnostics -def build_compile_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter_context: CompileAdapterContext, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, - run_id: str | None = None, - no_sql_validation: bool = False, - defer_model_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - resolved_connection: dict[str, object] | None = None, - no_cache: bool = False, -) -> CompileInputsView: - """Attach discovered metadata into the first compile input snapshot.""" - - effective_target_name: str | None - effective_target: TargetConfig | None - compile_cache_dir: Path | None - effective_target_name, effective_target, compile_cache_dir = ( - target_context_with_macro_call_store( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - no_cache=no_cache, - ) - ) - - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=effective_target, - cli_vars={} if cli_vars is None else cli_vars, - ) - effective_settings: SettingsConfig = build_effective_settings( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - macro_context: MacroContext = MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - sql_analysis_enabled=effective_settings.sql_analysis, - target_name=effective_target_name, - vars=effective_vars, - _value_renderer=adapter_context.value_renderer, - _collection_rendering=adapter_context.collection_rendering, - _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), - ) - resolved_run_id: str = resolve_run_id(selected_run_id=run_id) - loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) - declaration_scope: DeclarationScopeBuild = build_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - model_context: ModelInputBuildContext = ModelInputBuildContext( - effective_vars=effective_vars, - effective_settings=effective_settings, - target_config=effective_target, - effective_target_name=effective_target_name, - run_id=resolved_run_id, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - value_renderer=adapter_context.value_renderer, - collection_rendering=adapter_context.collection_rendering, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - declaration_resolver=declaration_scope.resolver, - ) - model_build: ModelInputScopeBuild = _build_models_with_declarations( - discovered_inputs=discovered_inputs, - context=model_context, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - model_context = model_build.context - seed_inputs: tuple[SeedInput, ...] - sql_function_inputs: tuple[FunctionInput, ...] - source_inputs: tuple[SourceInput, ...] - seed_inputs, sql_function_inputs, source_inputs = attach_resources( - discovered_inputs=discovered_inputs, - native_project=model_build.native_project, - settings=( - model_context.effective_settings, - model_context.target_config, - model_context.macro_context.adapter_name, - ), - no_sql_validation=no_sql_validation, - python_functions_inherit_default_namespace=( - adapter_context.python_functions_inherit_default_namespace - ), - ) - test_inputs: tuple[CompileSqlTestInput, ...] - scenario_inputs: tuple[CompileSqlScenarioInput, ...] - audit_inputs: tuple[AuditInput, ...] - diagnostics: tuple[CompilerDiagnostic, ...] - test_inputs, scenario_inputs, audit_inputs, diagnostics = attach_checks( - discovered_inputs=discovered_inputs, - native_project=model_build.native_project, - settings=( - model_context.effective_settings, - adapter_context.value_renderer, - model_context.sql_lexical_syntax, - ), - scan_cache=declaration_scope.sql_test_scans, - source_inputs=source_inputs, - ) - fields: dict[str, Any] = { - "project_config": discovered_inputs.project_config, - "local_config": discovered_inputs.local_config, - "discovered_inputs": discovered_inputs, - "sql_lexical_syntax": adapter_context.sql_lexical_syntax, - "run_id": resolved_run_id, - "effective_target_name": effective_target_name, - "effective_target": effective_target, - "compile_cache_dir": compile_cache_dir, - "effective_connection": ( - resolved_connection - if resolved_connection is not None - else build_effective_connection( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=effective_target, - effective_vars=effective_vars, - ) - ), - "effective_settings": effective_settings, - "no_sql_validation": no_sql_validation, - "effective_vars": effective_vars, - "macro_context": macro_context, - "loaded_macros": declaration_scope.loaded_macros, - "public_enums": model_build.declarations.enums, - "public_constants": model_build.declarations.constants, - "model_inputs": model_build.inputs, - "seed_inputs": seed_inputs, - "source_inputs": source_inputs, - "sql_function_inputs": sql_function_inputs, - "test_inputs": test_inputs, - "scenario_inputs": scenario_inputs, - "audit_inputs": audit_inputs, - "diagnostics": diagnostics, - "external_sql_reference_resolver": external_sql_reference_resolver, - "scope_index": declaration_scope.index, - "declaration_scope": declaration_scope, - } - return compile_inputs_view(fields=fields, native_project=model_build.native_project) - - -def _build_models_with_declarations( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool, - defer_model_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> ModelInputScopeBuild: - public_enums: dict[str, EnumDeclaration] - public_constants: dict[str, ConstantDeclaration] - public_enums, public_constants = _native.public_declaration_indexes(discovered_inputs) - public_model_schemas: dict[str, ModelSchemaDeclaration] = _native.public_model_schema_index( - discovered_inputs - ) - declarations: DeclarationResolutionContext = DeclarationResolutionContext( - enums=public_enums, - constants=public_constants, - ) - native_project: _native.NativeCompiledProject = _native.NativeCompiledProject() - model_inputs: tuple[ModelInput, ...] = build_model_inputs( - discovered_inputs=discovered_inputs, - context=replace( - context, - public_enums=public_enums, - public_constants=public_constants, - public_model_schemas=public_model_schemas, - ), - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - native_project=native_project, - ) - return ModelInputScopeBuild( - inputs=model_inputs, - native_project=native_project, - declarations=declarations, - context=replace( - context, - public_enums=declarations.enums, - public_constants=declarations.constants, - ), - ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index ccb8a873a8..ebc498b355 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -5,6 +5,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, @@ -24,7 +25,6 @@ SqlExpansionContext, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py index d15a8b542a..c652443256 100644 --- a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py +++ b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py @@ -25,7 +25,6 @@ SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.compiled_project._helpers import resource_locators from sqlbuild.compiler.compiled_project._helpers.resource_rows import ( loader_rows, python_resource_rows, @@ -208,7 +207,6 @@ def resource_diagnostics_impl( "python_resources": python_resource_rows( discovered_inputs=discovered_inputs, source_inputs=source_inputs ), - "locators": resource_locators, } ) return (*orphans, *descriptions, *cursors) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py deleted file mode 100644 index 9fdee295d1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py +++ /dev/null @@ -1,29 +0,0 @@ -"""The decorator locator the native resource diagnostics call for authored Python functions.""" - -from __future__ import annotations - -import ast - - -def decorator_location(*, contents: str, decorator: str) -> tuple[int, int]: - """The `@decorator` line and column offset, ignoring docstrings and comments.""" - - try: - module: ast.Module = ast.parse(contents) - except SyntaxError: - return 1, 1 - for node in ast.walk(module): - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - for expression in node.decorator_list: - if _decorator_name(expression) == decorator: - return expression.lineno, expression.col_offset - return 1, 1 - - -def _decorator_name(expression: ast.expr) -> str | None: - target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression - if isinstance(target, ast.Name): - return target.id - if isinstance(target, ast.Attribute): - return target.attr - return None diff --git a/src/sqlbuild/compiler/contract_adoption/main/write.py b/src/sqlbuild/compiler/contract_adoption/main/write.py index ae108c3b71..cbd553b741 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/write.py @@ -5,7 +5,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, @@ -25,7 +25,6 @@ ContractEvidence, ContractFinding, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.main.write_atomically import write_atomically diff --git a/src/sqlbuild/compiler/dag/_helpers/__init__.py b/src/sqlbuild/compiler/dag/_helpers/__init__.py deleted file mode 100644 index 3d14180233..0000000000 --- a/src/sqlbuild/compiler/dag/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Static DAG helper package.""" diff --git a/src/sqlbuild/compiler/dag/_helpers/artifact.py b/src/sqlbuild/compiler/dag/_helpers/artifact.py deleted file mode 100644 index 886342867f..0000000000 --- a/src/sqlbuild/compiler/dag/_helpers/artifact.py +++ /dev/null @@ -1,610 +0,0 @@ -"""Static DAG artifact construction helpers.""" - -from __future__ import annotations - -import json -from dataclasses import asdict -from enum import StrEnum - -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.compiler.dag.constants import REQUIRED_DAG_COLLECTION_KEYS -from sqlbuild.compiler.dag.models import ( - DagArtifact, - DagCheck, - DagColumn, - DagColumnLineageRef, - DagEdge, - DagFunctionArgument, - DagNode, - DagTarget, -) -from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type -from sqlbuild.python_nodes.models import ColumnLineageRef, SqlResourceRef -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts -from sqlbuild.spec.contracts.models import ( - LoaderDestinationParts, - SchemaColumn, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.sql_values.types import SqlValueKind - -_DAG_VERSION: int = 1 - - -def build_dag_artifact( - *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None -) -> DagArtifact: - """Build a static DAG artifact from a compiled project graph.""" - - nodes: tuple[DagNode, ...] = ( - *(_build_source_node(source) for source in graph.project.sources), - *( - _build_loader_node( - project=graph.project, loader=loader, source_by_loader=_source_by_loader(graph) - ) - for loader in graph.project.loader_functions - if loader.name not in _source_by_loader(graph) - ), - *(_build_seed_node(seed) for seed in graph.project.seeds), - *(_build_function_node(function) for function in graph.project.functions), - *(_build_model_node(model) for model in graph.project.models), - *(_build_python_nodes(python_graph) if python_graph is not None else ()), - ) - return DagArtifact( - version=_DAG_VERSION, - project_name=project_name, - nodes=nodes, - edges=_build_edges(graph=graph, python_graph=python_graph), - checks=_build_checks(graph=graph, python_graph=python_graph), - ) - - -def format_dag_json(*, artifact: DagArtifact) -> str: - """Serialize the static DAG artifact to JSON.""" - - return json.dumps(_drop_none(asdict(artifact)), indent=2) - - -def _build_source_node(source: CompiledSourceView) -> DagNode: - entry: SourceEntry = source.source_entry - target: DagTarget = DagTarget( - database=entry.database, - schema=entry.schema, - name=entry.table or entry.name, - qualified_name=_qualified_name( - database=entry.database, schema=entry.schema, name=entry.table or entry.name - ), - ) - return DagNode( - id=_node_id(source.key), - kind=NodeKind.SOURCE.value, - name=source.name, - asset_key=_source_asset_key(entry), - target=target, - path=str(source.source_file.relative_path), - description=entry.description, - meta=entry.meta, - columns=tuple(_source_column(column) for column in entry.columns), - ) - - -def _build_loader_node( - *, - project: CompiledProjectView, - loader: DiscoveredLoaderFunction, - source_by_loader: dict[str, SourceEntry], -) -> DagNode: - entry: SourceEntry = source_by_loader.get(loader.name) or _loader_to_source_entry( - project=project, loader=loader - ) - target: DagTarget = DagTarget( - database=entry.database, - schema=entry.schema, - name=entry.table or entry.name, - qualified_name=_qualified_name( - database=entry.database, schema=entry.schema, name=entry.table or entry.name - ), - ) - return DagNode( - id=_loader_node_id(loader.name), - kind=NodeKind.LOADER.value, - name=loader.name, - asset_key=(loader.name,), - target=target, - path=str(loader.relative_path), - meta=entry.meta, - columns=tuple(_source_column(column) for column in entry.columns), - loader=loader.name, - ) - - -def _build_python_nodes(python_graph: PythonNodeGraph) -> tuple[DagNode, ...]: - return tuple( - _build_python_node(node) - for node in python_graph.nodes - if node.kind != PythonNodeKind.LOADER - ) - - -def _build_python_node(node: DiscoveredPythonNode) -> DagNode: - columns: tuple[DagColumn, ...] = () - column_lineage: dict[str, tuple[DagColumnLineageRef, ...]] = {} - materialization_type: str | None = None - if node.kind == PythonNodeKind.ASSET and node.asset is not None: - columns = tuple(_source_column(column) for column in node.asset.columns) - column_lineage = _python_column_lineage(node.asset.column_lineage) - materialization_type = "python_asset" - return DagNode( - id=_python_node_id(kind=node.kind, node_name=node.name), - kind=NodeKind(node.kind.value).value, - name=node.name, - asset_key=(node.kind.value, node.name), - path=str(node.relative_path), - description=node.description, - tags=node.tags, - group=node.group, - meta=node.meta or {}, - columns=columns, - column_lineage=column_lineage, - materialization_type=materialization_type, - ) - - -def _build_seed_node(seed: CompiledSeedView) -> DagNode: - return DagNode( - id=_node_id(seed.key), - kind=NodeKind.SEED.value, - name=seed.name, - asset_key=_target_asset_key(seed.destination), - target=_dag_target(seed.destination), - path=str(seed.seed_file.relative_path), - description=seed.schema_entry.description, - tags=seed.schema_entry.tags, - meta=seed.schema_entry.meta, - columns=tuple(_schema_column(column) for column in seed.schema_entry.columns), - ) - - -def _build_function_node(function: CompiledFunctionView) -> DagNode: - node_type: str = function_node_type(return_columns=function.return_columns) - return DagNode( - id=f"{node_type}:{function.name}", - kind=NodeKind(node_type).value, - name=function.name, - asset_key=_target_asset_key(function.destination), - target=_dag_target(function.destination), - path=str(function.relative_path), - sql=function.body_sql, - language=function.language.value, - tags=function.tags, - return_kind="table" if function.return_columns else "scalar", - returns=function.returns, - columns=tuple( - DagColumn(name=column.name, type=column.type) for column in function.return_columns - ), - arguments=tuple( - DagFunctionArgument(name=argument.name, type=argument.type) - for argument in function.arguments - ), - ) - - -def _build_model_node(model: CompiledModelView) -> DagNode: - return DagNode( - id=_node_id(model.key), - kind=NodeKind.MODEL.value, - name=model.name, - asset_key=_target_asset_key(model.destination), - target=_dag_target(model.destination), - path=str(model.relative_path), - description=(model.schema_entry.description if model.schema_entry is not None else None), - sql=model.authored_sql, - tags=_model_tags(model), - meta=(model.schema_entry.meta if model.schema_entry is not None else {}), - columns=_model_columns(model), - materialization_type=str(model.config.values.get("materialized", "view")), - ) - - -def _build_edges( - *, graph: ProjectGraph, python_graph: PythonNodeGraph | None = None -) -> tuple[DagEdge, ...]: - edges: list[DagEdge] = [ - DagEdge(from_id=f"{dep_kind}:{dep_name}", to_id=f"{kind}:{name}") - for (dep_kind, dep_name), (kind, name) in graph.native.edges() - ] - edges.extend(_build_loader_edges(graph)) - if python_graph is not None: - edges.extend(_build_python_edges(python_graph)) - return tuple(sorted(edges, key=lambda edge: (edge.from_id, edge.to_id))) - - -def _build_loader_edges(graph: ProjectGraph) -> tuple[DagEdge, ...]: - source_by_loader: dict[str, SourceEntry] = _source_by_loader(graph) - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in graph.project.loader_functions - } - edges: list[DagEdge] = [] - loader: DiscoveredLoaderFunction - for loader in graph.project.loader_functions: - for dependency in loader.depends_on: - dependency_name: str | None = loader_name_by_function.get(dependency) - if dependency_name is None: - continue - edges.append( - DagEdge( - from_id=_loader_or_source_node_id( - loader_name=dependency_name, source_by_loader=source_by_loader - ), - to_id=_loader_or_source_node_id( - loader_name=loader.name, source_by_loader=source_by_loader - ), - ) - ) - return tuple(edges) - - -def _loader_or_source_node_id(*, loader_name: str, source_by_loader: dict[str, SourceEntry]) -> str: - source_entry: SourceEntry | None = source_by_loader.get(loader_name) - if source_entry is not None: - return _node_id(CompiledObjectKey(CompiledResourceType.SOURCE, source_entry.name)) - return _loader_node_id(loader_name) - - -def _build_python_edges(python_graph: PythonNodeGraph) -> tuple[DagEdge, ...]: - edges: list[DagEdge] = [] - for edge in python_graph.dependency_edges: - upstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[edge.upstream_name] - downstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[edge.downstream_name] - edges.append( - DagEdge( - from_id=_python_node_id(kind=upstream_node.kind, node_name=upstream_node.name), - to_id=_python_node_id(kind=downstream_node.kind, node_name=downstream_node.name), - ) - ) - for node in python_graph.nodes: - for sql_dep in node.sql_deps: - edges.append( - DagEdge( - from_id=_sql_ref_node_id(sql_dep), - to_id=_python_node_id(kind=node.kind, node_name=node.name), - ) - ) - return tuple(edges) - - -def _build_checks( - *, graph: ProjectGraph, python_graph: PythonNodeGraph | None = None -) -> tuple[DagCheck, ...]: - checks: list[DagCheck] = [] - checks.extend(_build_sql_test_check(test) for test in graph.project.sql_tests) - checks.extend(_build_audit_check(audit) for audit in graph.project.audits) - checks.extend(_build_scenario_check(scenario) for scenario in graph.project.sql_scenarios) - if python_graph is not None: - checks.extend( - _build_python_check(node=node, python_graph=python_graph) - for node in python_graph.nodes - if node.kind == PythonNodeKind.CHECK - ) - return tuple(sorted(checks, key=lambda check: (check.kind, check.name))) - - -def _build_sql_test_check(test: CompiledSqlTest) -> DagCheck: - return DagCheck( - id=_sql_test_check_id(test), - kind=NodeKind.SQL_TEST.value, - name=test.name, - checked_asset_ids=tuple(_node_id(key) for key in test.scope_deps), - path=str(test.test_file.relative_path), - mode=test.mode.value, - meta=_sql_test_meta(test), - ) - - -def _sql_test_check_id(test: CompiledSqlTest) -> str: - if test.case_name is None: - return _node_id(test.key) - source_path: str = (test.source_path or test.test_file.relative_path).as_posix() - return f"sql_test:{source_path}:{test.block_index}:{test.case_name}" - - -def _sql_test_meta(test: CompiledSqlTest) -> dict[str, object]: - if test.case_name is None: - return {} - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test.parameter_schema - } - return { - "source_path": (test.source_path or test.test_file.relative_path).as_posix(), - "block_index": test.block_index, - "parent_name": test.parent_name, - "case_name": test.case_name, - "case_index": test.case_index, - "case_fingerprint": test.case_fingerprint, - "parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test.parameter_schema - ], - "parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": (str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value), - } - for name, value in test.parameter_values - ], - } - - -def _build_audit_check(audit: CompiledAuditView) -> DagCheck: - checked_asset_ids: tuple[str, ...] = tuple(_node_id(key) for key in audit.scope_deps) - if audit.attached_target_kind is not None and audit.attached_target_name is not None: - resource_type: CompiledResourceType = AttachedAuditTargetKind( - audit.attached_target_kind - ).resource_type - checked_asset_ids = ( - _node_id(CompiledObjectKey(resource_type, audit.attached_target_name)), - ) - return DagCheck( - id=_audit_check_id(audit), - kind=NodeKind.AUDIT.value, - name=audit.name, - checked_asset_ids=checked_asset_ids, - path=str(audit.audit_file.relative_path), - severity=audit.severity, - attachment_kind=( - audit.attached_target_kind.value if audit.attached_target_kind is not None else None - ), - attached_target_name=audit.attached_target_name, - attached_column_name=audit.attached_column_name, - ) - - -def _audit_check_id(audit: CompiledAuditView) -> str: - parts: tuple[str | None, ...] = ( - _node_id(audit.key), - audit.attached_target_kind.value if audit.attached_target_kind is not None else None, - audit.attached_target_name, - audit.attached_column_name, - ) - return ":".join(part for part in parts if part) - - -def _build_scenario_check(scenario: CompiledSqlScenario) -> DagCheck: - target_keys: tuple[CompiledObjectKey, ...] = tuple( - CompiledObjectKey(CompiledResourceType.MODEL, name) - for name in scenario.expected_model_names - ) - graph_asset_ids: tuple[str, ...] = tuple( - _node_id(CompiledObjectKey(CompiledResourceType.MODEL, cte.name)) - for cte in scenario.authored_ctes - ) - return DagCheck( - id=_node_id(scenario.key), - kind=NodeKind.SCENARIO.value, - name=scenario.name, - checked_asset_ids=tuple(_node_id(key) for key in target_keys), - path=str(scenario.scenario_file.relative_path), - assertion_names=scenario.assertion_names, - expected_model_names=scenario.expected_model_names, - graph_asset_ids=graph_asset_ids, - fixture_refs=( - *scenario.source_fixture_names, - *scenario.ref_fixture_names, - *scenario.seed_fixture_names, - *scenario.dbt_ref_fixture_names, - ), - ) - - -def _build_python_check(*, node: DiscoveredPythonNode, python_graph: PythonNodeGraph) -> DagCheck: - checked_asset_ids: tuple[str, ...] = tuple( - _python_node_id( - kind=python_graph.nodes_by_name[edge.upstream_name].kind, - node_name=edge.upstream_name, - ) - for edge in python_graph.dependency_edges - if edge.downstream_name == node.name - ) + tuple(_sql_ref_node_id(sql_dep) for sql_dep in node.sql_deps) - return DagCheck( - id=_python_node_id(kind=node.kind, node_name=node.name), - kind=NodeKind.PYTHON_CHECK.value, - name=node.name, - checked_asset_ids=checked_asset_ids, - path=str(node.relative_path), - description=node.description, - severity=(AuditSeverity(node.check.severity.value) if node.check is not None else None), - tags=node.tags, - group=node.group, - meta=node.meta or {}, - ) - - -def _dag_target(target: CompiledRelationLocation) -> DagTarget: - return DagTarget( - database=target.database, - schema=target.schema, - name=target.name, - qualified_name=target.qualified_name, - logical_database=target.logical_database, - logical_schema=target.logical_schema, - ) - - -def _target_asset_key(target: CompiledRelationLocation) -> tuple[str, ...]: - return tuple( - part - for part in ( - target.logical_database or target.database, - target.logical_schema or target.schema, - target.name, - ) - if part - ) - - -def _source_asset_key(entry: SourceEntry) -> tuple[str, ...]: - parts: tuple[str | None, ...] = (entry.database, entry.schema, entry.table or entry.name) - return tuple(part for part in parts if part) or (entry.name,) - - -def _node_id(key: CompiledObjectKey) -> str: - return f"{key.resource_type}:{key.name}" - - -def _loader_node_id(loader_name: str) -> str: - return f"loader:{loader_name}" - - -def _python_node_id(*, kind: PythonNodeKind, node_name: str) -> str: - if kind == PythonNodeKind.LOADER: - return _loader_node_id(node_name) - return f"{kind.value}:{node_name}" - - -def _sql_ref_node_id(ref: SqlResourceRef) -> str: - return _node_id(sql_ref_key(ref)) - - -def _source_by_loader(graph: ProjectGraph) -> dict[str, SourceEntry]: - return { - source.source_entry.loader: source.source_entry - for source in graph.project.sources - if source.source_entry.loader is not None - } - - -def _loader_to_source_entry( - *, project: CompiledProjectView, loader: DiscoveredLoaderFunction -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=project.effective_target_database, - default_schema=project.effective_target_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _schema_column(column: SchemaColumn) -> DagColumn: - return DagColumn( - name=column.name, - type=column.type, - nullable=column.nullable, - description=column.description, - meta=column.meta, - ) - - -def _source_column(column: SourceColumnEntry) -> DagColumn: - return DagColumn( - name=column.name, - type=column.type, - nullable=column.nullable, - description=column.description, - meta=column.meta, - ) - - -def _model_columns(model: CompiledModelView) -> tuple[DagColumn, ...]: - if model.schema_entry is not None and model.schema_entry.columns: - return tuple(_schema_column(column) for column in model.schema_entry.columns) - if model.inferred_columns is not None: - return tuple( - DagColumn(name=column.name, type=column.type) for column in model.inferred_columns - ) - return () - - -def _model_tags(model: CompiledModelView) -> tuple[str, ...]: - schema_tags: tuple[str, ...] = () - if model.schema_entry is not None: - schema_tags = model.schema_entry.tags - config_tags: object = model.config.values.get("tags", ()) - if isinstance(config_tags, (list, tuple)): - return tuple(dict.fromkeys((*schema_tags, *(str(tag) for tag in config_tags)))) - return schema_tags - - -def _python_column_lineage( - lineage: dict[str, tuple[ColumnLineageRef, ...]] | None, -) -> dict[str, tuple[DagColumnLineageRef, ...]]: - if lineage is None: - return {} - result: dict[str, tuple[DagColumnLineageRef, ...]] = {} - for column, refs in lineage.items(): - converted_refs: list[DagColumnLineageRef] = [] - for ref in refs: - converted_refs.append(DagColumnLineageRef(node=ref.node, column=ref.column)) - result[column] = tuple(converted_refs) - return result - - -def _qualified_name(*, database: str | None, schema: str | None, name: str) -> str: - return ".".join(part for part in (database, schema, name) if part) - - -def _drop_none(value: object) -> object: - if isinstance(value, StrEnum): - return value.value - if isinstance(value, dict): - result: dict[object, object] = {} - for key, item in value.items(): - if item is None: - continue - cleaned: object = _drop_none(item) - if _is_empty_optional_field(key=key, value=cleaned): - continue - result[key] = cleaned - return result - if isinstance(value, list): - return [_drop_none(item) for item in value] - if isinstance(value, tuple): - return [_drop_none(item) for item in value] - return value - - -def _is_empty_optional_field(*, key: object, value: object) -> bool: - if key in REQUIRED_DAG_COLLECTION_KEYS: - return False - return value in ({}, []) diff --git a/src/sqlbuild/compiler/dag/constants.py b/src/sqlbuild/compiler/dag/constants.py deleted file mode 100644 index 6e85088d9d..0000000000 --- a/src/sqlbuild/compiler/dag/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Static DAG artifact constants.""" - -REQUIRED_DAG_COLLECTION_KEYS: frozenset[str] = frozenset({"nodes", "edges", "checks"}) diff --git a/src/sqlbuild/compiler/dag/main/build.py b/src/sqlbuild/compiler/dag/main/build.py deleted file mode 100644 index 38e1f8f0b9..0000000000 --- a/src/sqlbuild/compiler/dag/main/build.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Build the static SQLBuild DAG JSON artifact.""" - -from __future__ import annotations - -from sqlbuild.compiler.dag._helpers.artifact import build_dag_artifact, format_dag_json -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - -def build_dag_json( - *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None -) -> str: - """Build and serialize a static SQLBuild DAG artifact.""" - - return format_dag_json( - artifact=build_dag_artifact( - graph=graph, - project_name=project_name, - python_graph=python_graph, - ) - ) diff --git a/src/sqlbuild/compiler/dag/models.py b/src/sqlbuild/compiler/dag/models.py deleted file mode 100644 index 65c74c8223..0000000000 --- a/src/sqlbuild/compiler/dag/models.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Static DAG artifact models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from sqlbuild.compiler.auditing.types import AuditSeverity - - -@dataclass(frozen=True) -class DagTarget: - """Warehouse identity for one SQLBuild DAG node.""" - - database: str | None = None - schema: str | None = None - name: str | None = None - qualified_name: str | None = None - logical_database: str | None = None - logical_schema: str | None = None - - -@dataclass(frozen=True) -class DagColumn: - """Column metadata attached to one DAG node.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DagColumnLineageRef: - """Declared upstream column reference for one DAG node column.""" - - node: str - column: str - - -@dataclass(frozen=True) -class DagFunctionArgument: - """Function argument metadata attached to one function node.""" - - name: str - type: str - - -@dataclass(frozen=True) -class DagNode: - """One asset-like node in the static SQLBuild DAG.""" - - id: str - kind: str - name: str - asset_key: tuple[str, ...] - target: DagTarget | None = None - path: str | None = None - description: str | None = None - sql: str | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[DagColumn, ...] = field(default_factory=tuple) - column_lineage: dict[str, tuple[DagColumnLineageRef, ...]] = field(default_factory=dict) - materialization_type: str | None = None - language: str | None = None - return_kind: str | None = None - returns: str | None = None - loader: str | None = None - arguments: tuple[DagFunctionArgument, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DagEdge: - """One static asset dependency edge.""" - - from_id: str - to_id: str - - -@dataclass(frozen=True) -class DagCheck: - """One check definition attached to one or more DAG assets.""" - - id: str - kind: str - name: str - checked_asset_ids: tuple[str, ...] - path: str | None = None - description: str | None = None - severity: AuditSeverity | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - meta: dict[str, object] = field(default_factory=dict) - attachment_kind: str | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - mode: str | None = None - assertion_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - graph_asset_ids: tuple[str, ...] = field(default_factory=tuple) - fixture_refs: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DagArtifact: - """Complete static SQLBuild DAG artifact.""" - - version: int - project_name: str - nodes: tuple[DagNode, ...] - edges: tuple[DagEdge, ...] - checks: tuple[DagCheck, ...] diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 23ce3b53b9..72cc814c11 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -5,230 +5,35 @@ from collections.abc import Callable from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( - discover_command_output_sink_functions, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_adapter_file, +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, - discover_event_exporter_functions, - discover_hook_functions, discover_macro_files, - discover_materialization_files, + discover_model_files, discover_model_schema_files, - discover_provider_classes, - discover_python_function_files, - discover_python_node_functions, - discover_seed_files, + discover_scenario_files, + discover_source_files, discover_sql_function_files, discover_sql_hook_files, -) -from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( - build_integration_loader_functions, -) -from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( - retained_discovery_session, -) -from sqlbuild.compiler.discovery._helpers.native.model_files import ( - discover_native_model_files, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, - discover_native_test_files, -) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_schema_files, - discover_native_source_files, + discover_test_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredAuditFactory, - DiscoveredCheckFunction, - DiscoveredCommandOutputSink, DiscoveredConstantFile, - DiscoveredDeclarationFiles, DiscoveredEnumFile, - DiscoveredEventExporter, - DiscoveredHookFunction, - DiscoveredLoaderFunction, DiscoveredMacroFile, - DiscoveredMaterializationFile, DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredPythonNodeFunctions, - DiscoveredSourceFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestFile, - DiscoveredTaskFunction, DiscoveryFileFault, TolerantScopeDiscovery, ) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -def build_discovered_project_inputs( - *, - project_dir: Path, - project_config: ProjectConfig, - local_config: LocalConfig, - sql_analysis_enabled: bool, - extract_output_column_locations: bool = True, -) -> DiscoveredProjectInputs: - """Discover all project files and functions into one inputs bundle.""" - - with OperationLifecycle( - operation_kind="project", operation_name="discovery_declaration_parse" - ) as declaration_lifecycle: - declarations: DiscoveredDeclarationFiles = _discover_declaration_files( - project_dir=project_dir, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - declaration_lifecycle.completed( - metadata={ - "item_count": sum( - len(files) - for files in ( - declarations.source_files, - declarations.model_files, - declarations.enum_files, - declarations.constant_files, - declarations.model_schema_files, - declarations.sql_function_files, - declarations.sql_hook_files, - declarations.python_function_files, - declarations.schema_files, - declarations.seed_files, - declarations.test_files, - declarations.scenario_files, - declarations.audit_files, - declarations.macro_files, - ) - ) - + int(declarations.adapter_file is not None) - } - ) - with OperationLifecycle( - operation_kind="project", operation_name="discovery_python_import" - ) as python_lifecycle: - providers: tuple[DiscoveredProvider, ...] = discover_provider_classes( - project_dir=project_dir - ) - python_nodes: DiscoveredPythonNodeFunctions = discover_python_node_functions( - project_dir=project_dir, - providers=providers, - ) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = ( - discover_materialization_files( - project_dir=project_dir, - providers=providers, - ) - ) - hook_functions: tuple[DiscoveredHookFunction, ...] = discover_hook_functions( - project_dir=project_dir, - providers=providers, - ) - event_exporters: tuple[DiscoveredEventExporter, ...] = discover_event_exporter_functions( - project_dir=project_dir, providers=providers - ) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ( - discover_command_output_sink_functions( - project_dir=project_dir, - providers=providers, - ) - ) - python_paths: set[Path] = {provider.relative_path for provider in providers} - python_paths.update(node.relative_path for node in python_nodes.loaders) - python_paths.update(node.relative_path for node in python_nodes.tasks) - python_paths.update(node.relative_path for node in python_nodes.assets) - python_paths.update(node.relative_path for node in python_nodes.checks) - python_paths.update(factory.relative_path for factory in python_nodes.audit_factories) - python_paths.update(file.relative_path for file in materialization_files) - python_paths.update(function.relative_path for function in hook_functions) - python_paths.update(exporter.relative_path for exporter in event_exporters) - python_paths.update(sink.relative_path for sink in command_output_sinks) - python_lifecycle.completed(metadata={"item_count": len(python_paths)}) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = tuple( - python_nodes.loaders - ) + build_integration_loader_functions(declarations.source_files) - task_functions: tuple[DiscoveredTaskFunction, ...] = tuple(python_nodes.tasks) - asset_functions: tuple[DiscoveredAssetFunction, ...] = tuple(python_nodes.assets) - check_functions: tuple[DiscoveredCheckFunction, ...] = tuple(python_nodes.checks) - audit_factories: tuple[DiscoveredAuditFactory, ...] = tuple(python_nodes.audit_factories) - return DiscoveredProjectInputs( - project_config=project_config, - local_config=local_config, - project_dir=project_dir, - model_files=declarations.model_files, - enum_files=declarations.enum_files, - constant_files=declarations.constant_files, - model_schema_files=declarations.model_schema_files, - sql_function_files=declarations.sql_function_files, - sql_hook_files=declarations.sql_hook_files, - python_function_files=declarations.python_function_files, - schema_files=declarations.schema_files, - source_files=declarations.source_files, - seed_files=declarations.seed_files, - test_files=declarations.test_files, - scenario_files=declarations.scenario_files, - audit_files=declarations.audit_files, - macro_files=declarations.macro_files, - materialization_files=materialization_files, - loader_functions=loader_functions, - task_functions=task_functions, - asset_functions=asset_functions, - check_functions=check_functions, - audit_factories=audit_factories, - hook_functions=hook_functions, - event_exporters=event_exporters, - command_output_sinks=command_output_sinks, - providers=providers, - adapter_file=declarations.adapter_file, - native_session=declarations.native_session, - ) - - -def _discover_declaration_files( - *, - project_dir: Path, - sql_analysis_enabled: bool, - extract_output_column_locations: bool, -) -> DiscoveredDeclarationFiles: - with DirectorySnapshot.scope(project_dir=project_dir): - source_files: tuple[DiscoveredSourceFile, ...] = discover_native_source_files( - project_dir=project_dir - ) - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - return DiscoveredDeclarationFiles( - source_files=source_files, - model_files=model_files, - enum_files=discover_enum_files(project_dir=project_dir), - constant_files=discover_constant_files(project_dir=project_dir), - model_schema_files=discover_model_schema_files(project_dir=project_dir), - sql_function_files=discover_sql_function_files(project_dir=project_dir), - sql_hook_files=discover_sql_hook_files(project_dir=project_dir), - python_function_files=discover_python_function_files(project_dir=project_dir), - schema_files=discover_native_schema_files(project_dir=project_dir), - seed_files=discover_seed_files(project_dir=project_dir), - test_files=discover_native_test_files(project_dir=project_dir), - scenario_files=discover_native_scenario_files(project_dir=project_dir), - audit_files=discover_audit_files(project_dir=project_dir), - macro_files=discover_macro_files(project_dir=project_dir), - adapter_file=discover_adapter_file(project_dir=project_dir), - native_session=retained_discovery_session(project_dir=project_dir), - ) - - def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" @@ -248,7 +53,7 @@ def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscov function=discover_sql_hook_files, project_dir=project_dir ) sources, source_faults = _discover_category( - function=discover_native_source_files, project_dir=project_dir + function=discover_source_files, project_dir=project_dir ) audits, audit_faults = _discover_category( function=discover_audit_files, project_dir=project_dir @@ -318,7 +123,7 @@ def _discover_models( *, project_dir: Path ) -> tuple[tuple[DiscoveredSqlModelFile, ...], tuple[DiscoveryFileFault, ...]]: faults: list[DiscoveryFileFault] = [] - models: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + models: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, @@ -380,10 +185,10 @@ def _discover_relationships( tuple[DiscoveryFileFault, ...], ]: faults: list[DiscoveryFileFault] = [] - tests: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( + tests: tuple[DiscoveredSqlTestFile, ...] = discover_test_files( project_dir=project_dir, on_fault=faults.append ) - scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_native_scenario_files( + scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_scenario_files( project_dir=project_dir, on_fault=faults.append ) return tests, scenarios, tuple(faults) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 90c3091091..d40bb0f531 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -21,16 +21,6 @@ from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( project_relative_path, ) -from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( - native_audit_files, - native_constant_files, - native_enum_files, - native_macro_files, - native_model_schema_files, - native_seed_files, - native_sql_function_files, - native_sql_hook_files, -) from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( @@ -43,30 +33,20 @@ PythonNodeDiscoveryError, ) from sqlbuild.compiler.discovery.models import ( - DiscoveredAdapterFile, DiscoveredAssetFunction, DiscoveredAuditFactory, - DiscoveredAuditFile, DiscoveredCheckFunction, DiscoveredCommandOutputSink, - DiscoveredConstantFile, - DiscoveredEnumFile, DiscoveredEventExporter, DiscoveredEventExporterDeclaration, DiscoveredHookFunction, DiscoveredLoaderFunction, - DiscoveredMacroFile, DiscoveredMaterializationFile, - DiscoveredModelSchemaFile, DiscoveredProvider, DiscoveredProviderUsage, DiscoveredPythonFunctionFile, DiscoveredPythonNodeFunctions, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, DiscoveredTaskFunction, - DiscoveryFileFault, NamedDeclarationRoot, ) from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( @@ -132,52 +112,6 @@ def add_audit_factory(self, item: DiscoveredAuditFactory) -> None: self.audit_factories.append(item) -def discover_enum_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - isolate_declaration_kind: bool = False, -) -> tuple[DiscoveredEnumFile, ...]: - """Discover global and scoped enum declaration files.""" - - return native_enum_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - on_fault=on_fault, - ) - - -def discover_constant_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - isolate_declaration_kind: bool = False, -) -> tuple[DiscoveredConstantFile, ...]: - """Discover global and scoped constant declaration files.""" - - return native_constant_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - on_fault=on_fault, - ) - - -def discover_model_schema_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredModelSchemaFile, ...]: - """Discover project-wide and scoped reusable model schemas.""" - - return native_model_schema_files(project_dir=project_dir, on_fault=on_fault) - - -def discover_sql_function_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlFunctionFile, ...]: - """Discover SQL function files under functions/sql/.""" - - return native_sql_function_files(project_dir=project_dir, on_fault=on_fault) - - def discover_python_function_files( *, project_dir: Path ) -> tuple[DiscoveredPythonFunctionFile, ...]: @@ -210,30 +144,6 @@ def discover_python_function_files( return tuple(discovered_function_files) -def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: - """Discover seed CSV files under seeds/.""" - - return native_seed_files(project_dir=project_dir) - - -def discover_audit_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredAuditFile, ...]: - """Discover generic and singular audit SQL files in their project-wide and scoped roles.""" - - return native_audit_files(project_dir=project_dir, on_fault=on_fault) - - -def discover_macro_files( - *, project_dir: Path, isolate_declaration_kind: bool = False -) -> tuple[DiscoveredMacroFile, ...]: - """Discover global and scoped project macro Python files.""" - - return native_macro_files( - project_dir=project_dir, isolate_declaration_kind=isolate_declaration_kind - ) - - def discover_materialization_files( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> tuple[DiscoveredMaterializationFile, ...]: @@ -344,14 +254,6 @@ def discover_hook_functions( return tuple(discovered_hooks) -def discover_sql_hook_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlHookFile, ...]: - """Discover named SQL lifecycle hooks in their project-wide and scoped roles.""" - - return native_sql_hook_files(project_dir=project_dir, on_fault=on_fault) - - def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: """Discover provider classes under providers/.""" @@ -1313,19 +1215,6 @@ def _load_materialization_module(*, file_path: Path, project_dir: Path) -> Modul return module -def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: - """Detect a project-level adapter.py without importing it.""" - - file_path: Path = project_dir / "adapter.py" - if not file_path.is_file(): - return None - - return DiscoveredAdapterFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - ) - - def _sorted_glob(*, project_dir: Path, root: Path, pattern: str) -> list[Path]: """Return ``sorted(root.rglob(pattern))`` from the active discovery snapshot.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index d833e2b626..4608352f51 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from pathlib import Path -from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_groups +import sqlbuild._native as _native from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import NamedDeclarationRoot from sqlbuild.compiler.scopes.constants import ( @@ -70,5 +70,5 @@ def named_declaration_files( def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Path], ...]: return tuple( (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) - for root, directory in native_declaration_groups(project_dir=project_dir) + for root, directory in _native.declaration_groups(project_dir=project_dir) ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py index 0db80edd8a..0757686bc3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py @@ -2,23 +2,8 @@ from __future__ import annotations -from functools import lru_cache from pathlib import Path -from sqlbuild.compiler.discovery.constants import ( - CANONICAL_AUTHORED_ROOTS, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - SCOPED_DECLARATION_DIRECTORIES, -) -from sqlbuild.compiler.scopes.types import DeclarationKind - -_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" -_SQL_FILE_SUFFIX: str = ".sql" -_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts - def project_relative_path(*, path: Path, project_dir: Path) -> Path: """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" @@ -31,38 +16,3 @@ def project_relative_path(*, path: Path, project_dir: Path) -> Path: ): return path.relative_to(project_dir) return path.with_segments(*path_parts[len(root_parts) :]) - - -def is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: - """Return whether a file lives inside a scoped declaration tree rather than a resource tree.""" - - return _is_scoped_declaration_directory( - directory=file_path.parent, - project_dir=project_dir, - sql_file=file_path.suffix == _SQL_FILE_SUFFIX, - ) - - -@lru_cache(maxsize=8192) -def _is_scoped_declaration_directory(*, directory: Path, project_dir: Path, sql_file: bool) -> bool: - """Classify a file's directory once; every file in it shares the same scoped-tree answer.""" - - directory_parts: tuple[str, ...] = project_relative_path( - path=directory, project_dir=project_dir - ).parts - root_components: tuple[str, ...] - for root_components in CANONICAL_AUTHORED_ROOTS: - if directory_parts[: len(root_components)] != root_components: - continue - scoped_components: tuple[str, ...] = directory_parts[len(root_components) :] - if ( - root_components == _SQL_TEST_ROOT_COMPONENTS - and sql_file - and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) - ): - return False - return any( - component == DECLARATION_GROUP_DIRECTORY or component in SCOPED_DECLARATION_DIRECTORIES - for component in scoped_components - ) - return False diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py deleted file mode 100644 index 2b80061b1d..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Declaration files discovered by the native engine, materialised as Python discovery records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.auditing.models import MeasurementContract -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_failure, - native_locations, - native_project_tree, - native_read_error, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_PARSED_TAG, - SQL_AUDIT_HEADER_KEYS, - SQL_FUNCTION_HEADER_KEYS, - SQL_HOOK_HEADER_KEYS, -) -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredMacroFile, - DiscoveredModelSchemaFile, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveryFileFault, - EnumDeclaration, - EnumMember, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.discovery.types import NativeFileScope, NativeLocation, NativeScopeFields -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind - -type _NativeDeclarationFile = tuple[str, NativeFileScope | None, tuple[object, ...]] -type _Build[RecordT] = Callable[..., RecordT] -type _EnumPayload = tuple[str, dict[str, object], str] -type _ConstantPayload = tuple[str, dict[str, object], str | None, str | None] -type _SchemaPayload = tuple[str, str | None, str | None, dict[str, object], list[NativeLocation]] - -_SESSION_MEMO_KEY: str = "native_discovery_session" - - -def native_enum_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredEnumFile, ...]: - """Discover the enum files natively; Python parses the files native parsing defers.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.ENUM, - isolate_kind=isolate_declaration_kind, - build=_enum_file, - on_fault=on_fault, - ) - - -def native_constant_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredConstantFile, ...]: - """Discover the constant files natively; Python normalises each constant's value.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.CONSTANT, - isolate_kind=isolate_declaration_kind, - build=_constant_file, - on_fault=on_fault, - ) - - -def native_model_schema_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredModelSchemaFile, ...]: - """Discover the reusable model schema files natively; Python parses each schema's columns.""" - - return _native_records( - project_dir=project_dir, - kind="model_schema", - build=_schema_file, - on_fault=on_fault, - ) - - -def native_sql_function_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredSqlFunctionFile, ...]: - """Discover the SQL function files natively.""" - - return _native_records( - project_dir=project_dir, - kind="sql_function", - build=_function_file, - on_fault=on_fault, - ) - - -def native_sql_hook_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredSqlHookFile, ...]: - """Discover the named SQL hook files natively.""" - - return _native_records( - project_dir=project_dir, - kind="sql_hook", - build=_hook_file, - on_fault=on_fault, - ) - - -def native_audit_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredAuditFile, ...]: - """Discover the generic and singular audit files natively.""" - - return _native_records( - project_dir=project_dir, - kind="audit", - build=_audit_file, - on_fault=on_fault, - ) - - -def native_macro_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, -) -> tuple[DiscoveredMacroFile, ...]: - """Discover and read the macro files natively; Python reads the files native reading defers.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.MACRO, - isolate_kind=isolate_declaration_kind, - build=_macro_file, - on_fault=None, - ) - - -def native_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: - """Discover the seed files natively.""" - - seeds: tuple[DiscoveredSeedFile, ...] = tuple( - DiscoveredSeedFile(file_path=project_dir / relative_path, relative_path=Path(relative_path)) - for relative_path, _scope, _payload in _native_files(project_dir=project_dir, kind="seed") - ) - return seeds - - -def retained_discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession | None: - """Return the native session this discovery pass read its declaration files with, if any.""" - - session: object = DirectorySnapshot.current(project_dir=project_dir).memo.get(_SESSION_MEMO_KEY) - return session if isinstance(session, _native.NativeDiscoverySession) else None - - -def _native_records[RecordT]( - *, - project_dir: Path, - kind: str, - build: _Build[RecordT], - on_fault: Callable[[DiscoveryFileFault], None] | None, - isolate_kind: bool = False, -) -> tuple[RecordT, ...]: - files: list[_NativeDeclarationFile] = _native_files( - project_dir=project_dir, kind=kind, isolate_kind=isolate_kind - ) - return materialise_native_files( - project_dir=project_dir, - files=((Path(relative_path), (scope, payload)) for relative_path, scope, payload in files), - build=lambda relative_path, item: _record( - project_dir=project_dir, - relative_path=relative_path, - item=item, - build=build, - ), - on_fault=on_fault, - ) - - -def _native_files( - *, project_dir: Path, kind: str, isolate_kind: bool = False -) -> list[_NativeDeclarationFile]: - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: list[_NativeDeclarationFile] = native_collection( - result=_discovery_session(project_dir=project_dir).collection(kind, tree, isolate_kind), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return files - - -def _discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession: - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - session: _native.NativeDiscoverySession | None = retained_discovery_session( - project_dir=project_dir - ) - if session is not None: - return session - created: _native.NativeDiscoverySession = _native.NativeDiscoverySession( - native_request( - project_dir=project_dir, - fields={ - "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), - "audit_keys": sorted(SQL_AUDIT_HEADER_KEYS), - "hook_keys": sorted(SQL_HOOK_HEADER_KEYS), - }, - ) - ) - snapshot.memo[_SESSION_MEMO_KEY] = created - return created - - -def _record[RecordT]( - *, - project_dir: Path, - relative_path: Path, - item: tuple[NativeFileScope | None, tuple[object, ...]], - build: _Build[RecordT], -) -> RecordT: - """Build a parsed file or raise Python's error for it.""" - - scope, payload = item - tag: object = payload[0] - if tag == NATIVE_PARSED_TAG: - return build( - project_dir=project_dir, relative_path=relative_path, scope=scope, payload=payload - ) - if tag == NATIVE_FAILED_TAG: - raise native_failure(payload) - raise native_read_error(payload=payload, file_path=project_dir / relative_path) - - -def _raise_partial_failure(failure: object) -> None: - if failure is not None: - raise native_failure(cast(tuple[object, ...], failure)) - - -def _scope_fields(scope: NativeFileScope | None) -> NativeScopeFields: - if scope is None: - return NativeScopeFields() - _kind, scope_kind, ownership_root, owning_path, declaration_root = scope - return NativeScopeFields( - scope_kind=ScopeKind(scope_kind), - ownership_root=_optional_path(ownership_root), - owning_path=_optional_path(owning_path), - declaration_root=_optional_path(declaration_root), - ) - - -def _optional_path(text: str | None) -> Path | None: - return None if text is None else Path(text) - - -def _audit_block(block: tuple[object, ...]) -> DiscoveredAuditBlock: - audit_index, values, sql_body, name, mode, measure_sql, evidence_sql = block - header_values: dict[str, object] = cast(dict[str, object], values) - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode(str(mode)) - return DiscoveredAuditBlock( - audit_index=cast(int, audit_index), - header_values=header_values, - sql_body=str(sql_body), - name=cast(str | None, name), - evaluation_mode=evaluation_mode, - measurement_contract=( - MeasurementContract( - value_column=cast(str, header_values["value"]), - sample_count_column=cast(str | None, header_values.get("sample_count")), - sample_unit=cast(str | None, header_values.get("sample_unit")), - ) - if evaluation_mode is AuditEvaluationMode.MEASUREMENT - else None - ), - measure_sql=cast(str | None, measure_sql), - evidence_sql=cast(str | None, evidence_sql), - ) - - -def _enum_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredEnumFile: - _tag, contents, declarations = payload - return DiscoveredEnumFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - declarations=tuple( - _enum_declaration(relative_path=relative_path, declaration=declaration) - for declaration in cast(list[_EnumPayload], declarations) - ), - **_scope_fields(scope), - ) - - -def _enum_declaration(*, relative_path: Path, declaration: _EnumPayload) -> EnumDeclaration: - name, members, scalar_type = declaration - projected: dict[str, object] = members - return EnumDeclaration( - name=name, - members=tuple( - EnumMember(name=member, value=cast(str | int, value)) - for member, value in projected.items() - ), - scalar_type=scalar_type, - relative_path=relative_path, - ) - - -def _constant_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredConstantFile: - _tag, contents, declarations, failure = payload - file_path: Path = project_dir / relative_path - constants: tuple[ConstantDeclaration, ...] = tuple( - _constant_declaration( - file_path=file_path, relative_path=relative_path, declaration=declaration - ) - for declaration in cast(list[_ConstantPayload], declarations) - ) - _raise_partial_failure(failure) - return DiscoveredConstantFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - declarations=constants, - **_scope_fields(scope), - ) - - -def _constant_declaration( - *, file_path: Path, relative_path: Path, declaration: _ConstantPayload -) -> ConstantDeclaration: - name, value, explicit_type, render_as = declaration - return cast( - ConstantDeclaration, - _native.typed_constant_record( - (name, value["value"]), (explicit_type, render_as), (file_path, relative_path) - ), - ) - - -def _schema_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredModelSchemaFile: - _tag, contents, declarations, failure = payload - file_path: Path = project_dir / relative_path - schemas: tuple[ModelSchemaDeclaration, ...] = tuple( - _schema_declaration( - file_path=file_path, relative_path=relative_path, declaration=declaration - ) - for declaration in cast(list[_SchemaPayload], declarations) - ) - _raise_partial_failure(failure) - return DiscoveredModelSchemaFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - declarations=schemas, - **_scope_fields(scope), - ) - - -def _schema_declaration( - *, file_path: Path, relative_path: Path, declaration: _SchemaPayload -) -> ModelSchemaDeclaration: - name, description, extends, columns, locations = declaration - return ModelSchemaDeclaration( - name=name, - description=description, - extends=extends, - columns=parse_schema_columns( - raw_columns=columns.get("columns"), - file_path=file_path, - label=f"schema '{name}'", - error_class=DeclarationParseError, - column_locations=native_locations(locations=locations, relative_path=relative_path), - require_columns=True, - ), - relative_path=relative_path, - ) - - -def _function_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredSqlFunctionFile: - _ = scope - _tag, contents, header_values, body_sql = payload - return DiscoveredSqlFunctionFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - header_values=cast(dict[str, object], header_values), - body_sql=str(body_sql), - ) - - -def _hook_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredSqlHookFile: - _tag, contents, header_values, sql_body, name, description = payload - return DiscoveredSqlHookFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - header_values=cast(dict[str, object], header_values), - sql_body=str(sql_body), - name=str(name), - description=cast(str | None, description), - **_scope_fields(scope), - ) - - -def _audit_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredAuditFile: - _tag, contents, blocks = payload - return DiscoveredAuditFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - blocks=tuple(_audit_block(block) for block in cast(list[tuple[object, ...]], blocks)), - declaration_kind=DeclarationKind(cast(NativeFileScope, scope)[0]), - **_scope_fields(scope), - ) - - -def _macro_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredMacroFile: - _tag, contents = payload - return DiscoveredMacroFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - **_scope_fields(scope), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py deleted file mode 100644 index f8f83add05..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py +++ /dev/null @@ -1,57 +0,0 @@ -"""The declaration layout walked and validated natively for the declaration collections.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_collection, - native_failure, - native_project_tree, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import NATIVE_FAILED_TAG -from sqlbuild.compiler.scopes.types import DeclarationKind - -_LAYOUT_MEMO_KEY: str = "native_declaration_layout" -_GROUPS_MEMO_KEY: str = "native_declaration_groups" - - -def native_declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: - """Return the `(canonical root, group directory)` pairs of a valid named layout.""" - - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - groups: object = snapshot.memo.get(_GROUPS_MEMO_KEY) - if groups is None: - _facts, groups = _layout(project_dir=project_dir, declaration_kind=None) - return cast(list[tuple[str, str]], _rows(cast(tuple[object, ...], groups))) - - -def _layout( - *, project_dir: Path, declaration_kind: DeclarationKind | None -) -> tuple[tuple[object, ...], tuple[object, ...]]: - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - memo_key: tuple[str, DeclarationKind | None] = (_LAYOUT_MEMO_KEY, declaration_kind) - cached: object = snapshot.memo.get(memo_key) - if cached is not None: - return cast(tuple[tuple[object, ...], tuple[object, ...]], cached) - tree: _native.NativeProjectTree = native_project_tree(project_dir) - layout: tuple[tuple[object, ...], tuple[object, ...]] = native_collection( - result=_native.discover_declaration_layout( - tree, None if declaration_kind is None else declaration_kind.value - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - snapshot.memo[memo_key] = layout - _ = snapshot.memo.setdefault(_GROUPS_MEMO_KEY, layout[1]) - return layout - - -def _rows(payload: tuple[object, ...]) -> object: - if payload[0] == NATIVE_FAILED_TAG: - raise native_failure(payload) - return payload[1] diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py deleted file mode 100644 index bb40b05506..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ /dev/null @@ -1,81 +0,0 @@ -"""SQL model files discovered by the native engine as native records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_payload_error, - native_project_tree, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - REMOVED_SQL_MODEL_HEADER_KEYS, - SQL_MODEL_HEADER_KEYS, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveryFileFault -from sqlbuild.compiler.discovery.types import NativeModelFiles - -_RAISED_TAG: str = "raised" - - -def discover_native_model_files( - *, - project_dir: Path, - extract_implicit_alias_columns: bool, - extract_output_column_locations: bool, - selected_model_names: frozenset[str] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, -) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover the (selected) model files natively, reporting failing files to `on_fault`.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: NativeModelFiles = native_collection( - result=_native.discover_model_files( - native_request( - project_dir=project_dir, - fields={ - "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), - "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), - "extract_implicit_alias_columns": extract_implicit_alias_columns, - "extract_output_column_locations": extract_output_column_locations, - }, - ), - tree, - project_dir, - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return materialise_native_files( - project_dir=project_dir, - files=( - (relative_path, item) - for relative_path, item in files - if selected_model_names is None or relative_path.stem in selected_model_names - ), - build=lambda relative_path, item: _model_file( - project_dir=project_dir, relative_path=relative_path, item=item - ), - on_fault=on_fault, - ) - - -def _model_file(*, project_dir: Path, relative_path: Path, item: object) -> DiscoveredSqlModelFile: - if isinstance(item, DiscoveredSqlModelFile): - return item - payload: tuple[object, ...] = cast(tuple[object, ...], item) - if payload[0] == _RAISED_TAG: - raise cast(Exception, payload[1]) - error: Exception | None = native_payload_error( - payload=payload, file_path=project_dir / relative_path - ) - raise error if error is not None else ValueError(payload) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py deleted file mode 100644 index b6255bcc15..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Shared decoding of the plain-data payloads native discovery returns.""" - -from __future__ import annotations - -import os -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_UNLISTABLE_TAG, - NATIVE_UNREADABLE_TAG, -) -from sqlbuild.compiler.discovery.exceptions import ( - DeclarationParseError, - DiscoveryConflictError, - DiscoveryError, - ModelSqlParseError, - ProjectPathError, - SchemaParseError, - SeedDiscoveryError, - SourceParseError, - SqlAuditParseError, - SqlHookParseError, - SqlScenarioParseError, - SqlTestParseError, -) -from sqlbuild.compiler.discovery.models import DiscoveryFileFault -from sqlbuild.compiler.discovery.types import ( - DirectorySnapshotEntry, - NativeListing, - NativeLocation, -) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.spec.contracts.models import SourceLocation - -_DISPLAY_PROBE: str = "_" -_PATH_ENCODING: str = "utf-8" -_DECODE_ENCODING: str = "utf-8" -_DECODE_READ_KIND: str = "decode" -_NATIVE_TREE_MEMO_KEY: str = "native_project_tree" -_PROJECT_PATH_KIND: str = "project_path" -_NAMES_ARE_BYTES: bool = os.supports_bytes_environ -_SURROGATE_ESCAPE_BASE: int = 0xDC00 -_ESCAPED_BYTE_FIRST: int = 0xDC80 -_ESCAPED_BYTE_LAST: int = 0xDCFF -_SURROGATE_FIRST: int = 0xD800 -_SURROGATE_LAST: int = 0xDFFF -_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = { - "model_sql": ModelSqlParseError, - "declaration": DeclarationParseError, - "sql_test": SqlTestParseError, - "sql_scenario": SqlScenarioParseError, - "schema": SchemaParseError, - "source": SourceParseError, - _PROJECT_PATH_KIND: ProjectPathError, - "sql_hook": SqlHookParseError, - "sql_audit": SqlAuditParseError, - "resource_identity": ResourceIdentityError, - "conflict": DiscoveryConflictError, - "seed": SeedDiscoveryError, -} - - -def native_display_prefix(project_dir: Path) -> str: - """Return the text `str(project_dir / name)` prints before `name`.""" - - return str(project_dir / _DISPLAY_PROBE)[: -len(_DISPLAY_PROBE)] - - -def native_failure(payload: tuple[object, ...]) -> DiscoveryError: - """Build the discovery error a native failure payload describes.""" - - _tag, kind, message, help_text = payload - error_class: type[DiscoveryError] = _FAILURE_CLASSES[str(kind)] - return error_class(str(message), help=None if help_text is None else str(help_text)) - - -def native_read_error(*, payload: tuple[object, ...], file_path: Path) -> Exception: - """Build the error Python's `read_text` raises for a file native discovery could not read.""" - - if payload[1] == _DECODE_READ_KIND: - _tag, _kind, data, start, end, reason = payload - return UnicodeDecodeError( - _DECODE_ENCODING, cast(bytes, data), cast(int, start), cast(int, end), str(reason) - ) - _tag, _kind, errno, message, winerror = payload - if winerror is not None: - return OSError(None, str(message), str(file_path), cast(int, winerror)) - if errno is None: - return OSError(str(message)) - return OSError(errno, os.strerror(cast(int, errno)), str(file_path)) - - -def native_payload_error(*, payload: tuple[object, ...], file_path: Path) -> Exception | None: - """Return the error a failed or unreadable file payload raises, or `None` for a parsed one.""" - - tag: object = payload[0] - if tag == NATIVE_FAILED_TAG: - return native_failure(payload) - if tag == NATIVE_UNREADABLE_TAG: - return native_read_error(payload=payload, file_path=file_path) - return None - - -def native_collection[ResultT]( - *, result: ResultT | tuple[object, ...], project_dir: Path -) -> ResultT: - """Return a native collection, raising the error of a collection that could not be read.""" - - if not isinstance(result, tuple) or not isinstance(result[0], str): - return cast(ResultT, result) - payload: tuple[object, ...] = result - if payload[0] == NATIVE_UNLISTABLE_TAG: - _tag, relative_path, read_payload = payload - raise native_read_error( - payload=cast(tuple[object, ...], read_payload), - file_path=project_dir / str(relative_path), - ) - raise native_failure(payload) - - -def materialise_native_files[ItemT, RecordT]( - *, - project_dir: Path, - files: Iterable[tuple[Path, ItemT]], - build: Callable[[Path, ItemT], RecordT], - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[RecordT, ...]: - """Build each native file payload, reporting and skipping a failing file when asked.""" - - records: list[RecordT] = [] - for relative_path, payload in files: - try: - records.append(build(relative_path, payload)) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - if on_fault is None: - raise - on_fault( - DiscoveryFileFault( - path=relative_path, message=str(error).replace(str(project_dir), ".") - ) - ) - return tuple(records) - - -def native_locations( - *, locations: list[NativeLocation], relative_path: Path -) -> dict[str, SourceLocation]: - """Build authored locations; a repeated name keeps its first position and last value.""" - - return { - name: SourceLocation( - path=relative_path, - line=line, - column=column, - end_line=end_line, - end_column=end_column, - ) - for name, line, column, end_line, end_column in locations - } - - -def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: - """Return the native listings shared by every native call of this discovery pass.""" - - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - tree: object = snapshot.memo.get(_NATIVE_TREE_MEMO_KEY) - if isinstance(tree, _native.NativeProjectTree): - return tree - for text in (str(project_dir), native_display_prefix(project_dir)): - if not native_path_text_supported(text): - raise ProjectPathError( - f"Project directory {_escaped(text)} is not valid UTF-8; move the project to a " - "directory whose path is valid UTF-8" - ) - created: _native.NativeProjectTree = _native.NativeProjectTree(str(project_dir)) - snapshot.memo[_NATIVE_TREE_MEMO_KEY] = created - return created - - -def native_request(*, project_dir: Path, fields: dict[str, object]) -> dict[str, object]: - """Return a native discovery request for the project.""" - - return { - "project_dir": str(project_dir), - "display_prefix": native_display_prefix(project_dir), - **fields, - } - - -def seed_snapshot_listings(*, project_dir: Path, tree: _native.NativeProjectTree) -> None: - """Share the native walk's listings with the pass's Python directory snapshot.""" - - listings: list[NativeListing] = tree.listings() - DirectorySnapshot.current(project_dir=project_dir).seed_listings( - {project_dir / directory: _snapshot_entries(entries) for directory, entries in listings} - ) - - -def _snapshot_entries( - entries: list[tuple[str, bool, bool]], -) -> tuple[DirectorySnapshotEntry, ...]: - return tuple( - DirectorySnapshotEntry(name=name, is_dir=is_dir, is_walkable_dir=walkable) - for name, is_dir, walkable in entries - ) - - -def native_path_text_supported(text: str) -> bool: - """Whether a path's text crosses to the native engine unchanged (no surrogate escapes).""" - - try: - _ = text.encode(_PATH_ENCODING) - except UnicodeEncodeError: - return False - return True - - -def native_unreadable_path_payload(*, project_dir: Path, relative_text: str) -> tuple[object, ...]: - """Return the per-file failure payload for a path native discovery cannot name, if any.""" - - if native_path_text_supported(relative_text): - return () - return ( - NATIVE_FAILED_TAG, - _PROJECT_PATH_KIND, - f"Project path {_escaped(str(project_dir / relative_text))} is not valid UTF-8; rename it " - "so SQLBuild can read it", - None, - ) - - -def _escaped(text: str) -> str: - return "".join(_escaped_character(character) for character in text) - - -def _escaped_character(character: str) -> str: - code_point: int = ord(character) - if _NAMES_ARE_BYTES and _ESCAPED_BYTE_FIRST <= code_point <= _ESCAPED_BYTE_LAST: - return f"\\x{code_point - _SURROGATE_ESCAPE_BASE:02x}" - if _SURROGATE_FIRST <= code_point <= _SURROGATE_LAST: - return f"\\u{code_point:04x}" - return character diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py deleted file mode 100644 index 432fbc7113..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py +++ /dev/null @@ -1,190 +0,0 @@ -"""SQL unit-test and scenario files discovered by the native engine, as Python records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_failure, - native_payload_error, - native_project_tree, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIO_HEADER_KEYS, - SQL_TEST_HEADER_KEYS, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, - DiscoveryFileFault, -) -from sqlbuild.compiler.discovery.types import NativeFiles - -type _NativeDiscovery = Callable[[dict[str, object], _native.NativeProjectTree], object] - - -def discover_native_test_files( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, -) -> tuple[DiscoveredSqlTestFile, ...]: - """Discover the (selected) SQL test files natively, reporting failing files to `on_fault`.""" - - files: NativeFiles = _native_files( - project_dir=project_dir, discover=_native.discover_sql_test_files - ) - return materialise_native_files( - project_dir=project_dir, - files=( - (Path(relative_path), payload) - for relative_path, payload in files - if selected_paths is None or (project_dir / relative_path).resolve() in selected_paths - ), - build=lambda relative_path, payload: _test_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def discover_native_scenario_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlScenarioFile, ...]: - """Discover scenario files natively, reporting failing files to `on_fault`.""" - - files: NativeFiles = _native_files( - project_dir=project_dir, discover=_native.discover_scenario_files - ) - return materialise_native_files( - project_dir=project_dir, - files=((Path(relative_path), payload) for relative_path, payload in files), - build=lambda relative_path, payload: _scenario_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def parse_native_sql_test_contents( - *, contents: str, file_path: Path -) -> tuple[DiscoveredSqlTestBlock, ...]: - """Parse in-memory SQL test contents into ordered blocks, as test discovery does.""" - - payload: tuple[object, ...] = _native.parse_sql_test_contents( - _text_request(file_path), contents - ) - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, _contents, native_blocks, block_failure = payload - return _test_blocks( - file_path=file_path, - native_blocks=cast(list[tuple[dict[str, object], str, tuple[int, int]]], native_blocks), - block_failure=cast(tuple[object, ...] | None, block_failure), - ) - - -def _text_request(file_path: Path) -> dict[str, object]: - return { - "file_path": str(file_path), - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - } - - -def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> NativeFiles: - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: NativeFiles = native_collection( - result=cast( - NativeFiles | tuple[object, ...], - discover( - native_request( - project_dir=project_dir, - fields={ - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - }, - ), - tree, - ), - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return files - - -def _test_file( - *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] -) -> DiscoveredSqlTestFile: - file_path: Path = project_dir / relative_path - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, native_contents, native_blocks, block_failure = payload - return DiscoveredSqlTestFile( - file_path=file_path, - relative_path=relative_path, - contents=str(native_contents), - blocks=_test_blocks( - file_path=file_path, - native_blocks=cast(list[tuple[dict[str, object], str, tuple[int, int]]], native_blocks), - block_failure=cast(tuple[object, ...] | None, block_failure), - ), - ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), - ) - - -def _test_blocks( - *, - file_path: Path, - native_blocks: list[tuple[dict[str, object], str, tuple[int, int]]], - block_failure: tuple[object, ...] | None, -) -> tuple[DiscoveredSqlTestBlock, ...]: - blocks: tuple[DiscoveredSqlTestBlock, ...] = tuple( - cast(list[DiscoveredSqlTestBlock], _native.sql_test_blocks(str(file_path), native_blocks)) - ) - if block_failure is not None: - raise native_failure(block_failure) - _native.validate_sql_test_block_names( - str(file_path), [(block.test_index, block.name) for block in blocks] - ) - return blocks - - -def _scenario_file( - *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] -) -> DiscoveredSqlScenarioFile: - return _scenario_file_from_payload( - payload=payload, file_path=project_dir / relative_path, relative_path=relative_path - ) - - -def _scenario_file_from_payload( - *, payload: tuple[object, ...], file_path: Path, relative_path: Path -) -> DiscoveredSqlScenarioFile: - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, contents, values, sql_body, sql_body_span = payload - return build_sql_scenario_file( - header_values=cast(dict[str, object], values), - sql_body=str(sql_body), - sql_body_span=cast(tuple[int, int], sql_body_span), - contents=str(contents), - file_path=file_path, - relative_path=relative_path, - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py deleted file mode 100644 index 0760756762..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Source and schema YAML files read and loaded by the native engine.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import project_relative_path -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_display_prefix, - native_payload_error, - native_project_tree, - native_unreadable_path_payload, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( - schema_file_paths, - source_file_paths, -) -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_SCHEMA_YAML_KIND, - NATIVE_SOURCE_YAML_KIND, - NATIVE_YAML_BATCH_BYTES, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSourceFile, - DiscoveryFileFault, -) - -type _Payload = tuple[object, ...] - - -def discover_native_source_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSourceFile, ...]: - """Discover source YAML files natively, reporting failing files to `on_fault`.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) - return materialise_native_files( - project_dir=project_dir, - files=( - (project_relative_path(path=file_path, project_dir=project_dir), payload) - for file_path, payload in _native_payloads( - project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SOURCE_YAML_KIND - ) - ), - build=lambda relative_path, payload: _source_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def discover_native_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: - """Discover model schema.yml and seed declaration files natively.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) - return tuple( - _schema_file(project_dir=project_dir, file_path=file_path, payload=payload) - for file_path, payload in _native_payloads( - project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SCHEMA_YAML_KIND - ) - ) - - -def _source_file( - *, project_dir: Path, relative_path: Path, payload: _Payload -) -> DiscoveredSourceFile: - file_path: Path = project_dir / relative_path - contents, loaded = _loaded(payload=payload, file_path=file_path) - return DiscoveredSourceFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), - ) - - -def _schema_file(*, project_dir: Path, file_path: Path, payload: _Payload) -> DiscoveredSchemaFile: - contents, loaded = _loaded(payload=payload, file_path=file_path) - model_entries, seed_entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) - return DiscoveredSchemaFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - model_entries=model_entries, - seed_entries=seed_entries, - ) - - -def _loaded(*, payload: _Payload, file_path: Path) -> tuple[str, object]: - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, contents, loaded = payload - return str(contents), loaded - - -def _relative_text(*, project_dir: Path, file_path: Path) -> str: - return project_relative_path(path=file_path, project_dir=project_dir).as_posix() - - -def _native_payloads( - *, project_dir: Path, file_paths: tuple[Path, ...], kind: str -) -> Iterator[tuple[Path, _Payload]]: - """Load the files natively in batches bounded by size, releasing each file once used.""" - - if not file_paths: - return - tree: _native.NativeProjectTree = native_project_tree(project_dir) - request: dict[str, object] = { - "project_dir": str(project_dir), - "display_prefix": native_display_prefix(project_dir), - "kind": kind, - } - for batch in _batches(file_paths=file_paths): - relative_texts: list[str] = [ - _relative_text(project_dir=project_dir, file_path=path) for path in batch - ] - unnamed: list[_Payload] = [ - native_unreadable_path_payload(project_dir=project_dir, relative_text=text) - for text in relative_texts - ] - payloads: list[_Payload] = native_collection( - result=_native.load_yaml_files( - request, - [ - text - for text, failure in zip(relative_texts, unnamed, strict=True) - if not failure - ], - tree, - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - payloads.reverse() - for file_path, failure in zip(batch, unnamed, strict=True): - yield file_path, failure or payloads.pop() - - -def _batches(*, file_paths: tuple[Path, ...]) -> Iterator[tuple[Path, ...]]: - batch: list[Path] = [] - batch_bytes: int = 0 - for file_path in file_paths: - size: int = _file_size(file_path) - if batch and batch_bytes + size > NATIVE_YAML_BATCH_BYTES: - yield tuple(batch) - batch, batch_bytes = [], 0 - batch.append(file_path) - batch_bytes += size - if batch: - yield tuple(batch) - - -def _file_size(file_path: Path) -> int: - try: - return file_path.stat().st_size - except OSError: - return 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py deleted file mode 100644 index 9ddb5b669c..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Parsing helpers for authored SQL scenario files.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIOS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile - -_SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" -_SCENARIO_TAGS_HEADER_KEY: str = "tags" - - -def build_sql_scenario_file( - *, - header_values: dict[str, object], - sql_body: str, - contents: str, - file_path: Path, - relative_path: Path, - sql_body_span: tuple[int, int] | None = None, -) -> DiscoveredSqlScenarioFile: - """Validate a header-parsed SCENARIO file with only supported keys and build its record.""" - - description_value: object | None = header_values.get(_SCENARIO_DESCRIPTION_HEADER_KEY) - if _SCENARIO_DESCRIPTION_HEADER_KEY in header_values and not isinstance(description_value, str): - raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") - tags_value: object | None = header_values.get(_SCENARIO_TAGS_HEADER_KEY) - if _SCENARIO_TAGS_HEADER_KEY in header_values and ( - not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) - ): - raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") - if not sql_body: - raise SqlScenarioParseError( - f"SQL scenario '{file_path}' must define SQL after SCENARIO(...)" - ) - - return DiscoveredSqlScenarioFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - header_values=header_values, - sql_body=sql_body, - sql_body_span=sql_body_span, - name=file_path.stem, - ownership_root=Path(SQL_SCENARIOS_OWNERSHIP_ROOT), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 7301212521..46d967b628 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -2,13 +2,10 @@ from __future__ import annotations -import csv from collections.abc import Callable -from pathlib import Path -from typing import Protocol, cast +from typing import Protocol import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import native_failure, native_read_error from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -35,10 +32,6 @@ class _NodeDefinition(Protocol): def name(self) -> str: ... -_UNREADABLE_TAG: str = "unreadable" -_CSV_ERROR_TAG: str = "csv" -_CSV_DIALECT_TAG: str = "csv_dialect" -_CSV_DIALECT_ERRORS: dict[str, type[Exception]] = {"TypeError": TypeError, "ValueError": ValueError} _DEFINITION_READERS: tuple[Callable[[Callable[..., object]], _NodeDefinition | None], ...] = ( read_loader_definition, read_task_definition, @@ -50,21 +43,7 @@ def name(self) -> str: ... def validate_discovered_inputs(discovered_inputs: DiscoveredProjectInputs) -> None: """Validate cross-file conflicts across discovered project inputs.""" - payload: tuple[object, ...] | None = _native.discovered_inputs_failure( - discovered_inputs, _nodes(discovered_inputs) - ) - if payload is None: - return - tag: object = payload[0] - if tag == _UNREADABLE_TAG: - raise native_read_error( - payload=cast(tuple[object, ...], payload[2]), file_path=Path(str(payload[1])) - ) - if tag == _CSV_ERROR_TAG: - raise csv.Error(str(payload[1])) - if tag == _CSV_DIALECT_TAG: - raise _CSV_DIALECT_ERRORS[str(payload[1])](str(payload[2])) - raise native_failure(payload) + _native.validate_discovered_project_inputs(discovered_inputs, _nodes(discovered_inputs)) def _nodes(inputs: DiscoveredProjectInputs) -> list[dict[str, object]]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py deleted file mode 100644 index 39c9289c7a..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py +++ /dev/null @@ -1,48 +0,0 @@ -"""The source and schema YAML files discovery reads, in Python's discovery order.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( - is_in_scoped_declaration_tree, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import SCHEMA_FILE_NAME, YAML_FILE_SUFFIXES -from sqlbuild.compiler.discovery.exceptions import SchemaParseError - - -def source_file_paths(*, project_dir: Path) -> tuple[Path, ...]: - """Return the YAML files directly below sources/, in path order.""" - - sources_root: Path = project_dir / "sources" - if not sources_root.is_dir(): - return () - return tuple( - sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) - ) - - -def schema_file_paths(*, project_dir: Path) -> tuple[Path, ...]: - """Return the unscoped model schema.yml files and seed declaration files, in Python's order.""" - - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - schema_paths: list[Path] = [] - models_root: Path = project_dir / "models" - seeds_root: Path = project_dir / "seeds" - - if models_root.is_dir(): - schema_paths.extend( - path - for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) - if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) - ) - if seeds_root.is_dir(): - yaml_path: Path - for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): - raise SchemaParseError( - f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " - ".yaml is not supported" - ) - schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) - return tuple(dict.fromkeys(schema_paths)) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py deleted file mode 100644 index c2d1cd2033..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Parsing of authored schema.yml files, run natively.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes -from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry - - -def parse_loaded_schema_yml( - *, loaded: object, file_path: Path -) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: - """Parse the document one schema.yml file loads to into raw model and seed metadata.""" - - return (), _native.parse_schema_document(loaded, str(file_path), yaml_entry_classes()) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py deleted file mode 100644 index c92d543598..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Parsing of authored sources/*.yml files, run natively.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes -from sqlbuild.spec.contracts.models import SourceEntry - - -def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[SourceEntry, ...]: - """Parse the document one sources/*.yml file loads to into raw source declarations.""" - - return _native.parse_sources_document(loaded, str(file_path), yaml_entry_classes()) diff --git a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py index 63b49da16b..0effa19175 100644 --- a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py +++ b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py @@ -4,12 +4,12 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files -from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_schema_files, - discover_native_source_files, +from sqlbuild._native import ( + discover_model_files, + discover_model_schema_files, + discover_schema_files, + discover_source_files, + discover_test_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, @@ -39,16 +39,16 @@ def discover( project_config=project_config, local_config=local_config, project_dir=project_dir, - model_files=discover_native_model_files( + model_files=discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, selected_model_names=referenced_model_names, ), model_schema_files=discover_model_schema_files(project_dir=project_dir), - schema_files=discover_native_schema_files(project_dir=project_dir), - source_files=discover_native_source_files(project_dir=project_dir), - test_files=discover_native_test_files( + schema_files=discover_schema_files(project_dir=project_dir), + source_files=discover_source_files(project_dir=project_dir), + test_files=discover_test_files( project_dir=project_dir, selected_paths=selected_test_paths, ), diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index cf4d52c5ac..8d4c1a6f0d 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -7,7 +7,6 @@ LEGACY_PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.yml" LEGACY_LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.yml" TOML_FILE_SUFFIX: str = ".toml" -SCHEMA_FILE_NAME: str = "schema.yml" YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) PYTHON_INIT_MODULE_STEM: str = "__init__" @@ -173,11 +172,3 @@ "description", ) SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") - -NATIVE_FAILED_TAG: str = "error" -NATIVE_UNREADABLE_TAG: str = "read" -NATIVE_UNLISTABLE_TAG: str = "unlistable" -NATIVE_PARSED_TAG: str = "ok" -NATIVE_SOURCE_YAML_KIND: str = "source" -NATIVE_SCHEMA_YAML_KIND: str = "schema" -NATIVE_YAML_BATCH_BYTES: int = 4 * 1024 * 1024 diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py index 4784e2421d..775e3be5b5 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -4,8 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild._native import discover_model_files, discover_model_schema_files from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault @@ -20,7 +19,7 @@ def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProject return DiscoveredProjectInputs( project_config=project_config, local_config=LocalConfig(), - model_files=discover_native_model_files( + model_files=discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, diff --git a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py b/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py deleted file mode 100644 index a91f696df2..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public discovery entry point for parsing in-memory SQL test contents.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - parse_native_sql_test_contents, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock - - -def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: - """Parse current SQL test contents into ordered test blocks.""" - - return parse_native_sql_test_contents(contents=contents, file_path=file_path) diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py deleted file mode 100644 index 5b387ef28a..0000000000 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Discovery entrypoints.""" - -from __future__ import annotations - -from functools import partial -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( - build_discovered_project_inputs, -) -from sqlbuild.compiler.discovery._helpers.filesystem.python_paths import ( - validate_project_python_paths, -) -from sqlbuild.compiler.discovery._helpers.validation.discovery import validate_discovered_inputs -from sqlbuild.compiler.discovery._helpers.validation.target_connections import ( - validate_target_connections, -) -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, - validate_local_sql_analysis_policy, - validate_target_warehouses_adapter, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier -from sqlbuild.compiler.frontier.types import CompilerStage -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def discover_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None = None, - extract_output_column_locations: bool = True, -) -> DiscoveredProjectInputs: - """Load all raw project inputs from disk before semantic resolution.""" - - with OperationLifecycle(operation_kind="project", operation_name="project_discovery"): - return compile_frontier( - until=CompilerStage.DISCOVERED_PROJECT_INPUTS, - python_stage=partial( - _discover_project_inputs, - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ), - ) - - -def _discover_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None, - extract_output_column_locations: bool, -) -> DiscoveredProjectInputs: - with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): - return _assemble_discovered_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ) - - -def _assemble_discovered_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None, - extract_output_column_locations: bool, -) -> DiscoveredProjectInputs: - project_config: ProjectConfig = load_project_config(project_dir=project_dir) - local_config: LocalConfig = load_local_config(project_dir=project_dir) - validate_local_sql_analysis_policy( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_target_warehouses_adapter( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_target_connections( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_project_python_paths(project_dir=project_dir) - sql_analysis_enabled: bool = ( - sql_analysis_enabled_override - if sql_analysis_enabled_override is not None - else ( - local_config.settings.sql_analysis - if SQL_ANALYSIS_SETTING_KEY in local_config.setting_overrides - else project_config.settings.sql_analysis - ) - ) - discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - validate_discovered_inputs(discovered_inputs) - from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( - configure_discovered_event_exporters, - ) - - _ = configure_discovered_event_exporters(discovered_inputs) - return discovered_inputs diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index a054a52dca..916cf3e92a 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -5,7 +5,7 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, TypedDict +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol from sqlbuild.compiler.scopes.types import ScopeKind @@ -13,11 +13,6 @@ from sqlbuild.provider.classes.provider import Provider type ProjectProvider = Provider -type NativeLocation = tuple[str, int, int, int, int] -type NativeListing = tuple[str, list[tuple[str, bool, bool]]] -type NativeFiles = list[tuple[str, tuple[object, ...]]] -type NativeModelFiles = list[tuple[Path, object]] -type NativeFileScope = tuple[str, str, str | None, str | None, str | None] class LoaderConnectionMode(StrEnum): @@ -37,15 +32,6 @@ class ScopedDeclarationFile(Protocol): declaration_root: Path | None -class NativeScopeFields(TypedDict, total=False): - """The scope fields of a declaration file record that native discovery listed in a role.""" - - scope_kind: ScopeKind - ownership_root: Path | None - owning_path: Path | None - declaration_root: Path | None - - class DirectorySnapshotEntry(NamedTuple): """One directory entry in a discovery snapshot.""" diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/latest.py b/src/sqlbuild/compiler/fingerprints/_helpers/latest.py deleted file mode 100644 index 7a360e9f9e..0000000000 --- a/src/sqlbuild/compiler/fingerprints/_helpers/latest.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Latest fingerprint set assembly shared by warehouse reads and in-memory selection.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet - - -def build_latest_fingerprint_set(*, schema: str, latest: Iterable[Fingerprint]) -> FingerprintSet: - """Index latest rows by identity and by node name, preferring models on name collisions.""" - - fingerprints: dict[str, Fingerprint] = {} - fingerprints_by_identity: dict[tuple[str, str], Fingerprint] = {} - fingerprint: Fingerprint - for fingerprint in latest: - fingerprints_by_identity[(fingerprint.node_type, fingerprint.node_name)] = fingerprint - if fingerprint.node_type == NODE_TYPE_MODEL or fingerprint.node_name not in fingerprints: - fingerprints[fingerprint.node_name] = fingerprint - return FingerprintSet( - schema=schema, - fingerprints=fingerprints, - fingerprints_by_identity=fingerprints_by_identity, - ) diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py index 2869b5325a..c87981494e 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py @@ -2,25 +2,11 @@ from __future__ import annotations -import base64 -from collections.abc import Callable, Sequence +from collections.abc import Callable +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.fingerprints.constants import ( - COLUMN_DEFINITION_B64, - COLUMN_DEFINITION_HASH, - COLUMN_METADATA_JSON_B64, - COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, - COLUMN_RUN_ID, - COLUMN_SCHEMA_FINGERPRINT, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_TIMESTAMP, - COLUMN_VERSION_HASH, - FINGERPRINT_TABLE_NAME, -) +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -57,23 +43,12 @@ def build_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, ) - string_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - return ( - f"CREATE TABLE IF NOT EXISTS {qualified_name} (" - f"{COLUMN_NODE_TYPE} {string_type} NOT NULL, " - f"{COLUMN_NODE_NAME} {string_type} NOT NULL, " - f"{COLUMN_TARGET_DATABASE} {string_type}, " - f"{COLUMN_TARGET_SCHEMA} {string_type}, " - f"{COLUMN_TARGET_NAME} {string_type}, " - f"{COLUMN_RUN_ID} {string_type} NOT NULL, " - f"{COLUMN_DEFINITION_HASH} {string_type} NOT NULL, " - f"{COLUMN_VERSION_HASH} {string_type} NOT NULL, " - f"{COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, " - f"{COLUMN_DEFINITION_B64} {string_type} NOT NULL, " - f"{COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, " - f"{COLUMN_TIMESTAMP} {timestamp_type} NOT NULL" - f")" + return _native.fingerprint_create_table_sql( + qualified_name, + ( + render_framework_type(FrameworkType.STRING), + render_framework_type(FrameworkType.TIMESTAMP), + ), ) @@ -82,23 +57,12 @@ def build_read_latest_sql( ) -> str: """Build a windowed SELECT for the latest fingerprint per node identity.""" - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - selected_columns: str = _fingerprint_select_columns() - return ( - f"SELECT {selected_columns} " - f"FROM (" - f"SELECT {selected_columns}, " - f"ROW_NUMBER() OVER (" - f"PARTITION BY {COLUMN_NODE_TYPE}, {COLUMN_NODE_NAME} " - f"ORDER BY {COLUMN_TIMESTAMP} DESC, {COLUMN_RUN_ID} DESC" - f") AS __sqlbuild_latest_rank " - f"FROM {qualified_name}" - f") AS __sqlbuild_latest_fingerprints " - f"WHERE __sqlbuild_latest_rank = 1" + return _native.fingerprint_read_latest_sql( + build_qualified_table_name( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + ) ) @@ -111,109 +75,11 @@ def build_insert_sql( ) -> str: """Build a complete INSERT statement for appending one fingerprint row.""" - return build_insert_many_sql( - database=database, - schema=schema, - fingerprints=(fingerprint,), - render_qualified_name=render_qualified_name, + return _native.fingerprint_insert_sql( + build_qualified_table_name( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + ), + fingerprint, ) - - -def build_insert_many_sql( - *, - database: str | None, - schema: str, - fingerprints: Sequence[Fingerprint], - render_qualified_name: Callable[..., str | None], -) -> str: - """Build one append-only INSERT for a non-empty fingerprint batch.""" - - if not fingerprints: - raise FingerprintInputError("fingerprint insert requires at least one row") - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - rows: str = ", ".join(_fingerprint_values(fingerprint) for fingerprint in fingerprints) - return ( - f"INSERT INTO {qualified_name} (" - f"{COLUMN_NODE_TYPE}, " - f"{COLUMN_NODE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_DEFINITION_HASH}, " - f"{COLUMN_VERSION_HASH}, " - f"{COLUMN_SCHEMA_FINGERPRINT}, " - f"{COLUMN_DEFINITION_B64}, " - f"{COLUMN_METADATA_JSON_B64}, " - f"{COLUMN_TIMESTAMP}" - f") VALUES {rows}" - ) - - -def _fingerprint_values(fingerprint: Fingerprint) -> str: - run_id: str = fingerprint.run_id - definition_hash: str = fingerprint.definition_hash - version_hash: str = fingerprint.version_hash - schema_fingerprint: str = fingerprint.schema_fingerprint - ts: str = fingerprint.ts.isoformat() - encoded_definition: str = _encode_definition_storage(fingerprint.definition).replace("'", "''") - encoded_metadata_json: str = _encode_definition_storage(fingerprint.metadata_json).replace( - "'", "''" - ) - node_type_literal: str = _required_string_literal(fingerprint.node_type) - node_name_literal: str = _required_string_literal(fingerprint.node_name) - target_database_literal: str = _optional_string_literal(fingerprint.target_database) - target_schema_literal: str = _optional_string_literal(fingerprint.target_schema) - target_name_literal: str = _optional_string_literal(fingerprint.target_name) - return ( - "(" - f"{node_type_literal}, " - f"{node_name_literal}, " - f"{target_database_literal}, " - f"{target_schema_literal}, " - f"{target_name_literal}, " - f"'{run_id}', " - f"'{definition_hash}', " - f"'{version_hash}', " - f"'{schema_fingerprint}', " - f"'{encoded_definition}', " - f"'{encoded_metadata_json}', " - f"'{ts}'" - f")" - ) - - -def _fingerprint_select_columns() -> str: - return ( - f"{COLUMN_NODE_TYPE}, " - f"{COLUMN_NODE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_DEFINITION_HASH}, " - f"{COLUMN_VERSION_HASH}, " - f"{COLUMN_SCHEMA_FINGERPRINT}, " - f"{COLUMN_DEFINITION_B64}, " - f"{COLUMN_METADATA_JSON_B64}, " - f"{COLUMN_TIMESTAMP}" - ) - - -def _optional_string_literal(value: str | None) -> str: - if value is None: - return "NULL" - return _required_string_literal(value) - - -def _required_string_literal(value: str) -> str: - escaped_value: str = value.replace("'", "''") - return f"'{escaped_value}'" - - -def _encode_definition_storage(value: str) -> str: - return base64.b64encode(value.encode("utf-8")).decode("ascii") diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py b/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py index 2fdd597deb..1db5155485 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py @@ -7,11 +7,11 @@ from collections.abc import Callable, Sequence from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType from sqlbuild.compiler.fingerprints._helpers.sql import ( build_create_table_sql, - build_insert_many_sql, - build_insert_sql, + build_qualified_table_name, ) from sqlbuild.compiler.fingerprints.constants import ( FINGERPRINT_WRITE_ATTEMPTS, @@ -19,7 +19,6 @@ FINGERPRINT_WRITE_BATCH_SIZE, FINGERPRINT_WRITE_RETRY_BASE_SECONDS, ) -from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.fingerprints.types import FingerprintWriteProgress @@ -60,23 +59,19 @@ def write_fingerprints_impl( sqls=(create_sql, *index_sqls), fingerprints=fingerprints, ) - batches: tuple[tuple[Fingerprint, ...], ...] = _fingerprint_batches( - database=database, - schema=schema, - fingerprints=fingerprints, - render_qualified_name=render_qualified_name, - batch_size=FINGERPRINT_WRITE_BATCH_SIZE, - batch_max_bytes=FINGERPRINT_WRITE_BATCH_MAX_BYTES, - ) - completed_count: int = 0 - batch: tuple[Fingerprint, ...] - for batch in batches: - insert_sql: str = build_insert_many_sql( + batches: list[tuple[tuple[Fingerprint, ...], str]] = _native.fingerprint_insert_batches( + build_qualified_table_name( database=database, schema=schema, - fingerprints=batch, render_qualified_name=render_qualified_name, - ) + ), + list(fingerprints), + (FINGERPRINT_WRITE_BATCH_SIZE, FINGERPRINT_WRITE_BATCH_MAX_BYTES), + ) + completed_count: int = 0 + batch: tuple[Fingerprint, ...] + insert_sql: str + for batch, insert_sql in batches: _ = _execute_with_retry( connection=connection, execute=execute, @@ -88,45 +83,6 @@ def write_fingerprints_impl( on_progress(completed=completed_count, total=len(fingerprints)) -def _fingerprint_batches( - *, - database: str | None, - schema: str, - fingerprints: Sequence[Fingerprint], - render_qualified_name: Callable[..., str | None], - batch_size: int, - batch_max_bytes: int, -) -> tuple[tuple[Fingerprint, ...], ...]: - if batch_size < 1: - raise FingerprintInputError("fingerprint batch size must be positive") - if batch_max_bytes < 1: - raise FingerprintInputError("fingerprint batch byte limit must be positive") - batches: list[tuple[Fingerprint, ...]] = [] - current_batch: list[Fingerprint] = [] - current_bytes: int = 0 - fingerprint: Fingerprint - for fingerprint in fingerprints: - single_row_bytes: int = len( - build_insert_sql( - database=database, - schema=schema, - fingerprint=fingerprint, - render_qualified_name=render_qualified_name, - ).encode("utf-8") - ) - if current_batch and ( - len(current_batch) >= batch_size or current_bytes + single_row_bytes > batch_max_bytes - ): - batches.append(tuple(current_batch)) - current_batch = [] - current_bytes = 0 - current_batch.append(fingerprint) - current_bytes += single_row_bytes - if current_batch: - batches.append(tuple(current_batch)) - return tuple(batches) - - def _execute_with_retry( *, connection: Any, diff --git a/src/sqlbuild/compiler/fingerprints/main/_select_latest.py b/src/sqlbuild/compiler/fingerprints/main/_select_latest.py index 656c02090b..9e78817131 100644 --- a/src/sqlbuild/compiler/fingerprints/main/_select_latest.py +++ b/src/sqlbuild/compiler/fingerprints/main/_select_latest.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.fingerprints._helpers.latest import build_latest_fingerprint_set +import sqlbuild._native as _native from sqlbuild.compiler.fingerprints.models import FingerprintSet @@ -14,15 +14,4 @@ def select_latest_fingerprints( ) -> FingerprintSet: """Apply the ``read_latest_fingerprints`` node filter to an unfiltered latest read.""" - if not node_names and not filtered_node_types: - return FingerprintSet(schema=fingerprint_set.schema, fingerprints={}) - names: frozenset[str] = frozenset(node_names) - types: frozenset[str] = frozenset(filtered_node_types) - return build_latest_fingerprint_set( - schema=fingerprint_set.schema, - latest=( - fingerprint - for fingerprint in (fingerprint_set.fingerprints_by_identity or {}).values() - if fingerprint.node_name in names or (types and fingerprint.node_type not in types) - ), - ) + return _native.select_latest_fingerprints(fingerprint_set, node_names, filtered_node_types) diff --git a/src/sqlbuild/compiler/fingerprints/main/read.py b/src/sqlbuild/compiler/fingerprints/main/read.py index df1a170199..86cce8f3e9 100644 --- a/src/sqlbuild/compiler/fingerprints/main/read.py +++ b/src/sqlbuild/compiler/fingerprints/main/read.py @@ -2,19 +2,16 @@ from __future__ import annotations -import base64 -import binascii from collections.abc import Callable -from datetime import datetime from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.fingerprints._helpers.latest import build_latest_fingerprint_set from sqlbuild.compiler.fingerprints._helpers.sql import ( build_qualified_table_name, ) from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.fingerprints.models import FingerprintSet def read_latest_fingerprints( @@ -46,28 +43,13 @@ def read_latest_fingerprints( database=database, schema=schema, ) - if node_names is not None: - if not node_names and not filtered_node_types: - return FingerprintSet(schema=schema, fingerprints={}) - relevant_predicate: str = "" - if node_names: - literals: str = ", ".join( - "'" + node_name.replace("'", "''") + "'" for node_name in node_names - ) - relevant_predicate = f"node_name IN ({literals})" - if filtered_node_types: - type_literals: str = ", ".join( - "'" + node_type.replace("'", "''") + "'" for node_type in filtered_node_types - ) - python_identity_predicate: str = f"node_type NOT IN ({type_literals})" - relevant_predicate = ( - f"{relevant_predicate} OR {python_identity_predicate}" - if relevant_predicate - else python_identity_predicate - ) - read_sql = f"SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE {relevant_predicate}" + relevant_sql: str | None = _native.fingerprint_relevant_read_sql( + read_sql, node_names, filtered_node_types + ) + if relevant_sql is None: + return FingerprintSet(schema=schema, fingerprints={}) try: - result: Any = execute(connection=connection, sql=read_sql) + result: Any = execute(connection=connection, sql=relevant_sql) except Exception as error: raise FingerprintInputError( f"Unable to read fingerprints from {qualified_name}. This can happen after " @@ -75,46 +57,4 @@ def read_latest_fingerprints( "fingerprint table to regenerate fingerprints." ) from error rows: list[tuple[Any, ...]] = result.fetchall() - return build_latest_fingerprint_set( - schema=schema, - latest=(_row_to_fingerprint(row=row, qualified_name=qualified_name) for row in rows), - ) - - -def _row_to_fingerprint(*, row: tuple[Any, ...], qualified_name: str) -> Fingerprint: - raw_ts: Any = row[11] - ts: datetime = raw_ts if isinstance(raw_ts, datetime) else datetime.fromisoformat(str(raw_ts)) - node_type: str = str(row[0]) - node_name: str = str(row[1]) - raw_target_database: Any = row[2] - raw_target_schema: Any = row[3] - raw_target_name: Any = row[4] - definition_storage: str = str(row[9]) - metadata_json_storage: str = str(row[10]) - try: - definition: str = base64.b64decode( - definition_storage.encode("ascii"), validate=True - ).decode("utf-8") - metadata_json: str = base64.b64decode( - metadata_json_storage.encode("ascii"), validate=True - ).decode("utf-8") - except (binascii.Error, UnicodeDecodeError) as error: - raise FingerprintInputError( - f"Invalid fingerprint definition storage for '{node_name}' in {qualified_name}: " - "expected base64-encoded UTF-8. This can happen after upgrading from an older " - f"sqlbuild version; delete or rebuild {qualified_name} to regenerate fingerprints." - ) from error - return Fingerprint( - node_type=node_type, - node_name=node_name, - target_database=str(raw_target_database) if raw_target_database is not None else None, - target_schema=str(raw_target_schema) if raw_target_schema is not None else None, - target_name=str(raw_target_name) if raw_target_name is not None else None, - run_id=str(row[5]), - definition_hash=str(row[6]), - version_hash=str(row[7]), - schema_fingerprint=str(row[8]), - definition=definition, - metadata_json=metadata_json, - ts=ts, - ) + return _native.fingerprint_set_from_rows(schema, qualified_name, rows) diff --git a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py deleted file mode 100644 index 193ed835e1..0000000000 --- a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Ordering edges that let attached audits gate their target.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.graph.models import AttachedAuditGateEdge - -_READABLE_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( - { - CompiledResourceType.MODEL, - CompiledResourceType.SEED, - CompiledResourceType.SOURCE, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - } -) - - -@dataclass(frozen=True) -class _AuditGate: - audit_name: str - target_kind: AttachedAuditTargetKind - target: CompiledObjectKey - reads: tuple[CompiledObjectKey, ...] - - -def attached_audit_gate_edges_impl( - *, project: CompiledProjectView -) -> tuple[AttachedAuditGateEdge, ...]: - """Return edges from each extra audit read to its model or seed target, or source triggers.""" - - source_triggers: dict[CompiledObjectKey, set[CompiledObjectKey]] = {} - for model in project.models: - for dep in model.deps: - if dep.resource_type == CompiledResourceType.SOURCE: - source_triggers.setdefault(dep, set()).add(model.key) - gates: tuple[_AuditGate, ...] = _audit_gates(project=project) - edges: dict[AttachedAuditGateEdge, None] = {} - changed: bool = True - while changed: - changed = False - gate: _AuditGate - for gate in gates: - edge: AttachedAuditGateEdge - for edge in _gate_edges(gate=gate, source_triggers=source_triggers): - if edge in edges: - continue - edges[edge] = None - changed = True - if ( - edge.read.resource_type == CompiledResourceType.SOURCE - and edge.gated.resource_type == CompiledResourceType.MODEL - ): - source_triggers.setdefault(edge.read, set()).add(edge.gated) - return tuple(edges) - - -def _audit_gates(*, project: CompiledProjectView) -> tuple[_AuditGate, ...]: - gates: list[_AuditGate] = [] - audit: CompiledAuditView - for audit in project.audits: - if audit.attached_target_kind is None or audit.attached_target_name is None: - continue - target_kind: AttachedAuditTargetKind = AttachedAuditTargetKind(audit.attached_target_kind) - target: CompiledObjectKey = CompiledObjectKey( - resource_type=target_kind.resource_type, name=audit.attached_target_name - ) - reads: tuple[CompiledObjectKey, ...] = _extra_reads(audit=audit, target=target) - if reads: - gates.append( - _AuditGate( - audit_name=audit.name, target_kind=target_kind, target=target, reads=reads - ) - ) - return tuple(gates) - - -def _gate_edges( - *, - gate: _AuditGate, - source_triggers: dict[CompiledObjectKey, set[CompiledObjectKey]], -) -> tuple[AttachedAuditGateEdge, ...]: - gated_keys: tuple[CompiledObjectKey, ...] = ( - tuple(sorted(source_triggers.get(gate.target, ()), key=lambda key: key.name)) - if gate.target_kind is AttachedAuditTargetKind.SOURCE - else (gate.target,) - ) - edges: list[AttachedAuditGateEdge] = [] - read: CompiledObjectKey - for read in gate.reads: - edges.extend( - AttachedAuditGateEdge( - audit_name=gate.audit_name, target=gate.target, gated=gated, read=read - ) - for gated in gated_keys - ) - return tuple(edges) - - -def _extra_reads( - *, audit: CompiledAuditView, target: CompiledObjectKey -) -> tuple[CompiledObjectKey, ...]: - return tuple( - read - for read in audit.scope_deps - if read != target and read.resource_type in _READABLE_RESOURCE_TYPES - ) diff --git a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py new file mode 100644 index 0000000000..d2180b5c69 --- /dev/null +++ b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py @@ -0,0 +1,141 @@ +"""Ordering edges attached audits and model hooks add, computed natively from project facts.""" + +from __future__ import annotations + +from itertools import chain +from pathlib import Path + +import orjson + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledAuditView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge +from sqlbuild.compiler.graph.types import HookReadType, NativeKey +from sqlbuild.python_nodes.models import SqlResourceRef + +_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") + + +def model_python_hook_names_impl(model: CompiledModelView) -> tuple[str, ...]: + """Return the Python hooks attached to one model, in lifecycle order.""" + + names: list[str] = [] + for hook_key in _MODEL_HOOK_KEYS: + entries: object = model.config.values.get(hook_key) + if isinstance(entries, PythonHookEntry): + names.append(entries.name) + elif isinstance(entries, list | tuple): + names.extend(entry.name for entry in entries if isinstance(entry, PythonHookEntry)) + return tuple(dict.fromkeys(names)) + + +def attached_audit_gate_edges_impl( + *, project: CompiledProjectView +) -> tuple[AttachedAuditGateEdge, ...]: + """Return edges from each extra audit read to its model or seed target, or source triggers.""" + + gates, _ = _native.native_execution_gate_edges(_request(project)) + return tuple( + AttachedAuditGateEdge( + audit_name=audit_name, target=_key(target), gated=_key(gated), read=_key(read) + ) + for audit_name, target, gated, read in gates + ) + + +def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: + """Return edges from each hook read to the model that runs the hook.""" + + _, hooks = _native.native_execution_gate_edges(_request(project)) + return tuple( + HookReadEdge( + hook_name=hook_name, + gated=_key(gated), + read=_key(read), + hook_type=HookReadType(hook_type), + hook_path=None if path is None else Path(path), + ) + for hook_name, gated, read, hook_type, path in hooks + ) + + +def _key(pair: NativeKey) -> CompiledObjectKey: + return CompiledObjectKey(resource_type=CompiledResourceType(pair[0]), name=pair[1]) + + +def _request(project: CompiledProjectView) -> str: + return orjson.dumps( + { + "resources": [ + {"key": model.key_row, "deps": model.dep_rows} for model in project.models + ], + "models": [_model_row(model) for model in project.models], + "audits": [_audit_row(audit) for audit in project.audits], + "hooks": [ + { + "name": hook.name, + "path": hook.relative_path.as_posix(), + "reads": _ref_rows(hook.reads), + } + for hook in project.hook_functions + ], + } + ).decode() + + +def _model_row(model: CompiledModelView) -> dict[str, object]: + return { + "name": model.name, + "path": str(model.relative_path), + "python_hooks": list(model_python_hook_names_impl(model)), + "sql_hooks": list( + chain.from_iterable( + _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) + for hook_key in _MODEL_HOOK_KEYS + ) + ), + } + + +def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: + items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) + return [ + { + "hook_key": hook_key, + "index": index, + "name": entry.name, + "path": None if entry.relative_path is None else str(entry.relative_path), + "reads": _ref_rows(entry.reads), + } + for index, entry in enumerate(items) + if isinstance(entry, SqlHookEntry) and entry.reads + ] + + +def _audit_row(audit: CompiledAuditView) -> dict[str, object]: + target: tuple[str, str] | None = ( + ( + str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), + audit.attached_target_name, + ) + if audit.attached_target_kind is not None and audit.attached_target_name is not None + else None + ) + return { + "name": audit.name, + "path": "", + "target": target, + "scope_deps": [(str(key.resource_type), key.name) for key in audit.scope_deps], + } + + +def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: + return [(str(ref.kind), ref.name) for ref in refs] diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py deleted file mode 100644 index 003c47ceb4..0000000000 --- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Ordering edges from the resources model hooks read.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl -from sqlbuild.compiler.graph.models import HookReadEdge -from sqlbuild.compiler.graph.types import HookReadType -from sqlbuild.python_nodes.models import SqlResourceRef - -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") - - -def model_python_hook_names_impl(model: CompiledModelView) -> tuple[str, ...]: - """Return the Python hooks attached to one model, in lifecycle order.""" - - names: list[str] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - if isinstance(entries, PythonHookEntry): - names.append(entries.name) - elif isinstance(entries, list | tuple): - names.extend(entry.name for entry in entries if isinstance(entry, PythonHookEntry)) - return tuple(dict.fromkeys(names)) - - -def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: - """Return edges from each hook read to the model that runs the hook.""" - - reads_by_hook: dict[str, tuple[SqlResourceRef, ...]] = { - hook.name: hook.reads for hook in project.hook_functions if hook.reads - } - paths_by_hook: dict[str, Path] = { - hook.name: hook.relative_path for hook in project.hook_functions - } - candidates: list[HookReadEdge] = [] - for model in project.models: - for hook_name in model_python_hook_names_impl(model): - candidates.extend( - HookReadEdge( - hook_name=hook_name, - gated=model.key, - read=sql_ref_key_impl(ref), - hook_path=paths_by_hook.get(hook_name), - ) - for ref in reads_by_hook.get(hook_name, ()) - ) - candidates.extend(_sql_hook_read_edges(model)) - return tuple(dict.fromkeys(edge for edge in candidates if edge.read != edge.gated)) - - -def _sql_hook_read_edges(model: CompiledModelView) -> tuple[HookReadEdge, ...]: - edges: list[HookReadEdge] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - items: tuple[object, ...] = ( - tuple(entries) if isinstance(entries, list | tuple) else (entries,) - ) - for index, entry in enumerate(items): - if not isinstance(entry, SqlHookEntry) or not entry.reads: - continue - named: bool = entry.name is not None - edges.extend( - HookReadEdge( - hook_name=entry.name if entry.name is not None else f"{hook_key}[{index}]", - gated=model.key, - read=sql_ref_key_impl(ref), - hook_type=HookReadType.SQL if named else HookReadType.INLINE_SQL, - hook_path=entry.relative_path or model.relative_path, - ) - for ref in entry.reads - ) - return tuple(edges) diff --git a/src/sqlbuild/compiler/graph/_helpers/native_graph.py b/src/sqlbuild/compiler/graph/_helpers/native_graph.py index 9821d8e8dd..d22ce9832d 100644 --- a/src/sqlbuild/compiler/graph/_helpers/native_graph.py +++ b/src/sqlbuild/compiler/graph/_helpers/native_graph.py @@ -43,12 +43,6 @@ def native_keys(keys: Iterable[CompiledObjectKey]) -> list[NativeKey]: return [native_key(key) for key in keys] -def build_native_project_graph_impl(project: CompiledProjectView) -> _native.NativeProjectGraph: - """The project's native graph, built once per compiled project.""" - - return project.lineage_graph - - def project_lineage_views_impl(project: CompiledProjectView) -> LineageGraphViews: """The project's lineage edges and selector indexes as fresh dicts.""" diff --git a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py index d9ab03745d..292bd4d4bb 100644 --- a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py +++ b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl +from sqlbuild.compiler.graph._helpers.execution_gates import attached_audit_gate_edges_impl from sqlbuild.compiler.graph.models import AttachedAuditGateEdge diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py index 3dffa67abe..9d4fdaf66c 100644 --- a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py +++ b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_edges_impl +from sqlbuild.compiler.graph._helpers.execution_gates import hook_read_edges_impl from sqlbuild.compiler.graph.models import HookReadEdge diff --git a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py index b80abfb8c6..2a39037d7c 100644 --- a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py +++ b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.graph._helpers.hook_reads import model_python_hook_names_impl +from sqlbuild.compiler.graph._helpers.execution_gates import model_python_hook_names_impl def model_python_hook_names(*, model: CompiledModelView) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/graph/main/_native_project_graph.py b/src/sqlbuild/compiler/graph/main/_native_project_graph.py deleted file mode 100644 index 5032d11637..0000000000 --- a/src/sqlbuild/compiler/graph/main/_native_project_graph.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Native project graph entrypoint.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.native_graph import build_native_project_graph_impl - - -def build_native_project_graph(project: CompiledProjectView) -> _native.NativeProjectGraph: - """Index the project's lineage edges, tags, folders and names natively.""" - - return build_native_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/lineage/_helpers/__init__.py b/src/sqlbuild/compiler/lineage/_helpers/__init__.py deleted file mode 100644 index efd4cc4374..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Column lineage helper utilities.""" diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py deleted file mode 100644 index 48e0931995..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Fast column lineage built by the native engine, assembled exactly as Python assembles it.""" - -from __future__ import annotations - -import logging -from collections.abc import Sequence - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage.constants import ( - NATIVE_LINEAGE_BUILT, - NATIVE_LINEAGE_UNKNOWN_DIALECT, - NATIVE_LINEAGE_UNPARSED, -) -from sqlbuild.compiler.lineage.exceptions import UnknownLineageDialectError -from sqlbuild.compiler.lineage.models import ( - ColumnLineage, - ColumnLineageSource, - ModelColumnLineage, - ProjectColumnLineage, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") - -type _NativeColumn = tuple[str, str, str, list[tuple[str, str, str]]] -type _NativeOutcome = tuple[str, list[_NativeColumn], bool, str | None] - - -def build_native_fast_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None, - model_names: frozenset[str] | None, -) -> ProjectColumnLineage | None: - """Build the fast lineage graph with native star expansion and parsing.""" - - if not project.settings.sql_analysis: - return None - models: tuple[CompiledModelView, ...] = tuple( - model for model in project.models if model_names is None or model.name in model_names - ) - requests: list[tuple[bool, str, list[str]]] = [] - for model in models: - request: tuple[bool, str, list[str]] | None = _native_request(model) - if request is not None: - requests.append(request) - catalog: object | None = ( - project.binding_catalog.native if project.binding_catalog is not None else None - ) - outcomes: list[_NativeOutcome] = ( - _native.build_fast_column_lineage(catalog, (dialect, _schema_resources(project), requests)) - if requests - else [] - ) - outcome_index: int = 0 - model_results: dict[str, ModelColumnLineage] = {} - compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]] = {} - model_order: list[str] = [] - for model in models: - if model.fast_lineage_columns is not None: - columns: Sequence[CompiledLineageColumnFact] = model.fast_lineage_columns - if _needs_star_expansion(model): - _, star_columns, _, _ = outcomes[outcome_index] - outcome_index += 1 - columns = (*columns, *(_column_fact(column) for column in star_columns)) - compact_models[model.name] = (columns, model.fast_lineage_has_star) - model_order.append(model.name) - continue - status, built_columns, has_star, detail = outcomes[outcome_index] - outcome_index += 1 - result: ModelColumnLineage | None = None - if status == NATIVE_LINEAGE_BUILT: - result = ModelColumnLineage( - model_name=model.name, - columns=tuple(_column_lineage(column) for column in built_columns), - has_star=has_star, - ) - elif status == NATIVE_LINEAGE_UNPARSED: - log_debug_event( - logger=_DEBUG_LOGGER, - message="fast column lineage parse failed; falling back", - sqlbuild_model=model.name, - sqlbuild_error=detail, - ) - elif status == NATIVE_LINEAGE_UNKNOWN_DIALECT: - raise UnknownLineageDialectError(f"Unknown dialect: {detail}") - if result is None: - continue - model_results[model.name] = result - model_order.append(model.name) - return ProjectColumnLineage.from_fast_facts( - models=model_results, - compact_models=compact_models, - model_order=tuple(model_order), - ) - - -def _needs_star_expansion(model: CompiledModelView) -> bool: - return model.fast_lineage_has_star and not model.fast_lineage_star_resolved - - -def _native_request(model: CompiledModelView) -> tuple[bool, str, list[str]] | None: - columns: Sequence[CompiledLineageColumnFact] | None = model.fast_lineage_columns - if columns is None: - return (False, model.query_sql, [name for name, _, _ in model.inferred_rows or ()]) - if not _needs_star_expansion(model): - return None - if isinstance(columns, CompactLineageFacts): - return (True, model.query_sql, [columns.string_pool[row[0]] for row in columns.rows]) - return (True, model.query_sql, [column.output_column for column in columns]) - - -def _schema_resources(project: CompiledProjectView) -> list[tuple[str, str, list[str]]]: - resources: list[tuple[str, str, list[str]]] = [] - for model in project.models: - columns: list[str] = [name for name, _, _ in model.inferred_rows or ()] - columns.extend(name for name, _ in model.schema_column_rows or ()) - resources.append((CompiledResourceType.MODEL.value, model.name, columns)) - for source in project.sources: - resources.append( - ( - CompiledResourceType.SOURCE.value, - source.name, - [column.name for column in source.source_entry.columns], - ) - ) - for seed in project.seeds: - resources.append( - ( - CompiledResourceType.SEED.value, - seed.name, - [column.name for column in seed.schema_entry.columns], - ) - ) - return resources - - -def _column_fact(column: _NativeColumn) -> CompiledLineageColumnFact: - output_column, transform_kind, confidence, sources = column - return CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=tuple( - CompiledLineageSourceFact( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - transform_kind=ColumnTransformKind(transform_kind), - confidence=ColumnLineageConfidence(confidence), - ) - - -def _column_lineage(column: _NativeColumn) -> ColumnLineage: - output_column, transform_kind, confidence, sources = column - return ColumnLineage( - output_column=output_column, - transform_kind=ColumnTransformKind(transform_kind), - expression_sql=None, - upstream_columns=tuple( - ColumnLineageSource( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - nullability=InferredNullability.UNKNOWN, - confidence=ColumnLineageConfidence(confidence), - ) diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py deleted file mode 100644 index 84f3bdd6cc..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Rich column lineage built by the native engine, assembled exactly as Python assembles it.""" - -from __future__ import annotations - -import logging - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage._helpers.rich_columns import ( - _polyglot_dialect, - assemble_rich_project_column_lineage, -) -from sqlbuild.compiler.lineage.constants import ( - NATIVE_LINEAGE_BUILT, - NATIVE_RICH_LINEAGE_SKIPPED, -) -from sqlbuild.compiler.lineage.models import ( - ColumnLineage, - ColumnLineageSource, - ModelColumnLineage, - ProjectColumnLineage, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") - -type _TypedColumns = list[tuple[str, str | None]] -type _NativeColumn = tuple[str, str, str, str, list[tuple[str, str, str]]] -type _NativeOutcome = tuple[str, list[_NativeColumn], bool, str | None] - - -def build_native_rich_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None, - model_names: frozenset[str] | None, -) -> ProjectColumnLineage | None: - """Build the rich project column lineage graph with native query analysis.""" - - if not project.settings.sql_analysis: - return None - models: tuple[CompiledModelView, ...] = tuple( - model for model in project.models if model_names is None or model.name in model_names - ) - outcomes: list[_NativeOutcome] = ( - _native.build_rich_column_lineage( - ( - _polyglot_dialect(dialect).value, - _schema_resources(project), - [model.query_sql for model in models], - ) - ) - if models - else [] - ) - model_results: dict[str, ModelColumnLineage] = {} - for model, (status, columns, has_star, detail) in zip(models, outcomes, strict=True): - if status == NATIVE_LINEAGE_BUILT: - model_results[model.name] = ModelColumnLineage( - model_name=model.name, - columns=tuple(_column_lineage(column) for column in columns), - has_star=has_star, - ) - elif status == NATIVE_RICH_LINEAGE_SKIPPED: - log_debug_event( - logger=_DEBUG_LOGGER, - message="rich column lineage analysis failed; skipping model", - sqlbuild_model=model.name, - sqlbuild_error=detail, - ) - return assemble_rich_project_column_lineage(model_results) - - -def _schema_resources( - project: CompiledProjectView, -) -> list[tuple[str, str, _TypedColumns, _TypedColumns]]: - resources: list[tuple[str, str, _TypedColumns, _TypedColumns]] = [] - for model in project.models: - resources.append( - ( - CompiledResourceType.MODEL.value, - model.name, - [(name, column_type) for name, column_type, _ in model.inferred_rows or ()], - list(model.schema_column_rows or ()), - ) - ) - for source in project.sources: - resources.append( - ( - CompiledResourceType.SOURCE.value, - source.name, - [(column.name, column.type) for column in source.source_entry.columns], - [], - ) - ) - for seed in project.seeds: - resources.append( - ( - CompiledResourceType.SEED.value, - seed.name, - [(column.name, column.type) for column in seed.schema_entry.columns], - [], - ) - ) - return resources - - -def _column_lineage(column: _NativeColumn) -> ColumnLineage: - output_column, transform_kind, confidence, nullability, sources = column - return ColumnLineage( - output_column=output_column, - transform_kind=ColumnTransformKind(transform_kind), - expression_sql=None, - upstream_columns=tuple( - ColumnLineageSource( - resource_type=CompiledResourceType(resource_type), - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - nullability=InferredNullability(nullability), - confidence=ColumnLineageConfidence(confidence), - ) diff --git a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py deleted file mode 100644 index 42e4c58586..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Rich column lineage glue: the analysis dialect name and the project graph assembly.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.types import BuiltinAdapter, TypeDialect -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - ModelColumnLineage, - ProjectColumnLineage, - QualifiedLineageColumn, -) -from sqlbuild.compiler.lineage.types import PolyglotAnalysisDialect - -_POLYGLOT_DIALECT_ALIASES: dict[str, PolyglotAnalysisDialect] = { - "": PolyglotAnalysisDialect.GENERIC, - "ansi": PolyglotAnalysisDialect.GENERIC, - "generic": PolyglotAnalysisDialect.GENERIC, - TypeDialect.GENERIC.value: PolyglotAnalysisDialect.GENERIC, - TypeDialect.BIGQUERY.value: PolyglotAnalysisDialect.BIGQUERY, - TypeDialect.SNOWFLAKE.value: PolyglotAnalysisDialect.SNOWFLAKE, - TypeDialect.DUCKDB.value: PolyglotAnalysisDialect.DUCKDB, - TypeDialect.MOTHERDUCK.value: PolyglotAnalysisDialect.DUCKDB, - TypeDialect.DATABRICKS.value: PolyglotAnalysisDialect.DATABRICKS, - TypeDialect.POSTGRES.value: PolyglotAnalysisDialect.POSTGRESQL, - TypeDialect.TSQL.value: PolyglotAnalysisDialect.TSQL, - BuiltinAdapter.SQLSERVER.value: PolyglotAnalysisDialect.TSQL, - "postgresql": PolyglotAnalysisDialect.POSTGRESQL, -} - - -def assemble_rich_project_column_lineage( - model_results: dict[str, ModelColumnLineage], -) -> ProjectColumnLineage: - """Collapse per-model rich lineage, in model order, into the project graph and its edges.""" - - collapsed_edges: list[ColumnLineageEdge] = [] - for target_resource_name, result in model_results.items(): - for column in result.columns: - target: QualifiedLineageColumn = QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name=target_resource_name, - column_name=column.output_column, - ) - for upstream in column.upstream_columns: - collapsed_edges.append( - ColumnLineageEdge( - source=upstream.as_qualified_column(), - target=target, - transform_kind=column.transform_kind, - confidence=column.confidence, - ) - ) - - return ProjectColumnLineage(models=model_results, edges=tuple(collapsed_edges)) - - -def _polyglot_dialect(dialect: str | TypeDialect | None) -> PolyglotAnalysisDialect: - if dialect is None: - return PolyglotAnalysisDialect.GENERIC - normalized: str = dialect.strip().lower() - mapped: PolyglotAnalysisDialect | None = _POLYGLOT_DIALECT_ALIASES.get(normalized) - if mapped is not None: - return mapped - return PolyglotAnalysisDialect(normalized) diff --git a/src/sqlbuild/compiler/lineage/_helpers/tree_render.py b/src/sqlbuild/compiler/lineage/_helpers/tree_render.py deleted file mode 100644 index 1873b21e45..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/tree_render.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Generic box-drawing tree renderers shared by lineage output commands.""" - -from __future__ import annotations - -from collections.abc import Callable, Hashable -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from _typeshed import SupportsRichComparison as SupportsSortKey -else: - SupportsSortKey = object - -_BRANCH_LAST: str = "└── " -_BRANCH_MID: str = "├── " -_CONTINUATION_LAST: str = " " -_CONTINUATION_MID: str = "│ " - - -def render_dependency_branch[Node: Hashable]( - *, - node: Node, - deps: dict[Node, list[Node]], - prefix: str, - seen: set[Node], - format_node: Callable[[Node], str], - sort_key: Callable[[Node], SupportsSortKey], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render a box-drawing dependency branch, expanding each node once across the tree.""" - - lines: list[str] = [] - shown: set[Node] = set(seen) - pending: list[tuple[Node, str, bool]] = _pending_children( - children=sorted(deps.get(node, ()), key=sort_key), prefix=prefix - ) - while pending: - child: Node - child_prefix: str - is_last: bool - child, child_prefix, is_last = pending.pop() - branch: str = _BRANCH_LAST if is_last else _BRANCH_MID - suffix: str = already_shown() if child in shown else "" - lines.append(f"{branch_style(child_prefix + branch)}{format_node(child)}{suffix}") - if child in shown: - continue - shown.add(child) - continuation: str = _CONTINUATION_LAST if is_last else _CONTINUATION_MID - pending.extend( - _pending_children( - children=sorted(deps.get(child, ()), key=sort_key), - prefix=child_prefix + continuation, - ) - ) - return lines - - -def render_column_trace_branch[Column, Edge]( - *, - column: Column, - deps: dict[str, list[Edge]], - prefix: str, - seen: set[str], - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], - format_related: Callable[[Edge], str], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render a box-drawing column-trace branch, expanding each column once across the tree.""" - - lines: list[str] = [] - shown: set[str] = set(seen) - pending: list[tuple[Edge, str, bool]] = _pending_children( - children=_sorted_edges( - edges=deps.get(column_id(column), ()), - column_id=column_id, - related_column=related_column, - ), - prefix=prefix, - ) - while pending: - edge: Edge - edge_prefix: str - is_last: bool - edge, edge_prefix, is_last = pending.pop() - branch: str = _BRANCH_LAST if is_last else _BRANCH_MID - related: Column = related_column(edge) - related_id: str = column_id(related) - suffix: str = already_shown() if related_id in shown else "" - lines.append(f"{edge_prefix}{branch_style(branch)}{format_related(edge)}{suffix}") - if related_id in shown: - continue - shown.add(related_id) - continuation: str = _CONTINUATION_LAST if is_last else _CONTINUATION_MID - pending.extend( - _pending_children( - children=_sorted_edges( - edges=deps.get(related_id, ()), - column_id=column_id, - related_column=related_column, - ), - prefix=edge_prefix + continuation, - ) - ) - return lines - - -def _pending_children[Item](*, children: list[Item], prefix: str) -> list[tuple[Item, str, bool]]: - last: int = len(children) - 1 - return [(child, prefix, index == last) for index, child in reversed(list(enumerate(children)))] - - -def _sorted_edges[Column, Edge]( - *, - edges: list[Edge] | tuple[()], - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], -) -> list[Edge]: - return sorted(edges, key=lambda edge: column_id(related_column(edge))) - - -def render_column_trace_limit_note( - *, - total: int, - limit: int, - note_style: Callable[[str], str], -) -> list[str]: - """Render the shared truncation note when a column trace is limited.""" - - if total <= limit: - return [] - return [ - "", - note_style(f"Showing {limit} of {total} columns."), - note_style("Use --depth 1 to show direct column dependencies only."), - note_style("Use --format json for the full trace."), - ] diff --git a/src/sqlbuild/compiler/lineage/constants.py b/src/sqlbuild/compiler/lineage/constants.py deleted file mode 100644 index 8f5dc7f196..0000000000 --- a/src/sqlbuild/compiler/lineage/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Column lineage analysis constants.""" - -STAR_COLUMN_NAME: str = "*" -INDEXED_EDGE_RECORD_LENGTH: int = 6 -NATIVE_LINEAGE_BUILT: str = "built" -NATIVE_LINEAGE_UNPARSED: str = "unparsed" -NATIVE_LINEAGE_UNKNOWN_DIALECT: str = "unknown_dialect" -NATIVE_RICH_LINEAGE_SKIPPED: str = "skipped" diff --git a/src/sqlbuild/compiler/lineage/main/__init__.py b/src/sqlbuild/compiler/lineage/main/__init__.py deleted file mode 100644 index 800fd62391..0000000000 --- a/src/sqlbuild/compiler/lineage/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Column lineage analyzer entry points.""" diff --git a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py deleted file mode 100644 index c558388a77..0000000000 --- a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Build fast column lineage facts natively.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage._helpers.native_fast_columns import ( - build_native_fast_project_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage - - -def build_native_column_lineage( - *, project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] | None -) -> ProjectColumnLineage | None: - """Return the fast lineage graph, or None without SQL analysis.""" - - return build_native_fast_project_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) diff --git a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py b/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py deleted file mode 100644 index 07da567097..0000000000 --- a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Build on-demand direct non-projection semantic column uses.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.lineage._helpers.native_rich_columns import _schema_resources -from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse, QualifiedLineageColumn -from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, NativeSemanticUseRow - - -def build_direct_semantic_uses( - *, project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] -) -> tuple[DirectSemanticColumnUse, ...]: - """Extract semantic uses only for explicitly requested models.""" - - rows: list[NativeSemanticUseRow] = _native.build_direct_semantic_uses( - _schema_resources(project), - [ - (model.name, model.query_sql, model.authored_sql) - for model in project.models - if model.name in model_names - ], - dialect or "generic", - ) - return tuple( - DirectSemanticColumnUse( - consumer_model=model, - context=context, - expression_sql=expression, - source=QualifiedLineageColumn( - resource_type=CompiledResourceType(kind), - resource_name=resource, - column_name=column, - ), - confidence=ColumnLineageConfidence(confidence), - line=line, - column=offset, - ) - for model, context, expression, kind, resource, column, confidence, line, offset in rows - ) diff --git a/src/sqlbuild/compiler/lineage/main/columns.py b/src/sqlbuild/compiler/lineage/main/columns.py deleted file mode 100644 index 1de9498177..0000000000 --- a/src/sqlbuild/compiler/lineage/main/columns.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Column lineage analyzer entry point.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage._helpers.native_rich_columns import ( - build_native_rich_project_column_lineage, -) -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage -from sqlbuild.compiler.lineage.types import ColumnLineageMode - - -def build_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None = None, - mode: ColumnLineageMode = ColumnLineageMode.RICH, - model_names: frozenset[str] | None = None, -) -> ProjectColumnLineage | None: - """Build a sidecar project column lineage graph for compiled models.""" - - match mode: - case ColumnLineageMode.FAST: - return build_native_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) - case ColumnLineageMode.RICH: - return build_native_rich_project_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) diff --git a/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py b/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py deleted file mode 100644 index fff2e4b4af..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public entry for rendering the column-trace truncation note.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - render_column_trace_limit_note as _render_column_trace_limit_note, -) - - -def render_column_trace_limit_note( - *, - total: int, - limit: int, - note_style: Callable[[str], str], -) -> list[str]: - """Render the shared column-trace truncation note when the trace is limited.""" - - return _render_column_trace_limit_note(total=total, limit=limit, note_style=note_style) diff --git a/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py b/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py deleted file mode 100644 index d2e1ffcecf..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Public entry for rendering a box-drawing column-trace tree with limit note.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - render_column_trace_branch, - render_column_trace_limit_note, -) - - -def render_column_trace_tree[Column, Edge]( - *, - target: Column, - deps: dict[str, list[Edge]], - total: int, - limit: int, - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], - format_related: Callable[[Edge], str], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], - note_style: Callable[[str], str], -) -> list[str]: - """Render the box-drawing column-trace tree lines plus any truncation note.""" - - lines: list[str] = render_column_trace_branch( - column=target, - deps=deps, - prefix="", - seen={column_id(target)}, - column_id=column_id, - related_column=related_column, - format_related=format_related, - branch_style=branch_style, - already_shown=already_shown, - ) - lines.extend(render_column_trace_limit_note(total=total, limit=limit, note_style=note_style)) - return lines diff --git a/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py b/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py deleted file mode 100644 index 840c0e3899..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public entry for rendering a box-drawing lineage dependency tree.""" - -from __future__ import annotations - -from collections.abc import Callable, Hashable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - SupportsSortKey, - render_dependency_branch, -) - - -def render_dependency_tree[Node: Hashable]( - *, - focus: Node, - deps: dict[Node, list[Node]], - seen: set[Node], - format_node: Callable[[Node], str], - sort_key: Callable[[Node], SupportsSortKey], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render the box-drawing dependency tree lines for one focus node.""" - - return render_dependency_branch( - node=focus, - deps=deps, - prefix="", - seen=seen, - format_node=format_node, - sort_key=sort_key, - branch_style=branch_style, - already_shown=already_shown, - ) diff --git a/src/sqlbuild/compiler/lineage/main/serialize_column.py b/src/sqlbuild/compiler/lineage/main/serialize_column.py deleted file mode 100644 index 59239af8e4..0000000000 --- a/src/sqlbuild/compiler/lineage/main/serialize_column.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public entry for serializing a qualified lineage column to a JSON payload.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage.models import QualifiedLineageColumn - - -def serialize_column( - *, - column: QualifiedLineageColumn, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: - """Serialize a qualified lineage column using a caller-provided resource renderer.""" - - return { - "resource_type": render_resource_type(column), - "resource_name": column.resource_name, - "column_name": column.column_name, - } diff --git a/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py b/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py deleted file mode 100644 index 9a914e1c26..0000000000 --- a/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entry for serializing a column lineage edge to a stable JSON payload.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage.main.serialize_column import serialize_column -from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn - - -def serialize_column_edge( - *, - edge: ColumnLineageEdge, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: - """Serialize a column lineage edge using a caller-provided resource renderer.""" - - return { - "source": serialize_column(column=edge.source, render_resource_type=render_resource_type), - "target": serialize_column(column=edge.target, render_resource_type=render_resource_type), - "transform": str(edge.transform_kind), - "confidence": str(edge.confidence), - } diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py index 23b65bd473..a56889257c 100644 --- a/src/sqlbuild/compiler/lineage/models.py +++ b/src/sqlbuild/compiler/lineage/models.py @@ -1,19 +1,10 @@ -"""Column lineage result models and project graph.""" +"""Column lineage result models.""" from __future__ import annotations -from collections import defaultdict -from collections.abc import Sequence -from dataclasses import dataclass, field -from typing import cast +from dataclasses import dataclass -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.constants import INDEXED_EDGE_RECORD_LENGTH from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -103,401 +94,3 @@ class ModelColumnLineage: model_name: str columns: tuple[ColumnLineage, ...] has_star: bool = False - - -@dataclass(frozen=True, init=False) -class ProjectColumnLineage: - """Project-level column lineage graph with lazily materialized public views.""" - - _models: dict[str, ModelColumnLineage] = field(init=False, repr=False) - _compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]] = field( - init=False, repr=False - ) - _model_order: tuple[str, ...] = field(init=False, repr=False) - _edges: tuple[ColumnLineageEdge, ...] | None = field(init=False, repr=False) - _edge_records: tuple[object, ...] = field(init=False, repr=False) - _edge_cache: dict[ - tuple[str, str, str, str, str, ColumnTransformKind, ColumnLineageConfidence], - ColumnLineageEdge, - ] = field(init=False, repr=False) - _edge_counts_by_target_model: dict[str, int] = field(init=False, repr=False) - _indexes_initialized: bool = field(init=False, repr=False) - _edge_records_by_target_model: dict[str, tuple[object, ...]] = field(init=False, repr=False) - _edge_records_by_source_resource: dict[str, tuple[object, ...]] = field(init=False, repr=False) - _edge_record_by_target_column: dict[tuple[str, str], object] = field(init=False, repr=False) - - def __init__( - self, - *, - models: dict[str, ModelColumnLineage], - edges: tuple[ColumnLineageEdge, ...], - ) -> None: - object.__setattr__(self, "_models", models) - object.__setattr__(self, "_compact_models", {}) - object.__setattr__(self, "_model_order", tuple(models)) - object.__setattr__(self, "_edges", edges) - object.__setattr__(self, "_edge_records", edges) - object.__setattr__(self, "_edge_cache", {}) - object.__setattr__(self, "_edge_counts_by_target_model", {}) - object.__setattr__(self, "_indexes_initialized", False) - self._initialize_indexes() - - @classmethod - def from_fast_facts( - cls, - *, - models: dict[str, ModelColumnLineage], - compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]], - model_order: tuple[str, ...], - ) -> ProjectColumnLineage: - """Build indexed project lineage without expanding compact analysis facts.""" - - result: ProjectColumnLineage = cls(models=models, edges=()) - object.__setattr__(result, "_compact_models", compact_models) - object.__setattr__(result, "_model_order", model_order) - object.__setattr__(result, "_edges", None) - object.__setattr__(result, "_edge_records", ()) - object.__setattr__(result, "_indexes_initialized", False) - edge_counts: dict[str, int] = {} - for model_name in model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = compact_models.get( - model_name - ) - if compact is None: - continue - compact_columns: Sequence[CompiledLineageColumnFact] = compact[0] - if isinstance(compact_columns, CompactLineageFacts): - edge_counts[model_name] = sum(len(row[3]) for row in compact_columns.rows) - else: - edge_counts[model_name] = sum( - len(column.upstream_columns) for column in compact_columns - ) - for model_name, model in models.items(): - edge_counts[model_name] = sum(len(column.upstream_columns) for column in model.columns) - object.__setattr__(result, "_edge_counts_by_target_model", edge_counts) - return result - - def _build_fast_edge_records( - self, - ) -> tuple[object, ...]: - records: list[object] = [] - for model_name in self._model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = ( - self._compact_models.get(model_name) - ) - if compact is not None: - columns: Sequence[CompiledLineageColumnFact] = compact[0] - if isinstance(columns, CompactLineageFacts): - for name_index, transform_code, confidence_code, sources in columns.rows: - output_column: str = columns.string_pool[name_index] - for source in sources: - records.append( - ( - columns, - source, - model_name, - output_column, - transform_code, - confidence_code, - ) - ) - else: - for column in columns: - for source in column.upstream_columns: - records.append( - ( - source, - model_name, - column.output_column, - column.transform_kind, - column.confidence, - ) - ) - continue - model: ModelColumnLineage | None = self._models.get(model_name) - if model is None: - continue - for column in model.columns: - for source in column.upstream_columns: - records.append( - ( - source, - model_name, - column.output_column, - column.transform_kind, - column.confidence, - ) - ) - return tuple(records) - - @property - def models(self) -> dict[str, ModelColumnLineage]: - """Return model lineage, expanding compact model facts on first access.""" - - if self._compact_models: - for model_name in self._model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = ( - self._compact_models.get(model_name) - ) - if compact is not None and model_name not in self._models: - self._models[model_name] = self._model_lineage_from_compact_facts( - model_name=model_name, - columns=compact[0], - has_star=compact[1], - ) - object.__setattr__(self, "_compact_models", {}) - return self._models - - @property - def edges(self) -> tuple[ColumnLineageEdge, ...]: - """Return all graph edges, expanding compact edge records on first access.""" - - if self._edges is None: - self._ensure_indexes() - object.__setattr__( - self, - "_edges", - tuple(self._materialize_edge(record) for record in self._edge_records), - ) - edges: tuple[ColumnLineageEdge, ...] | None = self._edges - if edges is None: - return () - return edges - - def has_model(self, model_name: str) -> bool: - """Return whether model lineage is available without expanding its columns.""" - - return model_name in self._models or model_name in self._compact_models - - def model_has_star(self, model_name: str) -> bool: - """Return whether model lineage retains unresolved root-star uncertainty.""" - - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = self._compact_models.get( - model_name - ) - if compact is not None: - return compact[1] - model: ModelColumnLineage | None = self._models.get(model_name) - return model.has_star if model is not None else False - - def edge_count_targeting(self, model_name: str) -> int: - """Return a target model's direct edge count without expanding edge objects.""" - - return self._edge_counts_by_target_model.get( - model_name, - len(self._edge_records_by_target_model.get(model_name, ())), - ) - - def _initialize_indexes(self) -> None: - by_target_model: dict[str, list[object]] = defaultdict(list) - by_source_resource: dict[str, list[object]] = defaultdict(list) - by_target_column: dict[tuple[str, str], object] = {} - for record in self._edge_records: - source_name, target_name, target_column = self._edge_record_identity(record) - by_target_model[target_name].append(record) - by_source_resource[source_name].append(record) - by_target_column.setdefault((target_name, target_column), record) - object.__setattr__( - self, - "_edge_records_by_target_model", - {key: tuple(value) for key, value in by_target_model.items()}, - ) - object.__setattr__( - self, - "_edge_records_by_source_resource", - {key: tuple(value) for key, value in by_source_resource.items()}, - ) - object.__setattr__(self, "_edge_record_by_target_column", by_target_column) - object.__setattr__( - self, - "_edge_counts_by_target_model", - {key: len(value) for key, value in self._edge_records_by_target_model.items()}, - ) - object.__setattr__(self, "_indexes_initialized", True) - - def _ensure_indexes(self) -> None: - if self._indexes_initialized: - return - object.__setattr__(self, "_edge_records", self._build_fast_edge_records()) - self._initialize_indexes() - - def _materialize_edge(self, record: object) -> ColumnLineageEdge: - if isinstance(record, ColumnLineageEdge): - return record - ( - resource_type, - resource_name, - source_column, - target_model, - target_column, - transform_kind, - confidence, - ) = self._edge_record_values(record) - key: tuple[ - str, - str, - str, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ] = ( - str(resource_type), - resource_name, - source_column, - target_model, - target_column, - transform_kind, - confidence, - ) - cached: ColumnLineageEdge | None = self._edge_cache.get(key) - if cached is not None: - return cached - edge: ColumnLineageEdge = ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=resource_type, - resource_name=resource_name, - column_name=source_column, - ), - target=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name=target_model, - column_name=target_column, - ), - transform_kind=transform_kind, - confidence=confidence, - ) - self._edge_cache[key] = edge - return edge - - def edges_targeting(self, model_name: str) -> tuple[ColumnLineageEdge, ...]: - """Return edges whose target model is `model_name`.""" - - self._ensure_indexes() - return tuple( - self._materialize_edge(record) - for record in self._edge_records_by_target_model.get(model_name, ()) - ) - - def edges_sourced_from(self, resource_name: str) -> tuple[ColumnLineageEdge, ...]: - """Return edges whose source resource is `resource_name`.""" - - self._ensure_indexes() - return tuple( - self._materialize_edge(record) - for record in self._edge_records_by_source_resource.get(resource_name, ()) - ) - - def column_consumers( - self, - *, - resource_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Return direct downstream consumers of `resource_name.column_name`.""" - - return tuple( - edge - for edge in self.edges_sourced_from(resource_name) - if edge.source.column_name == column_name - ) - - @staticmethod - def _edge_record_identity(record: object) -> tuple[str, str, str]: - if isinstance(record, ColumnLineageEdge): - return ( - record.source.resource_name, - record.target.resource_name, - record.target.column_name, - ) - _, resource_name, _, target_model, target_column, _, _ = ( - ProjectColumnLineage._edge_record_values(record) - ) - return resource_name, target_model, target_column - - @staticmethod - def _edge_record_values( - record: object, - ) -> tuple[ - CompiledResourceType | str, - str, - str, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ]: - if isinstance(record, tuple) and len(record) == INDEXED_EDGE_RECORD_LENGTH: - facts, source, target_model, target_column, transform_code, confidence_code = cast( - tuple[CompactLineageFacts, tuple[int, int, int], str, str, int, int], record - ) - return ( - facts.string_pool[source[0]], - facts.resource_name(source[1]), - facts.string_pool[source[2]], - target_model, - target_column, - facts.transform_kind(transform_code), - facts.confidence(confidence_code), - ) - source, target_model, target_column, transform_kind, confidence = cast( - tuple[ - CompiledLineageSourceFact | ColumnLineageSource, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ], - record, - ) - return ( - source.resource_type, - source.resource_name, - source.column_name, - target_model, - target_column, - transform_kind, - confidence, - ) - - @staticmethod - def _model_lineage_from_compact_facts( - *, - model_name: str, - columns: Sequence[CompiledLineageColumnFact], - has_star: bool, - ) -> ModelColumnLineage: - lineage_columns: list[ColumnLineage] = [] - for column in columns: - upstream_columns: list[ColumnLineageSource] = [] - for source in column.upstream_columns: - upstream_columns.append( - ColumnLineageSource( - resource_type=source.resource_type, - resource_name=source.resource_name, - column_name=source.column_name, - ) - ) - lineage_columns.append( - ColumnLineage( - output_column=column.output_column, - transform_kind=column.transform_kind, - expression_sql=None, - upstream_columns=tuple(upstream_columns), - nullability=InferredNullability.UNKNOWN, - confidence=column.confidence, - ) - ) - return ModelColumnLineage( - model_name=model_name, - columns=tuple(lineage_columns), - has_star=has_star, - ) - - def __repr__(self) -> str: - return f"ProjectColumnLineage(models={self.models!r}, edges={self.edges!r})" - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ProjectColumnLineage): - return NotImplemented - return self.models == other.models and self.edges == other.edges diff --git a/src/sqlbuild/compiler/lineage/types.py b/src/sqlbuild/compiler/lineage/types.py index db075949c4..1a7ba6da30 100644 --- a/src/sqlbuild/compiler/lineage/types.py +++ b/src/sqlbuild/compiler/lineage/types.py @@ -4,8 +4,6 @@ from enum import StrEnum -type NativeSemanticUseRow = tuple[str, str, str, str, str, str, str, int | None, int | None] - class ColumnTransformKind(StrEnum): """High-level transform classification for one output column.""" diff --git a/src/sqlbuild/compiler/manifest/_helpers/__init__.py b/src/sqlbuild/compiler/manifest/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/manifest/_helpers/macros.py b/src/sqlbuild/compiler/manifest/_helpers/macros.py deleted file mode 100644 index 4f157bd70a..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/macros.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Macro node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_MACRO -from sqlbuild.compiler.scopes.models import DeclarationIdentity, DeclarationRecord, ScopeIndex -from sqlbuild.compiler.scopes.types import DeclarationKind - - -def build_macro_node( - *, - loaded_macro: LoadedMacro, - project_name: str, - dependencies: tuple[DeclarationIdentity, ...] = (), - scope_record: DeclarationRecord | None = None, -) -> dict[str, object]: - """Build one dbt-compatible macro node dict.""" - - unique_id: str = f"macro.{project_name}.{loaded_macro.name}" - - return { - "name": loaded_macro.name, - "resource_type": RESOURCE_TYPE_MACRO, - "package_name": project_name, - "path": loaded_macro.relative_path.as_posix(), - "original_file_path": loaded_macro.relative_path.as_posix(), - "unique_id": unique_id, - "macro_sql": loaded_macro.raw_source, - "depends_on": { - "macros": [f"macro.{project_name}.{dependency.name}" for dependency in dependencies] - }, - "description": "", - "meta": { - "sqlbuild_visibility": ( - scope_record.scope.value if scope_record is not None else "global" - ), - "sqlbuild_scope_path": (scope_record.owning_path if scope_record is not None else None), - }, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "arguments": [], - "created_at": 0.0, - "supported_languages": None, - } - - -def build_sql_hook_macro_node( - *, hook_file: DiscoveredSqlHookFile, project_name: str -) -> dict[str, object]: - """Expose one SQL hook definition through a dbt-compatible macro node.""" - - hook_package_name: str = f"{project_name}__sqlbuild_hooks" - return { - "name": hook_file.name, - "resource_type": RESOURCE_TYPE_MACRO, - "package_name": hook_package_name, - "path": hook_file.relative_path.as_posix(), - "original_file_path": hook_file.relative_path.as_posix(), - "unique_id": f"macro.{hook_package_name}.{hook_file.name}", - "macro_sql": hook_file.sql_body, - "depends_on": {"macros": []}, - "description": hook_file.description or "", - "meta": { - "sqlbuild_resource_type": "sql_hook", - "sqlbuild_hook_name": hook_file.name, - }, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "arguments": [], - "created_at": 0.0, - "supported_languages": None, - } - - -def build_manifest_macro_nodes( - *, - loaded_macros: dict[str, LoadedMacro], - sql_hook_files: tuple[DiscoveredSqlHookFile, ...], - project_name: str, - scope_index: ScopeIndex | None = None, -) -> dict[str, dict[str, object]]: - """Build all dbt-compatible macro entries, including SQL hook definitions.""" - - macros: dict[str, dict[str, object]] = {} - dependency_map: dict[str, tuple[DeclarationIdentity, ...]] = { - record.identity.name: record.macro.dependencies - for record in (() if scope_index is None else scope_index.declarations) - if record.identity.kind is DeclarationKind.MACRO and record.macro is not None - } - scope_records: dict[str, DeclarationRecord] = { - record.identity.name: record - for record in (() if scope_index is None else scope_index.declarations) - if record.identity.kind is DeclarationKind.MACRO - } - macro_name: str - loaded_macro: LoadedMacro - for macro_name, loaded_macro in loaded_macros.items(): - unique_id: str = f"macro.{project_name}.{macro_name}" - macros[unique_id] = build_macro_node( - loaded_macro=loaded_macro, - project_name=project_name, - dependencies=dependency_map.get(macro_name, ()), - scope_record=scope_records.get(macro_name), - ) - hook_file: DiscoveredSqlHookFile - for hook_file in sql_hook_files: - unique_id = f"macro.{project_name}__sqlbuild_hooks.{hook_file.name}" - macros[unique_id] = build_sql_hook_macro_node( - hook_file=hook_file, - project_name=project_name, - ) - return macros diff --git a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py deleted file mode 100644 index 095a59dc26..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Model node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS -from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash -from sqlbuild.compiler.manifest._helpers.shared import ( - build_columns_dict, - build_config_dict, - build_depends_on, - build_fqn, -) -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_MODEL -from sqlbuild.compiler.planner.models import ModelPlanEntry - - -def build_model_node( - *, - model: CompiledModelView, - plan_entry: ModelPlanEntry | None, - project_name: str, - python_hook_metadata: dict[str, dict[str, object]] | None = None, - sql_analysis_dialect: str | None = None, -) -> dict[str, object]: - """Build one dbt-compatible model node dict.""" - - unique_id: str = f"model.{project_name}.{model.name}" - relative_path: Path = model.relative_path - raw_code: str = model.query_sql - compiled_code: str = plan_entry.resolved_sql if plan_entry is not None else raw_code - query_hash: str = compute_query_hash(query_sql=raw_code, dialect=sql_analysis_dialect) - - return { - "database": model.destination.database, - "schema": model.destination.schema, - "name": model.name, - "resource_type": RESOURCE_TYPE_MODEL, - "package_name": project_name, - "path": str(relative_path), - "original_file_path": str(relative_path), - "unique_id": unique_id, - "fqn": build_fqn(project_name=project_name, relative_path=relative_path), - "alias": model.destination.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": query_hash, - }, - "config": build_config_dict(model.config.values), - "tags": _extract_tags(model), - "description": _extract_description(model), - "columns": build_columns_dict(model.schema_entry), - "meta": _extract_meta( - model=model, - python_hook_metadata=python_hook_metadata or {}, - ), - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": model.destination.qualified_name, - "raw_code": raw_code, - "compiled_code": compiled_code, - "depends_on": build_depends_on( - deps=model.deps, - project_name=project_name, - ), - } - - -def _extract_tags(model: CompiledModelView) -> list[str]: - """Extract tags from model config.""" - - raw: object | None = model.config.values.get("tags") - if isinstance(raw, list): - return [str(item) for item in raw] - if isinstance(raw, tuple): - return [str(item) for item in raw] - return [] - - -def _extract_description(model: CompiledModelView) -> str: - """Extract description from schema entry.""" - - if model.schema_entry is not None and model.schema_entry.description is not None: - return model.schema_entry.description - return "" - - -def _extract_meta( - *, model: CompiledModelView, python_hook_metadata: dict[str, dict[str, object]] -) -> dict[str, object]: - """Extract meta from schema entry.""" - - meta: dict[str, object] = {} - if model.schema_entry is not None and model.schema_entry.meta is not None: - meta.update(model.schema_entry.meta) - lifecycle_hooks: dict[str, list[dict[str, object]]] = {} - hook_key: str - for hook_key in ("pre_hooks", "post_hooks"): - serialized: list[dict[str, object]] = _serialize_hooks( - value=model.config.values.get(hook_key), - python_hook_metadata=python_hook_metadata, - ) - if serialized: - lifecycle_hooks[hook_key] = serialized - if lifecycle_hooks: - authored_sqlbuild_meta: object | None = meta.get("sqlbuild") - sqlbuild_meta: dict[str, object] = {} - if isinstance(authored_sqlbuild_meta, dict): - authored_key: object - authored_value: object - for authored_key, authored_value in authored_sqlbuild_meta.items(): - if isinstance(authored_key, str): - sqlbuild_meta[authored_key] = authored_value - elif authored_sqlbuild_meta is not None: - sqlbuild_meta["authored_value"] = authored_sqlbuild_meta - sqlbuild_meta["lifecycle_hooks"] = lifecycle_hooks - meta["sqlbuild"] = sqlbuild_meta - if model.schema_entry is not None and model.schema_entry.dynamic_columns: - authored_sqlbuild_meta = meta.get("sqlbuild") - sqlbuild_meta: dict[str, object] = {} - if isinstance(authored_sqlbuild_meta, dict): - authored_key: object - authored_value: object - for authored_key, authored_value in authored_sqlbuild_meta.items(): - if isinstance(authored_key, str): - sqlbuild_meta[authored_key] = authored_value - if authored_sqlbuild_meta is not None and not isinstance(authored_sqlbuild_meta, dict): - sqlbuild_meta["authored_value"] = authored_sqlbuild_meta - sqlbuild_meta["dynamic_columns"] = [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in model.schema_entry.dynamic_columns - ] - meta["sqlbuild"] = sqlbuild_meta - return meta - - -def _serialize_hooks( - *, value: object, python_hook_metadata: dict[str, dict[str, object]] -) -> list[dict[str, object]]: - return serialize_hook_entries( - value=value, - sql_fields=SQL_HOOK_OUTPUT_FIELDS, - python_hook_fields=python_hook_metadata, - ) diff --git a/src/sqlbuild/compiler/manifest/_helpers/payload.py b/src/sqlbuild/compiler/manifest/_helpers/payload.py deleted file mode 100644 index e68790fa21..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/payload.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Build a dbt-compatible manifest.json payload from compiled project state.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.compiler.manifest._helpers.macros import ( - build_manifest_macro_nodes, -) -from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node -from sqlbuild.compiler.manifest._helpers.seeds import build_seed_node -from sqlbuild.compiler.manifest._helpers.sources import build_source_node -from sqlbuild.compiler.manifest._helpers.tests import ( - build_audit_test_nodes, - build_sql_test_nodes, -) -from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION, RESOURCE_TYPE_PREFIX -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ModelPlanEntry, - PlanOutput, - SqlTestPlanEntry, -) -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity - - -def build_manifest_payload( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None, - project_name: str, - adapter_type: str, - graph: _native.NativeProjectGraph, -) -> dict[str, object]: - """A full dbt v12-compatible manifest; parent and child maps come from the native graph.""" - - model_plan_map: dict[str, ModelPlanEntry] = {} - source_load_names: frozenset[str] = frozenset() - if plan_output is not None: - model_plan_map = {entry.name: entry for entry in plan_output.model_entries} - source_load_names = frozenset(entry.name for entry in plan_output.source_load_entries) - - nodes: dict[str, dict[str, object]] = {} - sources: dict[str, dict[str, object]] = {} - macros: dict[str, dict[str, object]] = build_manifest_macro_nodes( - loaded_macros=project.loaded_macros, - sql_hook_files=project.sql_hook_files, - project_name=project_name, - scope_index=project.scope_index, - ) - python_hook_metadata: dict[str, dict[str, object]] = _build_python_hook_metadata(project) - - model: CompiledModelView - for model in project.models: - plan_entry: ModelPlanEntry | None = model_plan_map.get(model.name) - unique_id: str = f"model.{project_name}.{model.name}" - nodes[unique_id] = build_model_node( - model=model, - plan_entry=plan_entry, - project_name=project_name, - python_hook_metadata=python_hook_metadata, - sql_analysis_dialect=project.sql_analysis_dialect, - ) - - source: CompiledSourceView - for source in project.sources: - unique_id = f"source.{project_name}.{source.name}" - sources[unique_id] = build_source_node( - source=source, - project_name=project_name, - auto_load=source.name in source_load_names, - ) - - seed: CompiledSeedView - for seed in project.seeds: - unique_id = f"seed.{project_name}.{seed.name}" - nodes[unique_id] = build_seed_node( - seed=seed, - project_name=project_name, - ) - - if plan_output is not None: - audit_entry: AuditPlanEntry - for audit_entry in plan_output.audit_entries: - audit_nodes: dict[str, dict[str, object]] = build_audit_test_nodes( - audit_entry=audit_entry, - project_name=project_name, - ) - nodes.update(audit_nodes) - - test_entry: SqlTestPlanEntry - for test_entry in plan_output.test_entries: - test_nodes: dict[str, dict[str, object]] = build_sql_test_nodes( - test_entry=test_entry, - project_name=project_name, - ) - nodes.update(test_nodes) - - parent_rows, child_rows = graph.unique_id_maps(project_name, dict(RESOURCE_TYPE_PREFIX)) - parent_map: dict[str, list[str]] = dict(parent_rows) - child_map: dict[str, list[str]] = dict(child_rows) - - return { - "metadata": _build_metadata( - project_name=project_name, - adapter_type=adapter_type, - run_id=project.run_id, - ), - "nodes": nodes, - "sources": sources, - "macros": macros, - "exposures": {}, - "metrics": {}, - "groups": {}, - "selectors": {}, - "disabled": {}, - "parent_map": parent_map, - "child_map": child_map, - "group_map": {}, - "docs": {}, - "saved_queries": {}, - "semantic_models": {}, - "unit_tests": {}, - } - - -def _build_python_hook_metadata(project: CompiledProjectView) -> dict[str, dict[str, object]]: - hook_identities: dict[str, PythonNodeIdentity] = build_hook_identities(project.hook_functions) - return { - hook.name: { - "relative_path": hook.relative_path.as_posix(), - "definition_hash": hook_identities[hook.name].definition_hash, - "version_hash": hook_identities[hook.name].version_hash, - } - for hook in project.hook_functions - } - - -def _build_metadata( - *, - project_name: str, - adapter_type: str, - run_id: str, -) -> dict[str, object]: - """Build the top-level manifest metadata block.""" - - return { - "dbt_schema_version": DBT_MANIFEST_SCHEMA_VERSION, - "dbt_version": "1.12.0", - "generated_at": datetime.now(tz=UTC).isoformat(), - "invocation_id": run_id, - "env": {}, - "project_name": project_name, - "project_id": None, - "user_id": None, - "send_anonymous_usage_stats": False, - "adapter_type": adapter_type, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/seeds.py b/src/sqlbuild/compiler/manifest/_helpers/seeds.py deleted file mode 100644 index abe698b282..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/seeds.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Seed node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.manifest._helpers.shared import build_columns_dict, build_fqn -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_SEED - - -def build_seed_node( - *, - seed: CompiledSeedView, - project_name: str, -) -> dict[str, object]: - """Build one dbt-compatible seed node dict.""" - - unique_id: str = f"seed.{project_name}.{seed.name}" - relative_path_str: str = str(seed.seed_file.relative_path) - - return { - "database": seed.destination.database, - "schema": seed.destination.schema, - "name": seed.name, - "resource_type": RESOURCE_TYPE_SEED, - "package_name": project_name, - "path": relative_path_str, - "original_file_path": relative_path_str, - "unique_id": unique_id, - "fqn": build_fqn( - project_name=project_name, - relative_path=seed.seed_file.relative_path, - ), - "alias": seed.destination.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": "", - }, - "config": { - "enabled": True, - "alias": None, - "schema": seed.destination.schema, - "database": seed.destination.database, - "tags": list(seed.schema_entry.tags), - "meta": {}, - "group": None, - "materialized": "seed", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "quoting": {}, - "column_types": {}, - "full_refresh": None, - "unique_key": None, - "on_schema_change": "ignore", - "on_configuration_change": "apply", - "grants": {}, - "packages": [], - "docs": {"show": True, "node_color": None}, - "contract": {"enforced": False, "alias_types": True}, - "delimiter": ",", - "quote_columns": None, - }, - "tags": list(seed.schema_entry.tags), - "description": seed.schema_entry.description or "", - "columns": build_columns_dict(seed.schema_entry), - "meta": dict(seed.schema_entry.meta) if seed.schema_entry.meta else {}, - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": seed.destination.qualified_name, - "raw_code": "", - "root_path": None, - "depends_on": {"macros": []}, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/shared.py b/src/sqlbuild/compiler/manifest/_helpers/shared.py deleted file mode 100644 index d9bdd0aa70..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/shared.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Shared serialization helpers for manifest node building.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_PREFIX -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, -) - - -def build_fqn(*, project_name: str, relative_path: Path) -> list[str]: - """Build a dbt-style fully qualified name list from project name and path.""" - - parts: list[str] = [project_name] - parent: Path = relative_path.parent - part: str - for part in parent.parts: - parts.append(part) - stem: str = relative_path.stem - parts.append(stem) - return parts - - -def build_columns_dict( - schema_entry: SchemaModelEntry | SchemaSeedEntry | None, -) -> dict[str, dict[str, object]]: - """Build a dbt-compatible columns dict from schema entry columns.""" - - if schema_entry is None: - return {} - result: dict[str, dict[str, object]] = {} - column: SchemaColumn - for column in schema_entry.columns: - result[column.name] = { - "name": column.name, - "description": column.description or "", - "meta": dict(column.meta) if column.meta else {}, - "data_type": column.type, - "constraints": [], - "quote": None, - "tags": [], - } - return result - - -def build_source_columns_dict( - columns: tuple[SourceColumnEntry, ...], -) -> dict[str, dict[str, object]]: - """Build a dbt-compatible columns dict from source column entries.""" - - result: dict[str, dict[str, object]] = {} - column: SourceColumnEntry - for column in columns: - result[column.name] = { - "name": column.name, - "description": column.description or "", - "meta": dict(column.meta) if column.meta else {}, - "data_type": column.type, - "constraints": [], - "quote": None, - "tags": [], - } - return result - - -def build_config_dict(values: dict[str, object]) -> dict[str, object]: - """Build a dbt-compatible config dict from model config values.""" - - materialized: object = values.get("materialized", "view") - result: dict[str, object] = { - "enabled": values.get("enabled", True), - "alias": values.get("alias"), - "schema": values.get("schema"), - "database": values.get("database"), - "tags": _config_tags(values), - "meta": {}, - "group": None, - "materialized": str(materialized) if materialized is not None else "view", - "incremental_strategy": values.get("incremental_strategy"), - "merge_exclude_columns": values.get("merge_exclude_columns"), - "persist_docs": {}, - "post-hook": _dbt_sql_hooks(values.get("post_hooks")), - "pre-hook": _dbt_sql_hooks(values.get("pre_hooks")), - "quoting": {}, - "column_types": {}, - "full_refresh": values.get("full_refresh"), - "unique_key": values.get("unique_key"), - "on_schema_change": values.get("on_schema_change", "ignore"), - "on_configuration_change": "apply", - "grants": {}, - "packages": [], - "docs": {"show": True, "node_color": None}, - "contract": {"enforced": False, "alias_types": True}, - } - return result - - -def _dbt_sql_hooks(value: object) -> list[dict[str, object]]: - if not isinstance(value, list | tuple): - return [] - hooks: list[dict[str, object]] = [] - entry: object - for entry in value: - if not isinstance(entry, SqlHookEntry): - continue - hooks.append( - { - "sql": entry.statement, - "transaction": True, - "index": len(hooks), - } - ) - return hooks - - -def build_depends_on( - *, - deps: tuple[CompiledObjectKey, ...], - project_name: str, -) -> dict[str, list[str]]: - """Build a dbt-compatible depends_on dict from compiled deps.""" - - node_ids: list[str] = [] - dep: CompiledObjectKey - for dep in deps: - dep_id: str = _key_to_unique_id(key=dep, project_name=project_name) - node_ids.append(dep_id) - return { - "macros": [], - "nodes": node_ids, - } - - -def _key_to_unique_id(*, key: CompiledObjectKey, project_name: str) -> str: - """Convert a CompiledObjectKey to a dbt-style unique_id.""" - - resource_prefix: str = RESOURCE_TYPE_PREFIX.get(str(key.resource_type), "model") - return f"{resource_prefix}.{project_name}.{key.name}" - - -def _config_tags(values: dict[str, object]) -> list[str]: - """Extract tags from config values as a list.""" - - raw: object | None = values.get("tags") - if isinstance(raw, list): - return [str(item) for item in raw] - if isinstance(raw, tuple): - return [str(item) for item in raw] - return [] diff --git a/src/sqlbuild/compiler/manifest/_helpers/sources.py b/src/sqlbuild/compiler/manifest/_helpers/sources.py deleted file mode 100644 index 07c107a786..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/sources.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Source node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledSourceView -from sqlbuild.compiler.manifest._helpers.shared import build_source_columns_dict -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_SOURCE -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_source_node( - *, - source: CompiledSourceView, - project_name: str, - auto_load: bool = False, -) -> dict[str, object]: - """Build one dbt-compatible source node dict.""" - - entry: SourceEntry = source.source_entry - unique_id: str = f"source.{project_name}.{entry.name}" - identifier: str = entry.table if entry.table is not None else entry.name - - meta: dict[str, object] = dict(entry.meta) if entry.meta else {} - if entry.loader is not None: - sqlbuild_meta: dict[str, object] = { - "loader": entry.loader, - "auto_load": auto_load, - } - if entry.write_strategy is not None: - sqlbuild_meta["write_strategy"] = entry.write_strategy.value - if entry.cursor_column is not None: - sqlbuild_meta["cursor_column"] = entry.cursor_column - if entry.unique_key: - sqlbuild_meta["unique_key"] = entry.unique_key - meta["sqlbuild"] = sqlbuild_meta - - return { - "database": entry.database, - "schema": entry.schema or "", - "name": entry.name, - "resource_type": RESOURCE_TYPE_SOURCE, - "package_name": project_name, - "path": str(source.source_file.relative_path), - "original_file_path": str(source.source_file.relative_path), - "unique_id": unique_id, - "fqn": [project_name, entry.name], - "source_name": entry.name, - "source_description": "", - "loader": "", - "identifier": identifier, - "quoting": {}, - "loaded_at_field": None, - "freshness": None, - "external": None, - "description": entry.description or "", - "columns": build_source_columns_dict(entry.columns), - "meta": meta, - "source_meta": {}, - "tags": [], - "config": {}, - "patch_path": None, - "unrendered_config": {}, - "relation_name": None, - "created_at": 0.0, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/tests.py b/src/sqlbuild/compiler/manifest/_helpers/tests.py deleted file mode 100644 index baa2ea9c83..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/tests.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Test node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.manifest._helpers.shared import _key_to_unique_id -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_TEST -from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, SqlTestPlanEntry -from sqlbuild.sql_values.types import SqlValueKind - -_TEST_CONFIG: dict[str, object] = { - "enabled": True, - "alias": None, - "schema": None, - "database": None, - "tags": [], - "meta": {}, - "group": None, - "materialized": "test", - "severity": "ERROR", - "store_failures": None, - "store_failures_as": None, - "where": None, - "limit": None, - "fail_calc": "count(*)", - "warn_if": "!= 0", - "error_if": "!= 0", -} - - -def build_audit_test_nodes( - *, - audit_entry: AuditPlanEntry, - project_name: str, -) -> dict[str, dict[str, object]]: - """Build dbt-compatible GenericTest nodes from one audit plan entry.""" - - unique_id: str = f"test.{project_name}.{audit_entry.name}" - compiled_code: str = audit_entry.resolved_sql - - test_metadata: dict[str, object] = {"name": audit_entry.name} - kwargs: dict[str, object] = {} - if audit_entry.attached_target_name is not None: - kwargs["model"] = audit_entry.attached_target_name - if audit_entry.attached_column_name is not None: - kwargs["column_name"] = audit_entry.attached_column_name - test_metadata["kwargs"] = kwargs - - depends_on_nodes: list[str] = [ - _key_to_unique_id(key=dep, project_name=project_name) for dep in audit_entry.scope_deps - ] - - node: dict[str, object] = { - "database": None, - "schema": "", - "name": audit_entry.name, - "resource_type": RESOURCE_TYPE_TEST, - "package_name": project_name, - "path": "", - "original_file_path": "", - "unique_id": unique_id, - "fqn": [project_name, audit_entry.name], - "alias": audit_entry.name, - "checksum": {"name": CHECKSUM_HASH_NAME, "checksum": ""}, - "config": dict(_TEST_CONFIG), - "tags": [], - "description": "", - "columns": {}, - "meta": {"sqlbuild_test_type": "audit"}, - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": None, - "raw_code": compiled_code, - "compiled_code": compiled_code, - "depends_on": {"macros": [], "nodes": depends_on_nodes}, - "test_metadata": test_metadata, - "column_name": audit_entry.attached_column_name, - } - return {unique_id: node} - - -def build_sql_test_nodes( - *, - test_entry: SqlTestPlanEntry, - project_name: str, -) -> dict[str, dict[str, object]]: - """Build dbt-compatible SingularTest nodes from one SQL test plan entry.""" - - unique_id: str = _key_to_unique_id(key=test_entry.key, project_name=project_name) - - compiled_parts: list[str] = [] - step: ChainStep - for step in test_entry.chain: - if not step.resolved_sql: - continue - compiled_parts.append(f"-- step: {step.model_name}") - compiled_parts.append(step.resolved_sql) - compiled_code: str = "\n\n".join(compiled_parts) - - depends_on_nodes: list[str] = [ - _key_to_unique_id(key=dep, project_name=project_name) for dep in test_entry.scope_deps - ] - - node: dict[str, object] = { - "database": None, - "schema": "", - "name": test_entry.name, - "resource_type": RESOURCE_TYPE_TEST, - "package_name": project_name, - "path": "", - "original_file_path": "", - "unique_id": unique_id, - "fqn": [project_name, test_entry.name], - "alias": test_entry.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": test_entry.case_fingerprint or "", - }, - "config": dict(_TEST_CONFIG), - "tags": [], - "description": "", - "columns": {}, - "meta": _sql_test_meta(test_entry), - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": None, - "raw_code": compiled_code, - "compiled_code": compiled_code, - "depends_on": {"macros": [], "nodes": depends_on_nodes}, - } - return {unique_id: node} - - -def _sql_test_meta(test_entry: SqlTestPlanEntry) -> dict[str, object]: - meta: dict[str, object] = {"sqlbuild_test_type": "sql_native"} - if test_entry.case_name is None or test_entry.source_path is None: - return meta - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test_entry.parameter_schema - } - meta.update( - { - "sqlbuild_source_path": test_entry.source_path.as_posix(), - "sqlbuild_block_index": test_entry.block_index, - "sqlbuild_parent_name": test_entry.parent_name, - "sqlbuild_case_name": test_entry.case_name, - "sqlbuild_case_index": test_entry.case_index, - "sqlbuild_case_fingerprint": test_entry.case_fingerprint, - "sqlbuild_parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test_entry.parameter_schema - ], - "sqlbuild_parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": ( - str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value - ), - } - for name, value in test_entry.parameter_values - ], - } - ) - return meta diff --git a/src/sqlbuild/compiler/manifest/constants.py b/src/sqlbuild/compiler/manifest/constants.py deleted file mode 100644 index 6c70425595..0000000000 --- a/src/sqlbuild/compiler/manifest/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Manifest output constants aligned with dbt manifest v12 schema.""" - -from sqlbuild.compiler.compile.types import CompiledResourceType - -DBT_MANIFEST_SCHEMA_VERSION: str = "https://schemas.getdbt.com/dbt/manifest/v12.json" - -RESOURCE_TYPE_MODEL: str = "model" -RESOURCE_TYPE_SOURCE: str = "source" -RESOURCE_TYPE_SEED: str = "seed" -RESOURCE_TYPE_TEST: str = "test" -RESOURCE_TYPE_MACRO: str = "macro" - -CHECKSUM_HASH_NAME: str = "sha256" - -RESOURCE_TYPE_PREFIX: dict[str, str] = { - CompiledResourceType.MODEL: "model", - CompiledResourceType.SOURCE: "source", - CompiledResourceType.SEED: "seed", - CompiledResourceType.DBT_REF: "model", - CompiledResourceType.AUDIT: "test", - CompiledResourceType.SQL_TEST: "test", -} diff --git a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py index e43dec7fcb..6a2386322b 100644 --- a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py +++ b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py @@ -2,10 +2,12 @@ from __future__ import annotations +from datetime import UTC, datetime + import sqlbuild._native as _native from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities def build_graph_manifest( @@ -15,13 +17,17 @@ def build_graph_manifest( project_name: str, adapter_type: str, graph: _native.NativeProjectGraph, -) -> dict[str, object]: +) -> _native.NativeManifest: """Build the manifest with parent and child maps read from the native lineage graph.""" - return build_manifest_payload( + return _native.build_manifest( project=project, plan_output=plan_output, - project_name=project_name, - adapter_type=adapter_type, graph=graph, + facts={ + "project_name": project_name, + "adapter_type": adapter_type, + "generated_at": datetime.now(tz=UTC).isoformat(), + "hook_identities": build_hook_identities(project.hook_functions), + }, ) diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py index 913ee1df6c..9c6c173269 100644 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ b/src/sqlbuild/compiler/manifest/main/build.py @@ -6,7 +6,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph.main._native_graph_from_edges import native_graph_from_edges -from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload +from sqlbuild.compiler.manifest.main._build_graph_manifest import build_graph_manifest from sqlbuild.compiler.planner.models import PlanOutput @@ -18,13 +18,13 @@ def build_manifest( adapter_type: str, upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[str, object]: - """Build a full dbt v12-compatible manifest dictionary from caller-built edge maps.""" +) -> _native.NativeManifest: + """Build a full dbt v12-compatible manifest from caller-built edge maps.""" graph: _native.NativeProjectGraph = native_graph_from_edges( upstream=upstream_deps, downstream=downstream_deps ) - return build_manifest_payload( + return build_graph_manifest( project=project, plan_output=plan_output, project_name=project_name, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 452f64dafa..75b8a930ef 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -4,8 +4,8 @@ from dataclasses import replace +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, @@ -113,7 +113,7 @@ def resolve_analysis_model_names( if selection is None or (not selection.select and not selection.exclude): return None - structural_project: CompiledProjectView = assemble_project( + structural_project: CompiledProjectView = _native.assemble_project( inputs=compile_inputs, inference_profile=inference_profile, skip_column_inference=True, diff --git a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py deleted file mode 100644 index 274d369125..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Canonical project compilation phase.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( - resolve_compile_analysis_selection, -) -from sqlbuild.compiler.pipeline.models import CompiledProjectPhaseResult, CompilePipelineOptions -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker - - -def compile_project_phase( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions | None = None, - on_progress: Callable[[str], None] | None = None, -) -> CompiledProjectPhaseResult: - """Compile a project without opening a warehouse planning connection.""" - - resolved_options: CompilePipelineOptions = options or CompilePipelineOptions() - effective_config: dict[str, object] = ( - resolved_options.connection_config - if resolved_options.connection_config is not None - else build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=resolved_options.selected_target, - cli_vars=resolved_options.cli_vars, - ) - ) - if on_progress is not None: - on_progress("Compiling project...") - compile_start: float = time.monotonic() - try: - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=resolved_options.selected_target, - no_sql_validation=resolved_options.no_sql_validation, - cli_vars=resolved_options.cli_vars, - external_sql_reference_resolver=resolved_options.external_sql_reference_resolver, - resolved_connection=effective_config, - analysis_selection=resolve_compile_analysis_selection( - options=resolved_options, - discovered_inputs=discovered_inputs, - ), - ) - finally: - compile_seconds: float = time.monotonic() - compile_start - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.compile_seconds = compile_seconds - errors: tuple[CompilerDiagnostic, ...] = tuple( - diagnostic - for diagnostic in project.diagnostics - if diagnostic.is_error - and ( - resolved_options.plan_sql_tests - or diagnostic.resource_type != CompiledResourceType.SQL_TEST - ) - ) - if errors: - if on_progress is not None: - on_progress("Project compilation failed.") - raise CompileInputError( - "\n".join( - f"[{diagnostic.code}] {diagnostic.path}:" - f"{diagnostic.line or 1}:{diagnostic.column or 1}: {diagnostic.message}" - for diagnostic in errors - ), - code=errors[0].code, - help=_shared_help(errors), - ) - if on_progress is not None: - on_progress(f"Compiled project. ({compile_seconds:.2f}s)") - return CompiledProjectPhaseResult( - project=project, - connection_config=effective_config, - compile_seconds=compile_seconds, - ) - - -def _shared_help(errors: tuple[CompilerDiagnostic, ...]) -> str | None: - """Return the help every error shares, so one remedy is shown once.""" - - helps: frozenset[str | None] = frozenset(diagnostic.help for diagnostic in errors) - return next(iter(helps)) if len(helps) == 1 else None diff --git a/src/sqlbuild/compiler/pipeline/main/compile.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py similarity index 54% rename from src/sqlbuild/compiler/pipeline/main/compile.py rename to src/sqlbuild/compiler/pipeline/main/_plan_result.py index 7500686afe..31a26dfcf8 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile.py +++ b/src/sqlbuild/compiler/pipeline/main/_plan_result.py @@ -1,11 +1,8 @@ -"""Full compile-and-plan pipeline producing CLI artifacts.""" +"""Plan a compiled project over an open warehouse connection.""" from __future__ import annotations -import time from collections.abc import Callable -from dataclasses import replace -from pathlib import Path from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,10 +13,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( - apply_unit_test_selectors, - resolve_configured_rule_selection, -) from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( build_deferred_locations, gather_deferred_relations, @@ -31,12 +24,9 @@ build_python_run_plan_outputs, python_read_source_names_for, ) -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( - CompiledProjectPhaseResult, CompilePipelineOptions, CompilePipelineResult, - ProjectGraph, PythonRunPlanOutputs, ) from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan @@ -51,116 +41,10 @@ resolve_python_sql_run_selection_from_inputs, ) from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.render_result import format_result -from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress -from sqlbuild.rule_engine.main.run_rules import run_rules -from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult, RulesRunResult -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks -from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import TargetConfig -def run_compile_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions | None = None, - hooks: ConnectionHooks | None = None, -) -> CompilePipelineResult: - """Run compile inputs, assembly, planning, and manifest generation.""" - - resolved_options: CompilePipelineOptions = apply_unit_test_selectors( - options=options, discovered_inputs=discovered_inputs - ) - resolved_hooks: ConnectionHooks = hooks if hooks is not None else ConnectionHooks() - on_progress: Callable[[str], None] | None = resolved_hooks.on_progress - compile_result: CompiledProjectPhaseResult = compile_project_phase( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=resolved_options, - on_progress=on_progress, - ) - effective_config: dict[str, object] = compile_result.connection_config - project: CompiledProjectView = compile_result.project - compile_seconds: float = compile_result.compile_seconds - graph: ProjectGraph = build_project_graph_impl(project) - project_dir: Path | None = discovered_inputs.project_dir - if project_dir is None: - raise RulesError("compiler rules require a resolved project directory") - rules_config: RulesConfig = load_rules_config(project_dir=project_dir) - if resolved_options.no_cache: - rules_config = replace(rules_config, cache=RulesCacheConfig(enabled=False)) - if on_progress is not None: - on_progress("Evaluating configured rules...") - rules_start: float = time.monotonic() - rules_result: RulesRunResult = run_rules( - graph=graph, - discovered_inputs=discovered_inputs, - config=rules_config, - project_dir=project_dir, - dialect=adapter.sql_analysis_dialect() or "generic", - selected_keys=resolve_configured_rule_selection( - options=resolved_options, - discovered_inputs=discovered_inputs, - project=project, - ), - no_sql_analysis=resolved_options.no_sql_validation, - ) - if on_progress is not None: - on_progress( - format_rules_progress( - elapsed_seconds=time.monotonic() - rules_start, - result=rules_result, - note_skipped_rules=True, - ) - ) - if rules_result.findings: - rendered: str = format_result( - result=RulesResult( - findings=rules_result.findings, - evaluated_models=rules_result.evaluated_models, - unevaluated_resources=rules_result.unevaluated_resources, - cache_hits=rules_result.cache_hits, - cache_misses=rules_result.cache_misses, - ), - json_output=False, - ) - raise RulesError(f"configured compiler rules failed:\n{rendered}") - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - planning_start: float = time.monotonic() - connection: Any | None = None - result: CompilePipelineResult | None = None - try: - connection = open_connection_with_hooks( - adapter=adapter, - connection_config=effective_config, - hooks=resolved_hooks, - ) - result = _build_result( - discovered_inputs=discovered_inputs, - adapter=adapter, - connection=connection, - project=project, - options=resolved_options, - on_progress=on_progress, - ) - finally: - if connection is not None: - adapter.close(connection) - planning_seconds: float = time.monotonic() - planning_start - if timing_tracker is not None: - timing_tracker.planning_seconds = planning_seconds - return replace( - result, - compile_seconds=compile_seconds, - planning_seconds=planning_seconds, - ) - - -def _build_result( +def build_pipeline_result( *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, @@ -169,6 +53,8 @@ def _build_result( options: CompilePipelineOptions, on_progress: Callable[[str], None] | None = None, ) -> CompilePipelineResult: + """Plan the compiled project over an open connection and attach Python node plans.""" + select: tuple[str, ...] = options.select exclude: tuple[str, ...] = options.exclude deferred_locations: dict[str, CompiledRelationLocation] | None = None diff --git a/src/sqlbuild/compiler/pipeline/main/compile_only.py b/src/sqlbuild/compiler/pipeline/main/compile_only.py index 673f35162e..ea0416fe8f 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile_only.py +++ b/src/sqlbuild/compiler/pipeline/main/compile_only.py @@ -4,9 +4,9 @@ from collections.abc import Callable +from sqlbuild._native import compile_project_phase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, diff --git a/src/sqlbuild/compiler/pipeline/main/project.py b/src/sqlbuild/compiler/pipeline/main/project.py deleted file mode 100644 index 2d4cb068fc..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/project.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Compile a project without building an execution plan.""" - -from __future__ import annotations - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def compile_project( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None = None, - resolved_connection: dict[str, object] | None = None, - no_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> CompiledProjectView: - """Compile discovered inputs into a target-defaulted project view.""" - - return build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/static_command.py b/src/sqlbuild/compiler/pipeline/main/static_command.py index 3ce77cf45c..df9c0e7689 100644 --- a/src/sqlbuild/compiler/pipeline/main/static_command.py +++ b/src/sqlbuild/compiler/pipeline/main/static_command.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import compile_project_phase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -12,7 +13,6 @@ build_deferred_locations, resolve_deferred_target_config, ) -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index 04af510c1f..1259b69cca 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -5,7 +5,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.graph.main._lineage_graph_names import lineage_graph_names -from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, @@ -47,9 +46,7 @@ def build_planner_scope( resolved_selected_keys: frozenset[CompiledObjectKey] = ( selected_keys if selected_keys is not None - else resolve_graph_selectors( - graph=build_native_project_graph(project), select=select, exclude=exclude - ) + else resolve_graph_selectors(graph=project.lineage_graph, select=select, exclude=exclude) ) executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( _upstream_source_selector_keys( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 365cff4e0b..21e5b6bfd6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -5,6 +5,7 @@ import re from collections.abc import Callable, Iterator +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, @@ -32,11 +33,6 @@ reference_call_prefix_pattern_text, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token @@ -51,7 +47,7 @@ _TABLE_FUNCTION_PATTERN: re.Pattern[str] = quoted_reference_call_pattern( SqlReferenceKind.TABLE_FUNCTION ) -_SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) +_REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" _CLAUSE_KEYWORDS: frozenset[str] = frozenset( { "WHERE", @@ -185,11 +181,9 @@ def resolve_udf_references( last_index = match.end() continue - call_suffix_end: int = find_matching_paren( - sql=query_sql, - open_paren_index=call_suffix_start, - context="SQL UDF call", - ) + call_suffix_end: int = _native.SqlCodeScanner( + query_sql, None, "SQL UDF call" + ).matching_paren(call_suffix_start) call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] parts.append( adapter.render_udf_call( @@ -241,11 +235,9 @@ def resolve_table_function_references( last_index = match.end() continue - call_suffix_end: int = find_matching_paren( - sql=query_sql, - open_paren_index=call_suffix_start, - context="SQL table function call", - ) + call_suffix_end: int = _native.SqlCodeScanner( + query_sql, None, "SQL table function call" + ).matching_paren(call_suffix_start) call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] parts.append( adapter.render_table_function_call( @@ -281,33 +273,29 @@ def replace_executable_matches( def _iter_executable_matches(*, sql: str, pattern: re.Pattern[str]) -> Iterator[re.Match[str]]: - index: int = 0 - while index < len(sql): - if sql[index] in _SQL_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context="SQL reference resolution") - continue - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context="SQL reference resolution") - continue - match: re.Match[str] | None = pattern.match(sql, index) - if match is None: - index += 1 - continue - yield match - index = match.end() + return _iter_code_matches( + sql=sql, + pattern=pattern, + scanner=_native.SqlCodeScanner(sql, None, _REFERENCE_SCAN_CONTEXT), + ) def _iter_dialect_executable_matches( *, sql: str, pattern: re.Pattern[str], lexical_syntax: SqlLexicalSyntax +) -> Iterator[re.Match[str]]: + return _iter_code_matches( + sql=sql, + pattern=pattern, + scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), + ) + + +def _iter_code_matches( + *, sql: str, pattern: re.Pattern[str], scanner: _native.SqlCodeScanner ) -> Iterator[re.Match[str]]: index: int = 0 while index < len(sql): - non_code_end: int | None = dialect_non_code_end( - sql=sql, start=index, syntax=lexical_syntax, context="SQL reference resolution" - ) + non_code_end: int | None = scanner.non_code_end(index) if non_code_end is not None: index = non_code_end continue diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index d5320eac16..02625c3ac9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -15,7 +15,6 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, build_execution_upstream_deps, @@ -69,7 +68,7 @@ def build_clone_plan_output( upstream_deps ) selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selectors( - graph=build_native_project_graph(project), select=select, exclude=exclude + graph=project.lineage_graph, select=select, exclude=exclude ) return PlanOutput( execution_order=topologically_order_keys(upstream=upstream_deps), diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index 64dacfb4f4..011ea774ab 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -5,6 +5,7 @@ import re from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.sql_test_cte_extractor import SqlTestCteExtractor from sqlbuild.compiler.compile.constants import ( REF_TEST_CTE_PREFIX, @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.main._parse_sql_test_file import parse_sql_test_file from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, @@ -90,7 +90,7 @@ def format_redundant_fixture_nulls( if contents is None: continue try: - current_blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file( + current_blocks: tuple[DiscoveredSqlTestBlock, ...] = _native.parse_sql_test_file( contents=contents, file_path=file_path, ) diff --git a/src/sqlbuild/compiler/project_assembly/main/_native_resource.py b/src/sqlbuild/compiler/project_assembly/main/_native_resource.py deleted file mode 100644 index 61e833139a..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_native_resource.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Take one resource's native fact, raising the error Python's assembly raises there.""" - -from __future__ import annotations - -from sqlbuild.compiler.project_assembly.models import NativeProjectResources, NativeResourceFailure - - -def native_resource[T]( - *, resources: NativeProjectResources, slot: str, items: tuple[T, ...], index: int -) -> T: - """Return resource `index` of `slot`, raising the error Python's assembly raises there.""" - - failure: NativeResourceFailure | None = resources.failure - if failure is not None and failure.slot == slot and failure.index == index: - raise failure.error - return items[index] diff --git a/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py index df96a67817..fccb48461b 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py @@ -1,13 +1,11 @@ -"""Python `ast` scan for strings that name a renamed model or column.""" +"""Project Python files and the strings in them that name a renamed model or column.""" from __future__ import annotations -import ast -import re from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, SQL_LIKE_PATTERN from sqlbuild.compiler.refactoring.models import ManualLocation @@ -37,38 +35,10 @@ def string_locations( """Return the string constants of one Python file that name a renamed model or column.""" try: - tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) - except (OSError, SyntaxError, UnicodeError): + contents: str = (project_dir / relative_path).read_text(encoding="utf-8") + except (OSError, UnicodeError): return () - locations: list[ManualLocation] = [] - node: ast.AST - for node in ast.walk(tree): - if not isinstance(node, ast.Constant) or not isinstance(node.value, str): - continue - if _flags(value=node.value, names=names, context=context): - locations.append( - ManualLocation( - path=relative_path.as_posix(), - line=node.lineno, - column=node.col_offset + 1, - reason=reason, - ) - ) - return tuple(locations) - - -def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: - sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) - if context is not None and not (sql_like and _has_word(text=value, word=context)): - return False - return any( - value.strip().lower() == name.lower() or (sql_like and _has_word(text=value, word=name)) - for name in names - ) - - -def _has_word(*, text: str, word: str) -> bool: - pattern: re.Pattern[str] = re.compile( - rf"(? Any: """Return SQLBuild's required Polyglot SQL module.""" return import_polyglot() - - -def is_polyglot_available() -> bool: - """Return true because Polyglot SQL is a required dependency.""" - - return True diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py deleted file mode 100644 index b1a580421b..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Character-level SQL scanning implementations.""" - -from __future__ import annotations - -import re -from collections.abc import Iterator - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.constants import ( - SQL_BLOCK_COMMENT_CLOSE, - SQL_BLOCK_COMMENT_OPEN, - SQL_CLOSE_PARENTHESIS, - SQL_DOLLAR_QUOTE_CHARACTER, - SQL_ESCAPABLE_QUOTE_CHARACTERS, - SQL_ESCAPE_CHARACTER, - SQL_ESCAPE_STRING_PREFIX, - SQL_IDENTIFIER_PREFIX, - SQL_OPEN_PARENTHESIS, - SQL_RAW_STRING_PREFIX, - SQL_STRING_PREFIX_CHARACTERS, - SQL_STRING_PREFIX_MAX_LENGTH, - SQL_STRING_QUOTE_CHARACTER, - SQL_TEXT_START_CHARACTERS, - SQL_TRIPLE_QUOTE_LENGTH, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/#'\"`$()]") -_DOLLAR_QUOTE_DELIMITER: re.Pattern[str] = re.compile(r"\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$") - - -def skip_quoted_text_impl(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip quoted text at the position, consuming a non-opening ``$`` as one code character.""" - - end: int | None = quoted_text_end_impl(sql=sql, start=start, context=context) - return start + 1 if end is None else end - - -def quoted_text_end_impl(*, sql: str, start: int, context: str = "SQL") -> int | None: - """Return the end of quoted or dollar-quoted text starting at the position, if any.""" - - quote_character: str = sql[start] - if quote_character == SQL_DOLLAR_QUOTE_CHARACTER: - return _dollar_quoted_text_end(sql=sql, start=start, context=context) - index: int = start + 1 - while True: - index = sql.find(quote_character, index) - if index == -1: - raise CompileInputError(f"{context} contains an unclosed quoted string") - if ( - quote_character in SQL_ESCAPABLE_QUOTE_CHARACTERS - and index + 1 < len(sql) - and sql[index + 1] == quote_character - ): - index += 2 - continue - return index + 1 - - -def dialect_non_code_end_impl( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of the quoted text or comment starting at the position, if any.""" - - if sql.startswith(SQL_BLOCK_COMMENT_OPEN, start): - return _dialect_block_comment_end(sql=sql, start=start, syntax=syntax, context=context) - if any(sql.startswith(prefix, start) for prefix in syntax.line_comment_prefixes): - return skip_line_comment_impl(sql=sql, start=start) - if sql[start] in SQL_TEXT_START_CHARACTERS: - return dialect_quoted_text_end_impl(sql=sql, start=start, syntax=syntax, context=context) - return None - - -def _dialect_block_comment_end( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str -) -> int: - depth: int = 1 - index: int = start + len(SQL_BLOCK_COMMENT_OPEN) - while depth: - closing_index: int = sql.find(SQL_BLOCK_COMMENT_CLOSE, index) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed block comment") - opening_index: int = ( - sql.find(SQL_BLOCK_COMMENT_OPEN, index, closing_index) - if syntax.nested_block_comments - else -1 - ) - if opening_index == -1: - depth -= 1 - index = closing_index + len(SQL_BLOCK_COMMENT_CLOSE) - else: - depth += 1 - index = opening_index + len(SQL_BLOCK_COMMENT_OPEN) - return index - - -def dialect_quoted_text_end_impl( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of quoted text starting at the position under the dialect's escape rules.""" - - quote_character: str = sql[start] - if quote_character == SQL_DOLLAR_QUOTE_CHARACTER: - return _dollar_quoted_text_end(sql=sql, start=start, context=context) - prefix: str = _string_literal_prefix(sql=sql, start=start) - backslash_escapes: bool = quote_character in syntax.backslash_escape_quotes or ( - syntax.escape_string_prefix - and quote_character == SQL_STRING_QUOTE_CHARACTER - and prefix == SQL_ESCAPE_STRING_PREFIX - ) - if syntax.raw_string_prefix and SQL_RAW_STRING_PREFIX in prefix: - backslash_escapes = False - delimiter: str = quote_character - triple_quote: str = quote_character * SQL_TRIPLE_QUOTE_LENGTH - if syntax.triple_quoted_strings and sql.startswith(triple_quote, start): - delimiter = triple_quote - index: int = start + len(delimiter) - while index < len(sql): - if backslash_escapes and sql.startswith(SQL_ESCAPE_CHARACTER, index): - index += 2 - continue - if sql.startswith(delimiter, index): - if delimiter == quote_character and sql.startswith(quote_character, index + 1): - index += 2 - continue - return index + len(delimiter) - index += 1 - raise CompileInputError(f"{context} contains an unclosed quoted string") - - -def _string_literal_prefix(*, sql: str, start: int) -> str: - prefix_start: int = start - while ( - prefix_start > 0 - and start - prefix_start < SQL_STRING_PREFIX_MAX_LENGTH - and sql[prefix_start - 1] in SQL_STRING_PREFIX_CHARACTERS - ): - prefix_start -= 1 - if prefix_start > 0 and _continues_dollar_word(sql[prefix_start - 1]): - return "" - return sql[prefix_start:start].lower() - - -def _dollar_quoted_text_end(*, sql: str, start: int, context: str) -> int | None: - delimiter: re.Match[str] | None = _DOLLAR_QUOTE_DELIMITER.match(sql, start) - if delimiter is None or (start > 0 and _continues_dollar_word(sql[start - 1])): - return None - closing_index: int = sql.find(delimiter.group(), delimiter.end()) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed quoted string") - return closing_index + len(delimiter.group()) - - -def _continues_dollar_word(character: str) -> bool: - return ( - not character.isascii() - or character.isalnum() - or character in {SQL_IDENTIFIER_PREFIX, SQL_DOLLAR_QUOTE_CHARACTER} - ) - - -def skip_line_comment_impl(*, sql: str, start: int) -> int: - """Skip past an SQL line comment.""" - - newline_index: int = sql.find("\n", start) - return len(sql) if newline_index == -1 else newline_index + 1 - - -def skip_block_comment_impl(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past an SQL block comment.""" - - closing_index: int = sql.find("*/", start + 2) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed block comment") - return closing_index + 2 - - -def find_matching_paren_impl( - *, - sql: str, - open_paren_index: int, - context: str = "SQL", - syntax: SqlLexicalSyntax | None = None, -) -> int: - """Find the closing parenthesis matching an opening parenthesis.""" - - depth: int = 1 - index: int = open_paren_index + 1 - while index < len(sql): - special: re.Match[str] | None = _PAREN_SCAN_SPECIAL.search(sql, index) - if special is None: - break - index = special.start() - non_code_end: int | None = _non_code_end( - sql=sql, index=index, context=context, syntax=syntax - ) - if non_code_end is not None: - index = non_code_end - continue - if sql[index] == SQL_OPEN_PARENTHESIS: - depth += 1 - elif sql[index] == SQL_CLOSE_PARENTHESIS: - depth -= 1 - if depth == 0: - return index - index += 1 - raise CompileInputError(f"{context} contains an unclosed parenthesis") - - -def iter_code_positions_impl( - *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None -) -> Iterator[tuple[int, int]]: - """Yield code offsets outside comments, quotes, and parentheses with their nesting depth.""" - - depth: int = 0 - index: int = 0 - while index < len(sql): - non_code_end: int | None = _non_code_end( - sql=sql, index=index, context=context, syntax=syntax - ) - if non_code_end is not None: - index = non_code_end - continue - character: str = sql[index] - if character == SQL_OPEN_PARENTHESIS: - depth += 1 - elif character == SQL_CLOSE_PARENTHESIS: - depth -= 1 - else: - yield index, depth - index += 1 - - -def _non_code_end( - *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None -) -> int | None: - if syntax is not None: - return dialect_non_code_end_impl(sql=sql, start=index, syntax=syntax, context=context) - if sql.startswith("--", index): - return skip_line_comment_impl(sql=sql, start=index) - if sql.startswith("/*", index): - return skip_block_comment_impl(sql=sql, start=index, context=context) - if sql[index] in SQL_TEXT_START_CHARACTERS: - return quoted_text_end_impl(sql=sql, start=index, context=context) - return None diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py index eab2c51cf8..583d396b1b 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py @@ -1,22 +1,15 @@ """Orchestrate compile-owned catalogs; native code owns schemas and diagnostics.""" -import json -from typing import Any, cast - import sqlbuild._native as _native from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog -from sqlbuild.compiler.sql_analysis.constants import BINDING_SEVERITIES, NATIVE_DIALECT_ALIASES +from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES from sqlbuild.compiler.sql_analysis.exceptions import SqlAnalysisBoundaryError from sqlbuild.compiler.sql_analysis.models import ( SqlBindingDiagnostic, SqlBindingResult, SqlSchemaValidationRequest, ) -from sqlbuild.compiler.sql_analysis.types import ( - NativeDiagnosticRow, - NativePositionsModule, - NativeValidationModule, -) +from sqlbuild.compiler.sql_analysis.types import NativeDiagnosticRow def get_schema_validations( @@ -50,22 +43,10 @@ def get_schema_validations( results: list[SqlBindingResult] = [SqlBindingResult()] * len(requests) for catalog, group in groups.items(): batch: list[SqlSchemaValidationRequest] = [request for _, request in group] - if all(request.catalog is None for request in batch): - payload: str = catalog.native.validation_payload(catalog.prepare(batch)) - responses: object = json.loads( - cast(NativeValidationModule, _native).validate_sql_with_schemas_json(payload) - ) - if not isinstance(responses, list) or len(responses) != len(batch): - raise SqlAnalysisBoundaryError( - "native SQL schema validation returned an invalid batch response" - ) - for (index, request), response in zip(group, responses, strict=True): - results[index] = _binding_result( - sql=request.sql, dialect=request.dialect, response=response - ) - continue - rows: list[list[NativeDiagnosticRow]] = catalog.native.binding_results( - catalog.prepare(batch) + rows: list[list[NativeDiagnosticRow]] = ( + _native.schema_validation_rows(catalog.native, catalog.dialect, catalog.prepare(batch)) + if all(request.catalog is None for request in batch) + else catalog.native.binding_results(catalog.prepare(batch)) ) if len(rows) != len(batch): raise SqlAnalysisBoundaryError( @@ -78,39 +59,3 @@ def get_schema_validations( def _result(rows: list[NativeDiagnosticRow]) -> SqlBindingResult: return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in rows)) - - -def _binding_result(*, sql: str, dialect: str | None, response: object) -> SqlBindingResult: - if not isinstance(response, dict): - raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") - payload: dict[str, Any] = cast(dict[str, Any], response) - if not isinstance(payload.get("errors"), list): - raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") - rows: list[NativeDiagnosticRow] = [] - for value in payload["errors"]: - if not isinstance(value, dict) or value.get("severity") not in BINDING_SEVERITIES: - continue - if not isinstance(value.get("code"), str): - continue - rows.append( - ( - value["code"], - str(value.get("message") or "SQL binding failed"), - _optional_int(value.get("line")), - _optional_int(value.get("column")), - _optional_int(value.get("start")), - _optional_int(value.get("end")), - str(value["severity"]), - ) - ) - return _result( - cast(NativePositionsModule, _native).binding_diagnostics( - sql=sql, - dialect=NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect or "generic"), - rows=rows, - ) - ) - - -def _optional_int(value: object) -> int | None: - return value if isinstance(value, int) else None diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index be247a0320..c51f1ac0f5 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -3,24 +3,11 @@ from __future__ import annotations SQL_ESCAPABLE_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({*SQL_ESCAPABLE_QUOTE_CHARACTERS, "`"}) -SQL_TEXT_START_CHARACTERS: frozenset[str] = frozenset({*SQL_QUOTE_CHARACTERS, "$"}) -SQL_OPEN_PARENTHESIS: str = "(" -SQL_CLOSE_PARENTHESIS: str = ")" -SQL_STRING_QUOTE_CHARACTER: str = "'" SQL_ESCAPE_CHARACTER: str = "\\" -SQL_ESCAPE_STRING_PREFIX: str = "e" -SQL_RAW_STRING_PREFIX: str = "r" -SQL_STRING_PREFIX_CHARACTERS: frozenset[str] = frozenset("bBeErR") -SQL_STRING_PREFIX_MAX_LENGTH: int = 2 SQL_TRIPLE_QUOTE_LENGTH: int = 3 SQL_LINE_COMMENT_PREFIX: str = "--" SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) SQL_BLOCK_COMMENT_OPEN: str = "/*" -SQL_BLOCK_COMMENT_CLOSE: str = "*/" -SQL_DOLLAR_QUOTE_CHARACTER: str = "$" -SQL_IDENTIFIER_PREFIX: str = "_" -SNOWFLAKE_DIALECT_NAME: str = "snowflake" POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 @@ -33,7 +20,6 @@ TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) SQL_QUOTED_IDENTIFIER_DELIMITER: str = '"' CASE_SENSITIVE_BINDING_DIALECTS: frozenset[str] = frozenset({"snowflake", "postgresql"}) -BINDING_SEVERITIES: frozenset[str] = frozenset({"error", "warning"}) NATIVE_DIALECT_ALIASES: dict[str, str] = { "postgres": "postgresql", "motherduck": "duckdb", diff --git a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py deleted file mode 100644 index a5b31ae45c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py +++ /dev/null @@ -1,21 +0,0 @@ -"""SQL parenthesis matching entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import find_matching_paren_impl -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def find_matching_paren( - *, - sql: str, - open_paren_index: int, - context: str = "SQL", - syntax: SqlLexicalSyntax | None = None, -) -> int: - """Find the closing parenthesis, ending quoted text and comments per the optional dialect.""" - - return find_matching_paren_impl( - sql=sql, - open_paren_index=open_paren_index, - context=context, - syntax=syntax, - ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py b/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py deleted file mode 100644 index 352473adcd..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public Polyglot availability entry.""" - -from sqlbuild.compiler.sql_analysis._helpers.polyglot import ( - is_polyglot_available as _is_polyglot_available, -) - - -def is_polyglot_available() -> bool: - """Return whether Polyglot SQL can be imported.""" - - return _is_polyglot_available() diff --git a/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py b/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py deleted file mode 100644 index 0165127490..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Resolve schema column spellings using dialect identifier semantics.""" - -from collections.abc import Mapping - -from sqlbuild.compiler.sql_analysis.constants import ( - CASE_SENSITIVE_BINDING_DIALECTS, - NATIVE_DIALECT_ALIASES, - SNOWFLAKE_DIALECT_NAME, - SQL_QUOTED_IDENTIFIER_DELIMITER, -) - - -def resolve_binding_column( - *, name: str, columns: Mapping[str, str], dialect: str | None, ignore_quoted_case: bool = False -) -> str | None: - dialect = NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect) - - def identity(value: str) -> str: - quoted: bool = value.startswith(SQL_QUOTED_IDENTIFIER_DELIMITER) and value.endswith( - SQL_QUOTED_IDENTIFIER_DELIMITER - ) - raw: str = value[1:-1].replace('""', '"') if quoted else value - if quoted and dialect in CASE_SENSITIVE_BINDING_DIALECTS and not ignore_quoted_case: - return raw - return raw.upper() if dialect == SNOWFLAKE_DIALECT_NAME else raw.casefold() - - expected: str = identity(name) - return next((column for column in columns if identity(column) == expected), None) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py deleted file mode 100644 index 13f2482f2a..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py +++ /dev/null @@ -1,9 +0,0 @@ -"""SQL block comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_block_comment_impl - - -def skip_block_comment(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past an SQL block comment.""" - - return skip_block_comment_impl(sql=sql, start=start, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py deleted file mode 100644 index 2fa5b8c4ba..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Dialect-aware SQL quoted text and comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import dialect_non_code_end_impl -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def dialect_non_code_end( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of the quoted text or comment at the position under the dialect's rules.""" - - return dialect_non_code_end_impl(sql=sql, start=start, syntax=syntax, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py deleted file mode 100644 index 5708863245..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py +++ /dev/null @@ -1,9 +0,0 @@ -"""SQL line comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_line_comment_impl - - -def skip_line_comment(*, sql: str, start: int) -> int: - """Skip past an SQL line comment.""" - - return skip_line_comment_impl(sql=sql, start=start) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py deleted file mode 100644 index b131f0fbf3..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Quoted SQL text scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_quoted_text_impl - - -def skip_quoted_text(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past a quoted string starting at the supplied position.""" - - return skip_quoted_text_impl(sql=sql, start=start, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/types.py b/src/sqlbuild/compiler/sql_analysis/types.py index 266c536fef..717f5b0af9 100644 --- a/src/sqlbuild/compiler/sql_analysis/types.py +++ b/src/sqlbuild/compiler/sql_analysis/types.py @@ -62,11 +62,3 @@ def BindingPositions( ) -> NativeBindingPositions: ... def normalize_analysis_sql(self, request: dict[str, object]) -> str: ... def normalize_dialect_sql(self, *, sql: str, dialect: str) -> str: ... - - -class NativeValidationModule(Protocol): - """Native schema-validation boundary.""" - - def validate_sql_with_schema_json(self, request_json: str) -> str: ... - - def validate_sql_with_schemas_json(self, request_json: str) -> str: ... diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 77ce4fd6b9..d25ac65540 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -8,10 +8,9 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import dbt_failure_detail from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py index 0a835ab1e6..8b49a3203f 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.config._parse_config_overrides import ( diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index d0235ec799..4c7480a36a 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -7,6 +7,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN @@ -21,7 +22,6 @@ ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 477b9bce3d..2921cfd340 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -8,11 +8,11 @@ from dataclasses import dataclass, field, replace from pathlib import Path +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.lint._helpers.native_format import with_newline_style from sqlbuild.lint.constants import ( LINT_ENGINE_NATIVE, @@ -217,7 +217,9 @@ def _relation_keys(self) -> RelationKeys: if self.baseline is None: try: - self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + self.baseline = build_compiled_project( + discovered_inputs=self.inputs, adapter=self.adapter + ) except CompileInputError: return {} return compiled_relation_keys(self.baseline) @@ -228,12 +230,14 @@ def _verdict(self, *, files: dict[Path, str]) -> FixVerdict | None: verdict: FixVerdict | None try: if self.baseline is None: - self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + self.baseline = build_compiled_project( + discovered_inputs=self.inputs, adapter=self.adapter + ) if self.baseline_facts is None: self.baseline_facts = compile_facts( project=self.baseline, project_dir=self.project_dir ) - compiled: CompiledProjectView = compile_project( + compiled: CompiledProjectView = build_compiled_project( discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), adapter=self.adapter, ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 2fbbda280c..7096622f1d 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -6,11 +6,10 @@ from pathlib import Path from tempfile import TemporaryDirectory +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine.config import load_rules_config from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, @@ -110,4 +109,4 @@ def _toml_value(*, value: object) -> str: def _compile_rule_project(*, root: Path) -> CompiledProjectView: discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=root) adapter: DuckDbAdapter = DuckDbAdapter() - return compile_project(discovered_inputs=discovered, adapter=adapter) + return build_compiled_project(discovered_inputs=discovered, adapter=adapter) diff --git a/src/sqlbuild/runtime/event_exporting/main/configure_discovered_event_exporters.py b/src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py similarity index 100% rename from src/sqlbuild/runtime/event_exporting/main/configure_discovered_event_exporters.py rename to src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 6e246632b2..1161eb5b16 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -110,11 +110,11 @@ import sqlbuild._native as native -def _panicked(request): +def _panicked(*arguments): raise native.NativeCompilerError({_NATIVE_PANIC_ERROR!r}) -native.build_rich_column_lineage = _panicked +native.build_project_column_lineage = _panicked """ _RICH_COMPILE: tuple[str, ...] = ( "compile", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index fca7d74599..49f6b58631 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,6 +31,7 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization +import sqlbuild.cli.commands._helpers.compile.lineage as compile_lineage import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse.main._write_reusable_compile as write_reusable_module @@ -42,7 +43,6 @@ import sqlbuild.compiler.contracts.main.promotion_conflicts as promotion_conflicts import sqlbuild.compiler.contracts.main.validate as contract_validation import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module -import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( @@ -4843,7 +4843,7 @@ def full_macro_call_runs( (diagnostic_recovery, "complete_native_semantic_diagnostics"), (contract_validation, "evaluate_native_model_contracts"), (promotion_conflicts, "native_promotion_conflict_diagnostics"), - (column_lineage, "build_native_column_lineage"), + (compile_lineage, "build_compile_lineage"), (target_writer, "plan_native_sql_test_artifacts"), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index ff8ae2d638..e8f977cece 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,9 +8,9 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly @@ -36,7 +36,7 @@ "complete_native_semantic_diagnostics": [ANY], "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], - "build_native_column_lineage": [ANY], + "build_compile_lineage": [ANY], "plan_native_sql_test_artifacts": [ANY], }, ) @@ -62,7 +62,7 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_seams == test_case.expected_native_returns planned_counts: list[int] = [] for seams in (native_seams,): - assert isinstance(seams["build_native_column_lineage"][0], ProjectColumnLineage) + assert isinstance(seams["build_compile_lineage"][0], ProjectColumnLineage) assert isinstance(seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) assert seams["analyze_native_model_sql"][0].session is not None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index ab827cba70..968e55733b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -139,6 +139,16 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: '\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' ), } +ORDER_AMOUNT_CHECK: dict[str, str] = { + "python/checks/order_amounts.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import model\n\n\n" + '@check(depends_on=model("stg_orders"))\n' + "def order_amounts_present(ctx):\n" + ' """Compare with: select amount from stg_orders."""\n' + " return ctx.pass_()\n" + ), +} CUSTOMER_DECLARATIONS: dict[str, str] = { "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", "seeds/customers.yml": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index da5f81b6ae..a46b665363 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -18,6 +18,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( CENTS_MACRO, CUSTOMER_DECLARATIONS, + ORDER_AMOUNT_CHECK, ORDER_EXPORT, ORDER_SHAPE, ORDERS_MACRO, @@ -166,6 +167,14 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( expected_reason="would take macro:to_dollars along, which must stay in", expected_paths=("models/staging/_sqlbuild/_macros/cents.py",), ), + RefusedRefactorE2ETestCase( + description="column named in Python SQL text needs a manual check", + command=("rename", "column:stg_orders.amount", "revenue"), + extra_files=ORDER_AMOUNT_CHECK, + expected_status="refused", + expected_reason="Python SQL may read column amount of stg_orders", + expected_paths=("python/checks/order_amounts.py",), + ), RefusedRefactorE2ETestCase( description="rename onto an existing model", command=("rename", "model:stg_orders", "fact_orders"), diff --git a/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json b/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json index 4ec7fcb8f0..3a75d38701 100644 --- a/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json +++ b/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json @@ -564,18 +564,6 @@ "c44c8633473bde950e573766" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1140,18 +1128,6 @@ "f48ca046110efa93dc8e7b54" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1716,18 +1692,6 @@ "7f3cc4630798c6d611e9c144" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2292,18 +2256,6 @@ "767dbad2dc0bb5d013f7a196" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2868,18 +2820,6 @@ "165df7e540546e89f76b0819" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -3442,17 +3382,5 @@ [ "catalog", "8808b0d353112b97c19536d1" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" ] ] diff --git a/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json b/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json index b3c3149315..0fe10d8eb3 100644 --- a/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json +++ b/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json @@ -564,18 +564,6 @@ "9c7db1355dd2394241b43e81" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1140,18 +1128,6 @@ "b83e75232474640290815c9d" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1716,18 +1692,6 @@ "032717ee1f3f581899d54bf7" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2292,18 +2256,6 @@ "3d1560ed382e916606e5f85f" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2868,18 +2820,6 @@ "f19e292dff42d4c08b5da8ad" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -3442,17 +3382,5 @@ [ "catalog", "23b5038345131be7d003ad7b" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" ] ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 383fe8c38c..7aa8240085 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1071,3 +1071,12 @@ class InferredFreshnessCliTestCase: description: str sources_yaml: str expected_sources: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class BuildManifestTestCase: + """One real `sqb build --manifest` run and the manifest nodes it must publish.""" + + description: str + model_count: int + expected_test_type: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 9b8f37cf44..678f20d622 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -13,7 +13,6 @@ import duckdb import pytest -from sqlbuild._native import NativeProjectTree from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -29,9 +28,8 @@ from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.analysis_session.models import NativeQueryColumns from sqlbuild.compiler.compile._helpers.assembly import source_bindings as source_bindings_module -from sqlbuild.compiler.discovery._helpers.native import ( - model_files as native_model_files_module, -) +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( @@ -649,18 +647,23 @@ def recording(**kwargs: Any) -> tuple[NativeQueryColumns, ...]: return analysed -def record_eager_output_column_scans(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: - """Record every native model discovery request that locates output columns eagerly.""" +def eager_output_column_scans(*, capture_dir: Path) -> tuple[int, int]: + """Count the captured model files, and those discovery located output columns for eagerly.""" + + located: list[bool] = [] + + def record(item: dict[str, object]) -> dict[str, object]: + located.extend(map(bool, map(item.get, filter(item.__contains__, _OUTPUT_LOCATIONS)))) + return item - eager: list[dict[str, object]] = [] - original: Callable[..., object] = native_model_files_module._native.discover_model_files + for capture in sorted( + capture_dir.glob(f"*-{CompilerStage.DISCOVERED_PROJECT_INPUTS.value}{STAGE_CAPTURE_SUFFIX}") + ): + _ = json.loads(capture.read_text(encoding="utf-8"), object_hook=record) + return len(located), sum(located) - def recording(request: dict[str, object], tree: NativeProjectTree, project_dir: Path) -> object: - eager.extend(filter(lambda item: item["extract_output_column_locations"], (request,))) - return original(request, tree, project_dir) - monkeypatch.setattr(native_model_files_module._native, "discover_model_files", recording) - return eager +_OUTPUT_LOCATIONS: tuple[str, ...] = ("output_column_locations",) def write_repeated_json_parse_project( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py b/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py new file mode 100644 index 0000000000..f83b1fc74c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py @@ -0,0 +1,64 @@ +"""Integration coverage for the manifest a real build publishes.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import BuildManifestTestCase +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + run_build, + write_test_planning_project, +) + + +@pytest.mark.parametrize( + "test_case", + ( + BuildManifestTestCase( + description="planned models and SQL tests reach the build manifest", + model_count=3, + expected_test_type="sql_native", + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_project_when_building_with_manifest_then_plan_nodes_are_published( + test_case: BuildManifestTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project: Path = tmp_path / "orders" + write_test_planning_project( + project_dir=project, + model_count=test_case.model_count, + rules_config=f'\n[connection]\ndatabase = "{project / "orders.duckdb"}"\n', + ) + + exit_code, output = run_build(project_dir=project, flags=("--manifest",), capsys=capsys) + + assert exit_code == 0, output + contents: bytes = (project / "target" / "manifest.json").read_bytes() + manifest: dict[str, Any] = json.loads(contents) + assert contents.endswith(b"}\n") + nodes: dict[str, Any] = manifest["nodes"] + model_ids: list[str] = [ + f"model.orders.orders_{index:03}" for index in range(test_case.model_count) + ] + assert Counter( + (node["resource_type"], node["meta"].get("sqlbuild_test_type")) for node in nodes.values() + ) == { + ("model", None): test_case.model_count, + ("test", test_case.expected_test_type): test_case.model_count - 1, + } + assert nodes[model_ids[1]]["compiled_code"].startswith("SELECT") + assert model_ids[0] in manifest["parent_map"][model_ids[1]] + assert model_ids[1] in manifest["child_map"][model_ids[0]] + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index 249dd2d798..c98268963a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -41,7 +41,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor ]) Path(sys.argv[2]).write_text(json.dumps({ "aggregate_models": "sqlbuild.cli.commands.models.runtime" in sys.modules, - "dag": "sqlbuild.compiler.dag.main.build" in sys.modules, "manifest": "sqlbuild.compiler.manifest.main.build" in sys.modules, }), encoding="utf-8") raise SystemExit(exit_code) @@ -59,7 +58,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor assert summary["models"] == test_case.expected_count assert json.loads(import_status_path.read_text(encoding="utf-8")) == { "aggregate_models": False, - "dag": False, "manifest": False, } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 53c3a4c471..c359a31d5c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -7,10 +7,11 @@ import pytest from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import DiscoveryWorkTestCase from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + eager_output_column_scans, execute_duckdb_sql, - record_eager_output_column_scans, write_expression_source_project, ) @@ -57,14 +58,19 @@ def test_given_project_when_running_command_then_discovery_skips_eager_column_lo 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), ) - scanned: list[dict[str, object]] = record_eager_output_column_scans(monkeypatch) + capture_dir: Path = tmp_path / "captures" + monkeypatch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(capture_dir)) _ = capsys.readouterr() exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command]) output: str = "".join(capsys.readouterr()) - assert exit_code == test_case.expected_exit_code, output - assert len(scanned) == test_case.expected_eager_output_column_scans + model_files, eager_scans = eager_output_column_scans(capture_dir=capture_dir) + assert (exit_code, model_files > 0, eager_scans) == ( + test_case.expected_exit_code, + True, + test_case.expected_eager_output_column_scans, + ), output if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 4cc8af7813..66259d0b1d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -13,6 +13,7 @@ from _pytest.monkeypatch import MonkeyPatch from duckdb import DuckDBPyConnection +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.load.selection import select_load_entries @@ -32,7 +33,6 @@ TestCommandRequest, ) from sqlbuild.cli.output.main._load_execution_json import format_load_execution_json -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 3edd452745..2bfb3c12c1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -13,7 +13,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module -import sqlbuild.compiler.compile.main._build_compile_inputs as compile_inputs_module +import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rules as rules_module @@ -810,7 +810,7 @@ def counting_build( compile_cache_dir=compile_cache_dir, ) - monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -877,9 +877,9 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo macro_loads.append(loaded) return loaded - monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(compile_inputs_module, "load_project_macros", recording_load) + monkeypatch.setattr(macros_module, "load_project_macros", recording_load) monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index cfb0c8ed1b..dc5bba1d5b 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -12,6 +12,7 @@ type LauncherRun = tuple[int, bytes, bytes] _ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") +_PHASE_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d+s\)") NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") PROJECT_FILES: dict[str, str] = { @@ -43,7 +44,7 @@ def write_project(project_dir: Path) -> Path: def run(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: - """Run one CLI process with reuse enabled; return its exit code, stdout and stderr.""" + """Run one CLI process with reuse enabled; return its exit code, stdout and timeless stderr.""" completed: subprocess.CompletedProcess[bytes] = subprocess.run( [str(program), *args], @@ -53,7 +54,11 @@ def run(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: env={**os.environ, "NO_COLOR": "1", REUSE_DISABLE_ENV_VAR: "0"}, timeout=600, ) - return completed.returncode, completed.stdout, completed.stderr + return ( + completed.returncode, + completed.stdout, + _PHASE_SECONDS.sub(b"(s)", completed.stderr), + ) def launch_records(project_dir: Path) -> dict[Path, int]: diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 98dc669867..b919d9b2d3 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -14,8 +14,9 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main @@ -52,7 +53,6 @@ ) from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, CompactLineageFacts, @@ -61,8 +61,11 @@ InferredColumn, ModelSqlAnalysis, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompileInputsView, ModelInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompileInputsView, + ModelInput, +) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from sqlbuild.sql_values.types import CollectionRendering @@ -644,19 +647,20 @@ def recorded_shapes( runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native - assembled_model: Callable[..., CompiledModelView] = project_assembly._assemble_compiled_model + completed: Callable[..., CompiledProjectView] = semantic_recovery.complete_semantic_diagnostics - def recorded_model(*arguments: Any, **keywords: Any) -> CompiledModelView: - model: CompiledModelView = assembled_model(*arguments, **keywords) - runs.golden.append(golden_entry("proof", model.dynamic_column_contract)) - return model + def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: + runs.golden.extend( + golden_entry("proof", model.dynamic_column_contract) for model in project.models + ) + return completed(project=project, **keywords) with monkeypatch.context() as patch: - patch.setattr(project_assembly, "_assemble_compiled_model", recorded_model) - patch.setattr(project_assembly, "analyze_model_sql", analysed_three_ways) - patch.setattr(project_assembly, "expression_source_shapes", recorded_shapes) + patch.setattr(semantic_recovery, "complete_semantic_diagnostics", recorded_proofs) + patch.setattr(native_stage_assembly, "analyze_model_sql", analysed_three_ways) + patch.setattr(native_stage_assembly, "expression_source_shapes", recorded_shapes) with suppress(CompileInputError): - _ = project_assembly.assemble_compiled_project( + _ = native_module.assemble_project( inputs=inputs, inference_profile=replace(inference_profile), column_lineage_mode=lineage_mode, @@ -701,8 +705,8 @@ def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: monkeypatch.context() as patch, suppress(CompileInputError, native_module.NativeCompilerError), ): - patch.setattr(project_assembly, "analyze_model_sql", captured) - _ = project_assembly.assemble_compiled_project( + patch.setattr(native_stage_assembly, "analyze_model_sql", captured) + _ = native_module.assemble_project( inputs=inputs, inference_profile=inference_profile, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 1590999301..5590867d80 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -11,6 +11,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -18,7 +19,6 @@ analyze_native_model_sql, ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( @@ -399,7 +399,7 @@ def test_given_internal_native_failure_when_assembling_then_raises_without_pytho duplicate_analysed_model_names(monkeypatch=monkeypatch) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = assemble_compiled_project( + _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), ) @@ -443,7 +443,7 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ) recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) - project: CompiledProjectView = assemble_compiled_project( + project: CompiledProjectView = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), analysis_model_names=test_case.analysed_models, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index cc21e4bde6..3e507ee1c6 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,7 +12,7 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( infer_native_query_columns, @@ -132,7 +132,7 @@ def test_given_session_without_model_analysis_when_assembling_then_raises_native test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - project_assembly, + native_stage_assembly, "analyze_model_sql", lambda _request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 8e7d795f39..0895685c2b 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -14,10 +14,10 @@ import pytest +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, LoadedMacro, @@ -25,7 +25,6 @@ MacroExpansionResult, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index de07a054b3..7b2dfaa4e3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -13,14 +13,18 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileModelConfig, @@ -36,9 +40,7 @@ CompileInputsView, ) from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering @@ -432,7 +434,7 @@ def compile_source_project( write_files( project_dir=project_dir, files={**SOURCE_PROJECT_FILES, "sources/raw.yml": source_yml} ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, @@ -478,7 +480,7 @@ def compile_function_project( """Compile a project of `files`; return the project and its compile inputs.""" write_files(project_dir=project_dir, files=files) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, @@ -633,7 +635,7 @@ def _hand_built_model(name: str, sql: str) -> CompileModelInput: def hand_built_analysed_project(models: dict[str, str]) -> CompiledProjectView: """Assemble hand-built model inputs, which cross the explicit analysis boundary.""" - return assemble_compiled_project( + return assemble_project( inputs=CompileProjectInputs( project_config=_HAND_BUILT_CONFIG, local_config=LocalConfig(), @@ -660,7 +662,7 @@ def native_analysed_project(*, project_dir: Path, models: dict[str, str]) -> Com }, }, ) - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 08cd87cdac..e45dc2ab34 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -8,13 +8,12 @@ import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, @@ -72,7 +71,7 @@ def test_given_repeated_output_schemas_when_serializing_then_columns_share_immut }, }, ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index fb7b8a8336..f1b4bb5061 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -6,14 +6,13 @@ import pytest +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 8ead2728e1..56de61f101 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -12,7 +12,7 @@ import pytest import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -26,7 +26,6 @@ from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType from sqlbuild.spec.contracts.models import ( diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index a8cabf8e17..27b6a18d25 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -126,10 +126,10 @@ class UndecodablePathToleranceTestCase: @dataclass(frozen=True) class NativeOsErrorTestCase: - """An operating-system read payload and the Python error it becomes.""" + """An OS read failure `(errno, winerror, message)` and the Python error it becomes.""" description: str - payload: tuple[object, ...] + payload: tuple[int | None, int | None, str] expected_type: str expected_fields: tuple[object, ...] diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index a5413fe7f3..19f32584fb 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -20,19 +20,17 @@ from typing import Any, cast from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( - build_tolerant_scope_discovery, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_model_schema_files, + discover_project_inputs, discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_payload_error, +from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( + build_tolerant_scope_discovery, ) from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, @@ -45,7 +43,6 @@ from sqlbuild.compiler.discovery.main._model_description_inputs import ( discover_model_description_inputs, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, @@ -464,24 +461,25 @@ def random_undecodable_bytes(*, rng: random.Random) -> bytes: def native_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: """Return the type and text of the read error native reading reports for each file, if any.""" - payloads: list[tuple[object, ...]] = cast( - list[tuple[object, ...]], + return [ + {True: (type(row).__name__, str(row)), False: ("NoneType", "None")}[ + isinstance(row, UnicodeError | OSError) + ] + for row in _native_yaml_rows(project_dir=project_dir, relative_paths=relative_paths) + ] + + +def _native_yaml_rows(*, project_dir: Path, relative_paths: list[str]) -> list[object]: + """Each file's `(contents, loaded)`, or the error native source YAML loading raises for it.""" + + return cast( + list[object], _native.load_yaml_files( {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, relative_paths, _native.NativeProjectTree(str(project_dir)), ), ) - errors: list[Exception | None] = [ - native_payload_error(payload=payload, file_path=project_dir / path) - for payload, path in zip(payloads, relative_paths, strict=True) - ] - return [ - {True: (type(error).__name__, str(error)), False: ("NoneType", "None")}[ - isinstance(error, UnicodeError | OSError) - ] - for error in errors - ] def python_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: @@ -581,7 +579,7 @@ def declaration_files_outcome(*, project_dir: Path, kind: str) -> tuple[object, def native_declaration_tag(*, file_path: Path, kind: str) -> str: """Return whether native parsing parses, fails or defers one file's contents.""" - payload: tuple[object, ...] = _native.parse_declaration_contents( + return _native.parse_declaration_contents( { "kind": kind, "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), @@ -591,7 +589,6 @@ def native_declaration_tag(*, file_path: Path, kind: str) -> str: (str(file_path), file_path.stem), file_path.read_text(encoding="utf-8"), ) - return str(payload[0]) def tolerant_declaration_outcome( @@ -1035,18 +1032,14 @@ def deep_model_header_values(*, project_dir: Path, frames: int) -> dict[str, obj def native_yaml_tags_and_values( *, project_dir: Path, relative_paths: list[str] ) -> tuple[list[object], list[object]]: - """Return each file's native tag and the values of the files that loaded.""" + """Return whether each file loaded (`ok`) or failed (`error`), and the loaded values.""" - rows: list[tuple[object, ...]] = cast( - list[tuple[object, ...]], - _native.load_yaml_files( - {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, - relative_paths, - _native.NativeProjectTree(str(project_dir)), - ), + rows: list[object] = _native_yaml_rows(project_dir=project_dir, relative_paths=relative_paths) + failed: list[bool] = [isinstance(row, Exception) for row in rows] + loaded: list[tuple[str, object]] = cast( + list[tuple[str, object]], list(compress(rows, [not failure for failure in failed])) ) - loaded: list[tuple[object, ...]] = list(compress(rows, [row[0] == "ok" for row in rows])) - return [row[0] for row in rows], [row[2] for row in loaded] + return [{True: "error", False: "ok"}[failure] for failure in failed], [row[1] for row in loaded] VALIDATION_MODEL: tuple[str, bytes] = ("models/orders.sql", b"MODEL ();\nSELECT 1 AS order_id") diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 47249a2c5f..5e1eef9078 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -8,7 +8,7 @@ import pytest from sqlbuild import _native -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index 92108dc62d..4e5e6736bb 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -7,8 +7,8 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.model_files import ( - discover_native_model_files, +from sqlbuild._native import ( + discover_model_files, ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile @@ -52,7 +52,7 @@ def test_given_runtime_and_root_when_discovering_natively_then_supported_runs_an with capture: models.extend( - discover_native_model_files( + discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -82,7 +82,7 @@ def test_given_native_model_walk_when_globbing_models_later_in_the_pass_then_the ) -> None: write_project(project_dir=tmp_path, files=(("models/marts/orders.sql", _VALID_MODEL),)) with DirectorySnapshot.scope(project_dir=tmp_path) as snapshot: - _ = discover_native_model_files( + _ = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=True, extract_output_column_locations=True, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py index af15c466c9..cbc207d2f9 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.payloads import native_read_error +import sqlbuild._native as _native from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( NativeOsErrorTestCase, ReadErrorOracleTestCase, @@ -128,29 +128,29 @@ def test_given_unreadable_file_when_discovering_then_python_read_error_is_raised [ NativeOsErrorTestCase( description="a refused read is Python's errno error naming the file", - payload=("read", "os", 13, "Permission denied (os error 13)", None), + payload=(13, None, "Permission denied (os error 13)"), expected_type="PermissionError", expected_fields=(13, "Permission denied", "orders.sql"), ), NativeOsErrorTestCase( description="a Windows listing error carries its Win32 code for Python to map", - payload=("read", "os", None, "Access is denied", 5), + payload=(None, 5, "Access is denied"), expected_type="OSError", expected_fields=(None, "Access is denied", "orders.sql"), ), NativeOsErrorTestCase( description="an error without a code keeps the operating system message", - payload=("read", "os", None, "stream did not contain valid UTF-8", None), + payload=(None, None, "stream did not contain valid UTF-8"), expected_type="OSError", expected_fields=(None, None, None), ), ], ids=lambda case: case.description, ) -def test_given_os_read_payload_when_building_error_then_python_error_matches_platform( +def test_given_os_read_failure_when_building_error_then_python_error_matches_platform( test_case: NativeOsErrorTestCase, ) -> None: - error: Exception = native_read_error(payload=test_case.payload, file_path=Path("orders.sql")) + error: Exception = _native._oracle_read_error(*test_case.payload, file_path=Path("orders.sql")) assert ( type(error).__name__, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py index adef309087..784ea0cd32 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.exceptions import ProjectPathError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( UndecodableDeclarationNameTestCase, UndecodablePathToleranceTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 4765de1b71..fe1c968921 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -7,16 +7,14 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompilerDiagnostic, LiteralSqlRelation, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.sql_values.types import CollectionRendering _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py index 5abec348b2..b8ae9adfd3 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py @@ -23,7 +23,7 @@ class FormerlyDeferredLineageTestCase: description: str dialect: str keeps_catalog: bool - expected_native_statuses: dict[str, int] + expected_models_with_lineage: int expected_error: str | None @@ -33,7 +33,6 @@ class UnparsedLineageTestCase: description: str query_sql: str - expected_status: str expected_messages: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index a028d5803e..df42624aac 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -2,24 +2,20 @@ import random from collections import Counter -from collections.abc import Callable, Sequence +from collections.abc import Callable from dataclasses import replace from pathlib import Path -from typing import Any, cast +from typing import cast import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph +from sqlbuild._native import ProjectColumnLineage, build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage +from sqlbuild.compiler.lineage.types import ColumnLineageMode _PROJECT_TOML: str = ( 'name = "orders_lineage"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' @@ -244,11 +240,24 @@ def lineage_views( lineage: ProjectColumnLineage = cast( ProjectColumnLineage, - build_native_column_lineage(project=project, dialect=dialect, model_names=model_names), + fast_column_lineage(project=project, dialect=dialect, model_names=model_names), ) return _public_views(project=project, lineage=lineage) +def fast_column_lineage( + *, + project: CompiledProjectView, + dialect: str | None, + model_names: frozenset[str] | None, +) -> ProjectColumnLineage | None: + """The native fast lineage graph, or None without SQL analysis.""" + + return native_module.build_project_column_lineage( + project, dialect, ColumnLineageMode.FAST, model_names + ) + + def _public_views( *, project: CompiledProjectView, lineage: ProjectColumnLineage ) -> tuple[list[object], list[object]]: @@ -280,18 +289,21 @@ def lineage_log_records(caplog: pytest.LogCaptureFixture) -> list[tuple[str, str ] -def record_native_outcomes(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count the native engine's per-model outcome statuses for the rest of the test.""" +def fast_lineage_work( + *, project: CompiledProjectView, model_names: frozenset[str] | None +) -> Counter[str]: + """Count the selected models fast lineage parses and those whose root star it expands.""" - statuses: Counter[str] = Counter() - build: Callable[..., Sequence[tuple[str, object, bool, str | None]]] = ( - native_module.build_fast_column_lineage + names: frozenset[str] = model_names or frozenset(model.name for model in project.models) + return Counter( + parse=sum( + model.name in names and model.fast_lineage_columns is None for model in project.models + ), + star=sum( + model.name in names + and model.fast_lineage_columns is not None + and model.fast_lineage_has_star + and not model.fast_lineage_star_resolved + for model in project.models + ), ) - - def counted(*arguments: Any) -> Sequence[tuple[str, object, bool, str | None]]: - outcomes: Sequence[tuple[str, object, bool, str | None]] = build(*arguments) - statuses.update(outcome[0] for outcome in outcomes) - return outcomes - - monkeypatch.setattr(native_module, "build_fast_column_lineage", counted) - return statuses diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index a71200243c..44fddb43c8 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -7,17 +7,15 @@ from collections import Counter from dataclasses import replace from pathlib import Path +from typing import cast import pytest import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -34,10 +32,11 @@ ) from tests.integration.src.sqlbuild.compiler.lineage.helpers import ( compiled_project, + fast_column_lineage, + fast_lineage_work, generated_lineage_files, lineage_log_records, lineage_views, - record_native_outcomes, without_compact_facts, ) @@ -62,10 +61,9 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_projects_agree( test_case: GeneratedLineageParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) + work: Counter[str] = Counter() differences: list[tuple[object, object, object]] = [] golden: list[GoldenEntry] = [] for index in range(test_case.count): @@ -79,6 +77,8 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje for variant in (project, without_compact_facts(project)): for dialect in test_case.dialects: for model_names in (None, selected): + for _ in ("with catalog", "restored"): + work.update(fast_lineage_work(project=variant, model_names=model_names)) names, with_catalog = lineage_views( project=variant, dialect=dialect, model_names=model_names ) @@ -109,10 +109,9 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje == [] ) assert differences == [] - assert statuses["deferred"] == 0 - assert sum(statuses.values()) >= test_case.expected_minimum_native - assert statuses["built"] + statuses["omitted"] >= test_case.expected_minimum_parsed - assert statuses["star"] >= test_case.expected_minimum_star_expansions + assert sum(work.values()) >= test_case.expected_minimum_native + assert work["parse"] >= test_case.expected_minimum_parsed + assert work["star"] >= test_case.expected_minimum_star_expansions @pytest.mark.parametrize( @@ -121,13 +120,11 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje UnparsedLineageTestCase( description="an unterminated subquery", query_sql='SELECT order_id FROM (SELECT * FROM __ref("orders_model_0")', - expected_status="unparsed", expected_messages=("fast column lineage parse failed; falling back",), ), UnparsedLineageTestCase( description="two statements", query_sql="SELECT 1; SELECT 2", - expected_status="unparsed", expected_messages=("fast column lineage parse failed; falling back",), ), ], @@ -136,10 +133,8 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_it( test_case: UnparsedLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(11), model_count=_MODEL_COUNT), @@ -157,14 +152,13 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i ) with caplog.at_level(logging.DEBUG, logger="sqlbuild.lineage"): - native: ProjectColumnLineage | None = build_native_column_lineage( + native: ProjectColumnLineage | None = fast_column_lineage( project=broken, dialect=None, model_names=None ) native_records: list[tuple[str, str, object]] = lineage_log_records(caplog) assert native is not None assert not native.has_model(project.models[0].name) - assert statuses[test_case.expected_status] == 1 assert tuple(message for _, message, _ in native_records) == test_case.expected_messages @@ -175,14 +169,14 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i description="a dialect outside the old native parser build", dialect="mysql", keeps_catalog=True, - expected_native_statuses={"built": _MODEL_COUNT}, + expected_models_with_lineage=_MODEL_COUNT, expected_error=None, ), FormerlyDeferredLineageTestCase( description="a project restored without its analysis catalog", dialect="duckdb", keeps_catalog=False, - expected_native_statuses={"built": _MODEL_COUNT}, + expected_models_with_lineage=_MODEL_COUNT, expected_error=None, ), ], @@ -191,9 +185,7 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_answers( test_case: FormerlyDeferredLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", @@ -204,9 +196,13 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a project=compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] ) - _ = lineage_views(project=project, dialect=test_case.dialect, model_names=None) + lineage: ProjectColumnLineage = cast( + ProjectColumnLineage, + fast_column_lineage(project=project, dialect=test_case.dialect, model_names=None), + ) + models_with_lineage: int = sum(lineage.has_model(model.name) for model in project.models) - assert statuses == Counter(test_case.expected_native_statuses), test_case.description + assert models_with_lineage == test_case.expected_models_with_lineage, test_case.description @pytest.mark.parametrize( @@ -216,7 +212,7 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a description="a dialect name Polyglot does not know raises Python's error", dialect="nonsense", keeps_catalog=True, - expected_native_statuses={"unknown_dialect": _MODEL_COUNT}, + expected_models_with_lineage=0, expected_error="Unknown dialect: nonsense", ), ], @@ -225,9 +221,7 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_raises( test_case: FormerlyDeferredLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", @@ -241,8 +235,6 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais with pytest.raises(ValueError, match=test_case.expected_error): _ = lineage_views(project=project, dialect=test_case.dialect, model_names=None) - assert statuses == Counter(test_case.expected_native_statuses), test_case.description - @pytest.mark.parametrize( "test_case", @@ -279,7 +271,7 @@ def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compil ) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = build_native_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) + _ = fast_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) assert str(raised.value) == test_case.expected_message diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 8800958555..fa3f41315b 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -12,14 +12,13 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( resolve_effective_collection_rendering, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 0c9ead54d6..afbeadcaeb 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,21 +8,23 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ( + build_compile_inputs, + build_compiled_project, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledLineageColumnFact, ) from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.sql_values.types import CollectionRendering @@ -119,15 +121,17 @@ def build_manifest_for_pipeline_result( project_name: str, adapter_type: str, ) -> dict[str, object]: - """Build the manifest artifact from a compile pipeline result.""" - - return build_manifest( - project=result.project, - plan_output=result.plan_output, - project_name=project_name, - adapter_type=adapter_type, - upstream_deps=result.plan_output.upstream_deps, - downstream_deps=result.plan_output.downstream_deps, + """Build the manifest artifact from a compile pipeline result and parse its JSON.""" + + return json.loads( + build_manifest( + project=result.project, + plan_output=result.plan_output, + project_name=project_name, + adapter_type=adapter_type, + upstream_deps=result.plan_output.upstream_deps, + downstream_deps=result.plan_output.downstream_deps, + ).contents() ) @@ -172,7 +176,7 @@ def compile_reshaped_star_model( f"MODEL (description 'Test model reshaped_orders.', materialized table);\n{query_sql}", encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ) @@ -237,7 +241,7 @@ def postgres_semantic_shapes( path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=PostgresAdapter(), resolved_connection=_OFFLINE_POSTGRES_CONNECTION, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index 3ae7073678..b8323417ee 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.diff import ( compile_project_for_diff_environment, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index c53e80cd4f..3c250b11c2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -5,14 +5,13 @@ import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( GroupedDeclarationCompileTestCase, @@ -72,7 +71,7 @@ def test_given_active_adapter_when_macro_renders_constant_then_uses_adapter_sql( adapter: BaseAdapter = test_case.adapter_factory() discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index af90e57ef7..315866eb17 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -7,20 +7,18 @@ import pytest -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_compiled_project, build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.main import compile as compile_pipeline -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.runtime.contracts.main import open_connection as compile_pipeline from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, @@ -99,7 +97,7 @@ def test_given_typed_cte_chain_when_compiling_contract_then_type_remains_proven( write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=DuckDbAdapter(), ) @@ -567,7 +565,7 @@ def test_given_snowflake_local_override_without_target_namespace_when_compiling_ discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) with pytest.raises(ValueError, match=test_case.expected_error_fragment): - compile_project( + build_compiled_project( discovered_inputs=discovered_inputs, adapter=SnowflakeAdapter(), no_sql_validation=True, @@ -610,7 +608,7 @@ def test_given_snowflake_local_target_namespace_when_compiling_then_targets_reso write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=SnowflakeAdapter(), no_sql_validation=True, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index ac470932cd..14926d931c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main @@ -15,7 +15,6 @@ from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( NativeCatalogCase, ValidationCacheCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index c4f925d99d..201b39b40c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -6,11 +6,10 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SnowflakeOutputInferenceCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index d5fc551a24..808fea1149 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -8,14 +8,13 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( FunctionArgumentTypeCase, SemanticTriageCase, @@ -227,7 +226,7 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma repeated_validation: Mock = Mock( side_effect=AssertionError("cold compile repeated fused validation") ) - patch.setattr(native, "validate_sql_with_schemas_json", repeated_validation) + patch.setattr(native, "schema_validation_rows", repeated_validation) assert main(args) == 0 repeated_validation.assert_not_called() capsys.readouterr() @@ -235,7 +234,7 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma forbidden: Mock = Mock( side_effect=AssertionError("warm compile repeated native semantic work") ) - patch.setattr(native, "validate_sql_with_schemas_json", forbidden) + patch.setattr(native, "schema_validation_rows", forbidden) patch.setattr(native, "analyze_project_queries_compact_json", forbidden) assert main(args) == 0 forbidden.assert_not_called() diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index b0641ccc5f..a932dd2486 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -9,13 +9,12 @@ import pytest from duckdb import sqltypes +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SetOperationLineageIntegrationTestCase, SetOperationTypeIntegrationTestCase, @@ -60,7 +59,7 @@ def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) @@ -112,7 +111,7 @@ def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index a7d2d4464d..d55fbccb50 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -5,14 +5,13 @@ import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ExpectedCountTestCase @@ -53,7 +52,7 @@ def test_given_expanded_mixed_mode_tests_when_compiling_project_then_complete_in }, ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 3ef08e7615..a008d35f29 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, @@ -16,7 +16,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index eb63b583cc..eafda6ddd7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -5,11 +5,10 @@ import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import SqlTestPlanEntry, SqlTestSelection from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -66,7 +65,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) @@ -135,7 +134,7 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 94fec55212..7c839aea91 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import Any, cast +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -37,7 +38,6 @@ ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSchemaFile, @@ -47,7 +47,6 @@ from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 38cdc4e4bc..5b7ea297e8 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,17 +13,14 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.assembly import project as project_module -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering @@ -160,7 +157,7 @@ def assemble_with( ) -> CompiledProjectView: """Assemble `inputs`.""" - return assemble_compiled_project(inputs=inputs) + return assemble_project(inputs=inputs) def recorded_assembly( @@ -234,9 +231,9 @@ def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: calls: Counter[str] = Counter() _ = [ monkeypatch.setattr( - project_module._native, + _native, name, - _counted(name=name, function=getattr(project_module._native, name), calls=calls), + _counted(name=name, function=getattr(_native, name), calls=calls), ) for name in _PYTHON_RESOURCE_FUNCTIONS ] diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index c099f7b21f..795b552976 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -10,8 +10,8 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.project_assembly.types import ProjectRequestRow @@ -342,7 +342,7 @@ def test_given_native_assembly_panics_when_assembling_then_raises_without_python calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = assemble_compiled_project( + _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect=test_case.dialect), ) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 451d1d21c5..fd1c905a1e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -13,6 +13,7 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope @@ -26,7 +27,6 @@ LoadedMacro, ) from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredProjectInputs, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 6bdde6b886..578c6a74b6 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,9 +13,9 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -25,7 +25,6 @@ from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( @@ -209,7 +208,7 @@ def capture(**arguments: Any) -> CompiledProjectView: return complete_semantic_diagnostics(**arguments) with monkeypatch.context() as patch: - patch.setattr(assembly_project, "complete_semantic_diagnostics", capture) + patch.setattr(semantic_recovery, "complete_semantic_diagnostics", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index e1a86455e8..86af91738f 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -13,14 +13,13 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssembly, NativeSqlTestAssemblyRequest, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 886917a107..8c27d98cef 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,11 +7,10 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 9785765a12..51185b7896 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,11 +9,10 @@ import pytest +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 4447ccf2d0..abc6853e1d 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -425,10 +425,9 @@ def run_scheduler_build( ) -> tuple[Any, Any]: """Run a full build through the scheduler with custom materializations.""" + from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index e515555b70..4dd945f428 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,12 +10,11 @@ import duckdb +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 011b11dff5..059b3a7488 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,10 +13,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index ebac3b9955..0a032be40f 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,10 +8,9 @@ import pytest +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 1d20d66dff..f34f336cba 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,10 +5,8 @@ import pytest -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( build_dbt_combined_graph, @@ -84,7 +82,7 @@ def test_given_real_dbt_manifest_and_sqlbuild_project_when_building_graph_then_e manifest_source=manifest_source ), ) - project: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + project: CompiledProjectView = assemble_project(inputs=compile_inputs) manifest: DbtManifestIndex = build_dbt_manifest_index( raw_data=json.loads(manifest_source.read_text(encoding="utf-8")) ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 2d80ea5cec..27d3dc8921 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index 0a4772bf8f..7bbe835f06 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -5,12 +5,11 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -24,7 +23,7 @@ def compile_models( (project_dir / "models").mkdir(exist_ok=True) for name, sql in models: (project_dir / "models" / name).write_text(sql, encoding="utf-8") - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index f3901cd630..f31880b78a 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -5,11 +5,10 @@ import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine import custom_rules, fact_replay from sqlbuild.rule_engine._helpers.engine.fact_replay import ( fact_outcome_digest, @@ -63,7 +62,7 @@ def compile_files(*, root: Path, files: dict[str, str]) -> CompiledProjectView: path: Path = root / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(contents, encoding="utf-8") - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=root), adapter=DuckDbAdapter(), external_sql_reference_resolver=OrdersReferenceResolver(), diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index c007623560..4a091b1aab 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -291,7 +291,7 @@ def test_given_freshness_code_when_checking_sql_ownership_then_matches_contract( ), CustomRuleTestCase( description="compile input macro load passes", - path="src/sqlbuild/compiler/compile/main/_build_compile_inputs.py", + path="src/sqlbuild/compiler/compile/main/sql_expansion_context.py", source="loaded = load_project_macros(())\n", expected_fault_count=0, ), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py index c2b51307d9..5094b0e206 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py @@ -4,6 +4,8 @@ import pytest +import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.lineage.selection import ( normalize_lineage_target, select_column_target_lineage, @@ -16,7 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, - ProjectColumnLineage, QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -136,10 +137,12 @@ def test_given_column_target_when_selecting_then_returns_column_trace( received_model_names: list[frozenset[str]] = [] received_modes: list[ColumnLineageMode] = [] - def build_column_lineage_spy(*args: object, **kwargs: object) -> ProjectColumnLineage: - del args - received_model_names.append(cast(frozenset[str], kwargs["model_names"])) - received_modes.append(cast(ColumnLineageMode, kwargs["mode"])) + def build_column_lineage_spy( + project: object, dialect: object, mode: object, model_names: object + ) -> ProjectColumnLineage: + del project, dialect + received_model_names.append(cast(frozenset[str], model_names)) + received_modes.append(cast(ColumnLineageMode, mode)) return ProjectColumnLineage( models={}, edges=( @@ -170,10 +173,7 @@ def build_column_lineage_spy(*args: object, **kwargs: object) -> ProjectColumnLi ), ) - monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.lineage.selection.build_project_column_lineage", - build_column_lineage_spy, - ) + monkeypatch.setattr(native_module, "build_project_column_lineage", build_column_lineage_spy) result: ColumnLineageTrace | None = select_column_target_lineage( graph=graph, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py index d88a23fb5c..af05cab4c6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py @@ -6,10 +6,10 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.exceptions import SpecConfigError from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 1a73fdec19..16f9d7d814 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -9,6 +9,7 @@ import pytest +import sqlbuild._native as discovery_entry from sqlbuild.cli.commands._helpers.compile import lineage as compile_lineage from sqlbuild.cli.commands._helpers.compile import target_writer from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters @@ -17,7 +18,6 @@ from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.discovery.main import discover as discovery_entry from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.presentation.classes import transient_status_reporter diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index ac3fe4e037..bfc8b5a41f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, @@ -19,7 +20,6 @@ from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index b9d26f4f90..8382d5bb4b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os from dataclasses import replace from pathlib import Path @@ -8,6 +7,7 @@ import pytest +import sqlbuild._native as _native from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module from sqlbuild.cli.commands._helpers.compile.target_writer import ( @@ -26,6 +26,7 @@ ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -41,6 +42,7 @@ build_target_writer_plan_output, read_target_files, ) +from tests.unit.src.sqlbuild.compiler.manifest.helpers import build_test_project @pytest.mark.parametrize( @@ -75,7 +77,14 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( expected_test_sql: str = ( build_sql_test_comparison_sql(test_entry=plan_output.test_entries[0]) + "\n" ) - manifest: dict[str, object] = {"metadata": {"project_name": "demo"}} + manifest: _native.NativeManifest = build_manifest( + project=build_test_project(), + project_name="demo", + adapter_type="duckdb", + upstream_deps={}, + downstream_deps={}, + ) + manifest_contents: bytes = manifest.contents() expected_files: dict[str, str] = dict(test_case.expected_files) expected_files["compiled/tests/_chain_/orders__stg_orders/orders_chain.sql"] = expected_test_sql @@ -97,7 +106,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( ) assert read_target_files(tmp_path / "target", expected_files) == expected_files - assert json.loads(manifest_path.read_text()) == manifest + assert manifest_path.read_bytes() == manifest_contents assert manifest_path.stat().st_mtime_ns == unchanged_mtime_ns assert not (tmp_path / "target" / "run").exists() diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index 6ccf45f542..c6345f23a3 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -16,7 +16,7 @@ _FULL_COMPILE_MODULES: tuple[str, ...] = ( "sqlbuild.cli.commands._helpers.compile.output", - "sqlbuild.compiler.compile.main._assemble_project", + "sqlbuild.compiler.compile._helpers.native_stages.assembly", ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 7af83aa85b..5abc2b166c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -3,19 +3,18 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.model_config import ( build_native_model_config, native_model_config_session, native_validation_error, ) -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from sqlbuild.spec.contracts.models import ( DefaultsConfig, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index a094251b3a..e63daf8241 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,9 +1,11 @@ import itertools +import json from collections.abc import Callable from pathlib import Path from types import MappingProxyType from typing import cast +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -22,8 +24,6 @@ expand_sql_macros, load_project_macros, ) -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledDirectLogicSqlTestPayload, @@ -49,7 +49,6 @@ CompileInputsView, ModelInput, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredMacroFile, DiscoveredProjectInputs, @@ -60,6 +59,7 @@ ColumnTransformKind, InferredNullability, ) +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( build_native_declaration_contexts, ) @@ -483,3 +483,18 @@ def reanalysed_columns( profile=inference_profile or ExpressionInferenceProfile(), column_types_by_table=column_types_by_table, )[0] + + +def manifest_model_node(*, project: CompiledProjectView, name: str) -> dict[str, object]: + """Build the project's manifest and return one model node from its published JSON.""" + + manifest: dict[str, dict[str, dict[str, object]]] = json.loads( + build_manifest( + project=project, + project_name="demo", + adapter_type="duckdb", + upstream_deps={}, + downstream_deps={}, + ).contents() + ) + return manifest["nodes"][f"model.demo.{name}"] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 41e1f8d240..6f0a067c5e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -19,7 +18,6 @@ CompiledSeedView, CompileInputsView, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AssembleCompiledProjectEffectiveTargetTestCase, @@ -519,7 +517,7 @@ def test_given_compile_inputs_when_assembling_compiled_project_then_returns_expe adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(m.name for m in compiled.models) == test_case.expected_model_names assert tuple(m.deps for m in compiled.models) == test_case.expected_model_deps @@ -624,7 +622,7 @@ def test_given_named_generic_audit_instances_when_assembling_then_each_name_is_p run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(audit.name for audit in compiled.audits) == test_case.expected_names @@ -697,7 +695,7 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph ) seeds: dict[str, CompiledSeedView] = { - seed.name: seed for seed in assemble_compiled_project(inputs=compile_inputs).seeds + seed.name: seed for seed in assemble_project(inputs=compile_inputs).seeds } assert ( @@ -720,9 +718,7 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, selected_target="physical", ) - physical_seeds: tuple[CompiledSeedView, ...] = assemble_compiled_project( - inputs=physical_inputs - ).seeds + physical_seeds: tuple[CompiledSeedView, ...] = assemble_project(inputs=physical_inputs).seeds assert {(seed.destination.database, seed.destination.schema) for seed in physical_seeds} == { test_case.expected_literal_target_location } @@ -799,7 +795,7 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert compiled.effective_target_database == test_case.expected_effective_target_database assert compiled.effective_target_schema == test_case.expected_effective_target_schema @@ -861,7 +857,7 @@ def test_given_sql_hooks_when_assembling_then_applies_adapter_aware_validation( run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project( + compiled: CompiledProjectView = assemble_project( inputs=compile_inputs, inference_profile=ExpressionInferenceProfile( sql_analysis_dialect=test_case.dialect, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index b6f7ddbcba..4facad6a36 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -7,10 +7,9 @@ import pytest +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 40097e240e..9d6cdffbaf 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,11 +9,9 @@ import pytest +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSourceFile, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index ce03a1beb0..e5a19eb161 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -7,9 +7,8 @@ import pytest +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -17,7 +16,6 @@ ModelInput, ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index e4eec95179..e900e30506 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -7,10 +7,9 @@ import pytest +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) @@ -20,9 +19,7 @@ CompileInputsView, ) from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( build_model_version_identity_metadata_json, ) @@ -39,6 +36,7 @@ from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, compile_first_model, + manifest_model_node, ) _PROJECT_FILE: str = """ @@ -305,11 +303,7 @@ def test_given_inherited_model_schemas_when_compiling_then_attaches_resolved_con ], } assert evaluate_model_contracts(project=project, dialect="duckdb").diagnostics == () - manifest_node: dict[str, object] = build_model_node( - model=superset, - plan_entry=None, - project_name="demo", - ) + manifest_node: dict[str, object] = manifest_model_node(project=project, name="order_superset") assert manifest_node["description"] == "Canonical order schema" manifest_columns: object = manifest_node["columns"] assert isinstance(manifest_columns, dict) @@ -490,11 +484,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac "name_pattern": None, } ] - manifest_node: dict[str, object] = build_model_node( - model=model, - plan_entry=None, - project_name="demo", - ) + manifest_node: dict[str, object] = manifest_model_node(project=project, name=model.name) manifest_meta: dict[str, object] = cast(dict[str, object], manifest_node["meta"]) sqlbuild_meta: dict[str, object] = cast(dict[str, object], manifest_meta["sqlbuild"]) assert sqlbuild_meta["dynamic_columns"] == contract_identity["dynamic_columns"] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 359efd5874..6b0e9aed81 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import assemble_project from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -247,7 +247,7 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq ) -> None: write_repo_files(tmp_path, test_case.repo_files) compile_inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(test.name for test in compiled.sql_tests) == test_case.expected_names assert tuple(test.key.name for test in compiled.sql_tests) == test_case.expected_names @@ -289,7 +289,7 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq model_test_path.read_text(encoding="utf-8").replace('amount "2.4700"', 'amount "2.4800"'), encoding="utf-8", ) - changed: CompiledProjectView = assemble_compiled_project( + changed: CompiledProjectView = assemble_project( inputs=compile_project_inputs(project_dir=tmp_path) ) changed_fingerprints: tuple[str | None, ...] = tuple( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index ff1fe2a463..2df8110bb7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -7,13 +7,13 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index c33c1c63cb..8235239f3c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 0e6cc220e6..fe89344a7f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,12 +7,11 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompileInputsView, FunctionInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, PythonHookEntry, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index 1117887f2f..33024ae81d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 5226a56150..621b130840 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -8,16 +8,18 @@ import pytest -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_dag_json, + discover_project_inputs, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.dag.main.build import build_dag_json from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -236,7 +238,7 @@ def test_given_resource_namespace_defaults_when_building_dag_then_targets_retain discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) - project: CompiledProjectView = assemble_compiled_project(inputs=inputs) + project: CompiledProjectView = assemble_project(inputs=inputs) graph: ProjectGraph = project_graph_from_indexes( project=project, upstream_deps={}, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index 7b19f83687..d06ac4c3f9 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -10,20 +10,20 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.discovery._helpers.filesystem import aggregation -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +import sqlbuild._native as _native +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, + discover_model_files, discover_model_schema_files, + discover_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, + discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_schema_files from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, @@ -63,7 +63,7 @@ _PATTERNS: tuple[str, ...] = ("*", "*.sql", "*.py", "*.yml", "schema.yml", "_sqlbuild", "enums") _ISOLATED_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { "model_files": partial( - discover_native_model_files, + discover_model_files, extract_implicit_alias_columns=True, extract_output_column_locations=True, ), @@ -72,9 +72,9 @@ "model_schema_files": discover_model_schema_files, "sql_function_files": discover_sql_function_files, "sql_hook_files": discover_sql_hook_files, - "schema_files": discover_native_schema_files, + "schema_files": discover_schema_files, "seed_files": discover_seed_files, - "test_files": discover_native_test_files, + "test_files": discover_test_files, "audit_files": discover_audit_files, "macro_files": discover_macro_files, } @@ -168,7 +168,7 @@ def write_scoped_project(*, root: Path) -> None: def isolated_discovery_mismatches(*, project_dir: Path) -> tuple[str, ...]: """Return categories where one shared-snapshot pass differs from isolated discovery.""" - inputs: DiscoveredProjectInputs = aggregation.build_discovered_project_inputs( + inputs: DiscoveredProjectInputs = _native.build_discovered_project_inputs( project_dir=project_dir, project_config=load_project_config(project_dir=project_dir), local_config=load_local_config(project_dir=project_dir), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index dad7c3bfbc..03612eb17c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py index 9a02d85a88..44fe4ceeff 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_helpers import ( isolated_discovery_mismatches, model_paths, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py index 743ac9f319..c53b3b7445 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py @@ -7,23 +7,23 @@ import pytest +from sqlbuild._native import ( + discover_audit_files, + discover_constant_files, + discover_enum_files, + discover_scenario_files, + discover_source_files, + discover_sql_function_files, + discover_sql_hook_files, +) from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_declarations, ) from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_audit_files, - discover_constant_files, - discover_enum_files, discover_event_exporter_declarations, discover_materialization_files, discover_provider_classes, - discover_sql_function_files, - discover_sql_hook_files, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, ) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import ( EventExporterDiscoveryError, ProviderDiscoveryError, @@ -40,8 +40,8 @@ "audits": discover_audit_files, "constants": discover_constant_files, "enums": discover_enum_files, - "scenarios": discover_native_scenario_files, - "sources": discover_native_source_files, + "scenarios": discover_scenario_files, + "sources": discover_source_files, "sql_functions": discover_sql_function_files, "sql_hooks": discover_sql_hook_files, } diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py index f57ad57df5..b73f54fb5d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_macro_files +from sqlbuild._native import ( + discover_macro_files, +) from sqlbuild.compiler.discovery.models import DiscoveredMacroFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMacroFilesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index c329b98e23..703405a9ba 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -8,21 +8,21 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, - discover_hook_functions, discover_macro_files, - discover_python_function_files, + discover_project_inputs, + discover_scenario_files, + discover_source_files, discover_sql_function_files, discover_sql_hook_files, + discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, - discover_native_test_files, +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_hook_functions, + discover_python_function_files, ) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, @@ -43,7 +43,7 @@ declaration_contents, discover_declarations, ) -from tests.unit.src.sqlbuild.compiler.discovery.helpers import discover_model_files +from tests.unit.src.sqlbuild.compiler.discovery.helpers import discover_all_model_files class _DiscoveredFile(Protocol): @@ -385,13 +385,13 @@ def test_given_file_in_declaration_role_when_discovering_resources_then_returns_ file_path.parent.mkdir(parents=True) file_path.write_text(test_case.contents, encoding="utf-8") discoverers: dict[str, Callable[..., tuple[_DiscoveredFile, ...]]] = { - "model": discover_model_files, - "test": discover_native_test_files, - "scenario": discover_native_scenario_files, + "model": discover_all_model_files, + "test": discover_test_files, + "scenario": discover_scenario_files, "hook": discover_sql_hook_files, "function": discover_sql_function_files, "audit": discover_audit_files, - "source": discover_native_source_files, + "source": discover_source_files, "python_function": discover_python_function_files, "python_hook": discover_hook_functions, } @@ -424,7 +424,7 @@ def test_given_similarly_prefixed_directory_when_discovering_then_treats_it_as_o file_path.parent.mkdir(parents=True) file_path.write_text("MODEL ();\nSELECT 1\n", encoding="utf-8") - result: tuple[_DiscoveredFile, ...] = discover_model_files(project_dir=tmp_path) + result: tuple[_DiscoveredFile, ...] = discover_all_model_files(project_dir=tmp_path) assert tuple(file.relative_path.as_posix() for file in result) == ( test_case.expected_relative_paths diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index fcf38552a4..d427cd28b2 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py index a90d183b86..3ee4442bd7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_sql_hook_files +from sqlbuild._native import ( + discover_sql_hook_files, +) from sqlbuild.compiler.discovery.exceptions import SqlHookParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 173822217a..460ef2ab2f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -4,7 +4,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild._native import ( + discover_model_files, +) from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers from sqlbuild.compiler.discovery._helpers.sql.model_files import model_output_column_locations from sqlbuild.compiler.discovery.models import ( @@ -27,7 +29,7 @@ ParseModelSqlHeaderTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( - discover_model_files, + discover_all_model_files, model_header_column_locations, parse_model_sql, ) @@ -53,7 +55,7 @@ def test_given_unique_model_headers_when_discovering_then_header_values_and_loca f"MODEL ({second_header});\nSELECT 2 AS total\n", encoding="utf-8" ) - discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files(project_dir=tmp_path) + discovered: tuple[DiscoveredSqlModelFile, ...] = discover_all_model_files(project_dir=tmp_path) assert len(discovered) == test_case.expected_count assert [model.header_values for model in discovered] == [ @@ -126,7 +128,7 @@ def test_given_multiple_invalid_model_headers_when_discovering_then_fault_order_ (models_dir / "second.sql").write_text("MODEL (schema ${MISSING); SELECT 2", encoding="utf-8") faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -164,7 +166,7 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection encoding="utf-8", ) - model_file: DiscoveredSqlModelFile = discover_native_model_files( + model_file: DiscoveredSqlModelFile = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=(test_case.expected_extract_implicit_alias_columns), extract_output_column_locations=False, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 23b5edbcfc..3f28cc8e09 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files +from sqlbuild._native import ( + discover_test_files, +) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( omitted_ceremonial_select_offset, @@ -96,7 +98,7 @@ def test_given_invalid_test_file_when_discovering_tolerantly_then_reports_file_f (tests_dir / "b_valid.sql").write_text('TEST (name "valid_order");\nSELECT 1', encoding="utf-8") faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( + discovered: tuple[DiscoveredSqlTestFile, ...] = discover_test_files( project_dir=tmp_path, on_fault=faults.append, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index a753905b74..340f686c7b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,12 +6,12 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, ) from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import TargetWarehousesConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index bf0435e6b5..f6b19d46e7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,30 +1,16 @@ from pathlib import Path -from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_locations, - native_payload_error, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - parse_native_sql_test_contents, -) -from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes from sqlbuild.compiler.discovery.constants import ( REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, - SQL_SCENARIO_HEADER_KEYS, - SQL_TEST_HEADER_KEYS, ) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, ) -from sqlbuild.compiler.discovery.types import NativeLocation from sqlbuild.spec.contracts.models import ( SchemaModelEntry, SchemaSeedEntry, @@ -47,10 +33,10 @@ def write_lifecycle_event_sink_project(*, project_dir: Path, exporter_config: st ) -def discover_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, ...]: +def discover_all_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, ...]: """Discover every model file with the column extraction full compilation uses.""" - return discover_native_model_files( + return _native.discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -60,8 +46,8 @@ def discover_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: """Parse model contents as model discovery does, returning header values and query SQL.""" - payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=file_path) - return cast(dict[str, object], payload[2]), str(payload[5]) + header_values, query_sql, _ = _parsed_model(contents=contents, file_path=file_path) + return header_values, query_sql def model_header_column_locations( @@ -69,14 +55,13 @@ def model_header_column_locations( ) -> dict[str, SourceLocation]: """Return the authored MODEL(columns) locations model discovery records.""" - payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=relative_path) - return native_locations( - locations=cast(list[NativeLocation], payload[3]), relative_path=relative_path - ) + return _parsed_model(contents=contents, file_path=relative_path)[2] -def _parsed_model(*, contents: str, file_path: Path) -> tuple[object, ...]: - payload: tuple[object, ...] = _native.parse_model_contents( +def _parsed_model( + *, contents: str, file_path: Path +) -> tuple[dict[str, object], str, dict[str, SourceLocation]]: + return _native.parse_model_contents( { "file_path": str(file_path), "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), @@ -86,14 +71,12 @@ def _parsed_model(*, contents: str, file_path: Path) -> tuple[object, ...]: }, contents, ) - _raise_payload_error(payload=payload, file_path=file_path) - return payload def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: """Parse SQL test contents as test discovery does.""" - return parse_native_sql_test_contents(contents=contents, file_path=file_path) + return _native.parse_sql_test_file(contents=contents, file_path=file_path) def parse_sql_scenario_file( @@ -101,31 +84,18 @@ def parse_sql_scenario_file( ) -> DiscoveredSqlScenarioFile: """Parse scenario contents as scenario discovery does.""" - payload: tuple[object, ...] = _native.parse_scenario_contents( - { - "file_path": str(file_path), - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - }, - contents, - ) - _raise_payload_error(payload=payload, file_path=file_path) - return build_sql_scenario_file( - header_values=cast(dict[str, object], payload[2]), - sql_body=str(payload[3]), - contents=str(payload[1]), - file_path=file_path, - relative_path=relative_path, - sql_body_span=cast(tuple[int, int], payload[4]), + return _native.parse_sql_scenario_file( + contents=contents, file_path=file_path, relative_path=relative_path ) def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: """Load and parse sources YAML contents as source discovery does.""" - return parse_loaded_sources_yml( - loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="source"), - file_path=file_path, + return _native.parse_sources_document( + _native.load_yaml_document(str(file_path), contents, "source"), + str(file_path), + yaml_entry_classes(), ) @@ -134,18 +104,8 @@ def parse_schema_yml( ) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: """Load and parse schema or seed YAML contents as schema discovery does.""" - return parse_loaded_schema_yml( - loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="schema"), - file_path=file_path, + return (), _native.parse_schema_document( + _native.load_yaml_document(str(file_path), contents, "schema"), + str(file_path), + yaml_entry_classes(), ) - - -def _loaded_yaml(*, contents: str, file_path: Path, kind: str) -> object: - payload: tuple[object, ...] = _native.load_yaml_document(str(file_path), contents, kind) - _raise_payload_error(payload=payload, file_path=file_path) - return payload[2] - - -def _raise_payload_error(*, payload: tuple[object, ...], file_path: Path) -> None: - for error in filter(None, (native_payload_error(payload=payload, file_path=file_path),)): - raise error diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 71d18333b1..bf96a4ee0f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.exceptions import ProjectConfigError, ProjectPythonPathError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestBlock from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.compiler.discovery._test_helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 2206c7981f..b83ea020b9 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -36,8 +36,7 @@ class RichLineageAnalysisScalingTestCase: description: str chain_length: int unrelated_model_counts: tuple[int, ...] - expected_schema_builds: int - expected_analysis_calls: int + expected_model_names: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 14d2b573af..086dbddaff 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -2,6 +2,8 @@ from pathlib import Path +import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -18,6 +20,7 @@ CompiledSeedView, ) from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSeedFile +from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig @@ -122,3 +125,15 @@ def make_chain_models( for index in range(unrelated_model_count) ) return (root, *chain, *unrelated) + + +def project_column_lineage( + *, + project: CompiledProjectView, + dialect: str | None = None, + mode: ColumnLineageMode = ColumnLineageMode.RICH, + model_names: frozenset[str] | None = None, +) -> ProjectColumnLineage | None: + """The project's native column lineage graph, rich by default.""" + + return native.build_project_column_lineage(project, dialect, mode, model_names) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 6a2d2784f8..76def5dffc 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -1,11 +1,10 @@ from __future__ import annotations from dataclasses import replace -from typing import Any import pytest -import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -16,9 +15,7 @@ CompiledResourceType, ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.lineage._helpers import native_rich_columns -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage -from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage, ProjectColumnLineage +from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnLineageMode, @@ -37,6 +34,7 @@ make_compiled_model, make_compiled_project, make_compiled_seed, + project_column_lineage, ) @@ -177,7 +175,7 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte ), ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None model_lineage: ModelColumnLineage = result.models[test_case.model_name] @@ -309,7 +307,7 @@ def test_given_compiled_project_when_building_fast_column_lineage_then_infers_ex ), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, mode=ColumnLineageMode.FAST, ) @@ -366,7 +364,7 @@ def test_given_select_star_when_building_column_lineage_then_expands_known_schem ) ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None model_lineage: ModelColumnLineage = result.models[test_case.model_name] @@ -405,7 +403,7 @@ def test_given_linear_project_when_listing_column_consumers_then_returns_expecte ) ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None consumers: tuple[str, ...] = tuple( @@ -457,7 +455,7 @@ def test_given_compact_analysis_facts_when_building_fast_lineage_then_preserves_ ), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=make_compiled_project(models=(upstream, target)), mode=ColumnLineageMode.FAST, ) @@ -553,7 +551,7 @@ def test_given_cte_sourced_column_when_building_rich_column_lineage_then_resolve models=upstream_models + (target_model,), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, mode=ColumnLineageMode.RICH, ) @@ -612,7 +610,7 @@ def test_given_postgres_dialect_when_building_rich_column_lineage_then_uses_poly ) project: CompiledProjectView = make_compiled_project(models=upstream_models + (target_model,)) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, dialect="postgres", mode=ColumnLineageMode.RICH, @@ -660,7 +658,7 @@ def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_n sql_analysis_enabled=test_case.sql_analysis_enabled, ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert (result is None) is test_case.expected_result_is_none @@ -669,41 +667,22 @@ def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_n "test_case", [ RichLineageAnalysisScalingTestCase( - description="builds the schema once and analyses each traced model against its refs", + description="analyses only the traced models, whatever else the project holds", chain_length=4, unrelated_model_counts=(0, 250), - expected_schema_builds=1, - expected_analysis_calls=4, + expected_model_names=("orders_0", "orders_1", "orders_2", "orders_3"), ) ], ids=lambda case: case.description, ) def test_given_unrelated_models_when_tracing_rich_lineage_then_analysis_work_stays_linear( test_case: RichLineageAnalysisScalingTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: - build_lineage: Any = native.build_rich_column_lineage - schema_resources: Any = native_rich_columns._schema_resources - schema_builds: list[CompiledProjectView] = [] - analysed_queries: list[str] = [] - - def counting_schema_mapping(project: CompiledProjectView) -> Any: - schema_builds.append(project) - return schema_resources(project) - - def recording_native_lineage(request: Any) -> Any: - analysed_queries.extend(request[2]) - return build_lineage(request) - - monkeypatch.setattr(native_rich_columns, "_schema_resources", counting_schema_mapping) - monkeypatch.setattr(native, "build_rich_column_lineage", recording_native_lineage) chain_names: frozenset[str] = frozenset( f"orders_{index}" for index in range(test_case.chain_length) ) edges_by_project_size: list[tuple[str, ...]] = [] for unrelated_model_count in test_case.unrelated_model_counts: - schema_builds.clear() - analysed_queries.clear() project: CompiledProjectView = make_compiled_project( models=make_chain_models( chain_length=test_case.chain_length, @@ -711,13 +690,12 @@ def recording_native_lineage(request: Any) -> Any: ) ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, model_names=chain_names ) assert result is not None - assert len(schema_builds) == test_case.expected_schema_builds - assert len(analysed_queries) == test_case.expected_analysis_calls + assert tuple(result.models) == test_case.expected_model_names, test_case.description edges_by_project_size.append( tuple( edge_label( diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index a459ffea88..1974575b1c 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -138,3 +138,10 @@ class FqnTestCase: project_name: str relative_path: str expected_fqn: list[str] + + +@dataclass(frozen=True) +class ManifestEncodingErrorTestCase: + description: str + meta: dict[object, object] + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index b689ed3aae..9dfcd8b46e 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any, cast @@ -69,10 +70,9 @@ def source_key(name: str) -> CompiledObjectKey: def run_manifest(**kwargs: Any) -> dict[str, Any]: - """Build a manifest and return it with Any typing for test convenience.""" + """Build a manifest and return its parsed `manifest.json` contents.""" - result: dict[str, object] = build_manifest(**kwargs) - return result # type: ignore[return-value] + return json.loads(build_manifest(**kwargs).contents()) def manifest_nodes(result: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 485555ada0..7a549c4bce 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -1,11 +1,13 @@ from __future__ import annotations +import re from dataclasses import replace from pathlib import Path -from typing import Any +from typing import Any, cast import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( CompiledObjectKey, LoadedMacro, @@ -18,8 +20,7 @@ ) from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry -from sqlbuild.compiler.manifest._helpers.shared import build_fqn -from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, @@ -33,6 +34,7 @@ from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, + ManifestEncodingErrorTestCase, ManifestMacroNodeTestCase, ManifestModelNodeTestCase, ManifestParentMapTestCase, @@ -61,6 +63,7 @@ _PROJECT: str = "demo" _ADAPTER: str = "duckdb" +_DBT_MANIFEST_SCHEMA_VERSION: str = "https://schemas.getdbt.com/dbt/manifest/v12.json" _MODEL_ORDERS: CompiledModelView = build_test_model( name="orders", @@ -158,7 +161,7 @@ expected_macro_count=0, expected_metadata_project_name=_PROJECT, expected_metadata_adapter_type=_ADAPTER, - expected_metadata_schema_version=DBT_MANIFEST_SCHEMA_VERSION, + expected_metadata_schema_version=_DBT_MANIFEST_SCHEMA_VERSION, ), ManifestTopLevelTestCase( description="produces correct counts for multi-resource project", @@ -179,7 +182,7 @@ expected_macro_count=0, expected_metadata_project_name=_PROJECT, expected_metadata_adapter_type=_ADAPTER, - expected_metadata_schema_version=DBT_MANIFEST_SCHEMA_VERSION, + expected_metadata_schema_version=_DBT_MANIFEST_SCHEMA_VERSION, ), ], ids=lambda case: case.description, @@ -608,15 +611,23 @@ def test_given_seed_when_building_manifest_then_produces_correct_node( ], ids=lambda case: case.description, ) -def test_given_relative_path_when_building_fqn_then_returns_expected_parts( +def test_given_relative_path_when_building_manifest_then_fqn_has_expected_parts( test_case: FqnTestCase, ) -> None: - result: list[str] = build_fqn( + model: CompiledModelView = build_test_model( + name="orders", relative_path=test_case.relative_path + ) + + result: dict[str, Any] = run_manifest( + project=build_test_project(models=(model,)), project_name=test_case.project_name, - relative_path=Path(test_case.relative_path), + adapter_type=_ADAPTER, + upstream_deps={}, + downstream_deps={}, ) - assert result == test_case.expected_fqn + node: dict[str, Any] = manifest_nodes(result)[f"model.{test_case.project_name}.orders"] + assert node["fqn"] == test_case.expected_fqn @pytest.mark.parametrize( @@ -1038,3 +1049,37 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc "sqlbuild_resource_type": "sql_hook", "sqlbuild_hook_name": "record_access", } + + +@pytest.mark.parametrize( + "test_case", + [ + ManifestEncodingErrorTestCase( + description="unencodable meta value raises when the manifest is written", + meta={"owners": {"orders"}}, + expected_message="Object of type set is not JSON serializable", + ), + ManifestEncodingErrorTestCase( + description="unencodable meta key raises when the manifest is written", + meta={("orders", 1): "owner"}, + expected_message="keys must be str, int, float, bool or None, not tuple", + ), + ], + ids=lambda case: case.description, +) +def test_given_unencodable_meta_when_writing_manifest_then_raises_json_type_error( + test_case: ManifestEncodingErrorTestCase, +) -> None: + model: CompiledModelView = build_test_model( + name="orders", meta=cast(dict[str, object], test_case.meta) + ) + manifest: _native.NativeManifest = build_manifest( + project=build_test_project(models=(model,)), + project_name=_PROJECT, + adapter_type=_ADAPTER, + upstream_deps={}, + downstream_deps={}, + ) + + with pytest.raises(TypeError, match=re.escape(test_case.expected_message)): + manifest.contents() diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 3629e4b28c..68bdc913c3 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -2,19 +2,23 @@ from __future__ import annotations +import time from unittest.mock import Mock import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main import _compile_phase as compile_phase_module -from sqlbuild.compiler.pipeline.main import compile as compile_module +from sqlbuild.compiler.pipeline._helpers import analysis_selection +from sqlbuild.compiler.pipeline.main import _plan_result as plan_result from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings +from sqlbuild.rule_engine.main import load_config, run_rules from sqlbuild.rule_engine.models import RulesConfig, RulesRunResult +from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -59,20 +63,19 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di project=project, plan_output=PlanOutput() ) adapter: Mock = Mock() - monkeypatch.setattr(compile_module.time, "monotonic", monotonic) - monkeypatch.setattr(compile_phase_module.time, "monotonic", monotonic) - monkeypatch.setattr(compile_phase_module, "build_compiled_project", Mock(return_value=project)) + monkeypatch.setattr(time, "monotonic", monotonic) + monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) monkeypatch.setattr( - compile_phase_module, "resolve_compile_analysis_selection", Mock(return_value=None) + analysis_selection, "resolve_compile_analysis_selection", Mock(return_value=None) ) - monkeypatch.setattr(compile_module, "open_connection_with_hooks", Mock(return_value=object())) - monkeypatch.setattr(compile_module, "_build_result", Mock(return_value=pipeline_result)) - monkeypatch.setattr(compile_module, "load_rules_config", Mock(return_value=RulesConfig())) + monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) + monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) + monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) monkeypatch.setattr( - compile_module, "resolve_configured_rule_selection", Mock(return_value=None) + analysis_selection, "resolve_configured_rule_selection", Mock(return_value=None) ) monkeypatch.setattr( - compile_module, + run_rules, "run_rules", Mock( return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) @@ -86,7 +89,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di } with tracker.scope(): - result: CompilePipelineResult = compile_module.run_compile_pipeline( + result: CompilePipelineResult = _native.run_compile_pipeline( discovered_inputs=Mock(), adapter=adapter, options=CompilePipelineOptions(connection_config={}), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 00f0fa9c35..5ccf38ab2f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -10,12 +10,11 @@ import duckdb import pytest +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( _list_source_candidates, get_semantic_source_columns, @@ -59,7 +58,7 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) listings: Mock = Mock(wraps=adapter.list_relations) @@ -116,7 +115,7 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) with duckdb.connect() as connection: diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index 9303c99533..64026852d6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -24,15 +24,6 @@ class SkipQuotedTextErrorTestCase: expected_error: str -@dataclass(frozen=True) -class IterCodePositionsTestCase: - """Code-position scan case.""" - - description: str - sql: str - expected_positions: tuple[tuple[int, int], ...] - - @dataclass(frozen=True) class DialectQuotedTextSuccessTestCase: """Successful dialect-aware quoted-text scan case.""" @@ -89,6 +80,6 @@ class DialectMatchingParenTestCase: """Dialect-aware parenthesis matching case.""" description: str - syntax: SqlLexicalSyntax | None + native_syntax: dict[str, object] | None sql: str expected_index: int diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py new file mode 100644 index 0000000000..79b8981a2d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py @@ -0,0 +1,18 @@ +"""Native SQL code scanners for the scanning tests.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + + +def generic_scanner(*, sql: str, context: str = "SQL") -> _native.SqlCodeScanner: + """A native scanner over `sql` under the generic compiler rules.""" + + return _native.SqlCodeScanner(sql, None, context) + + +def dialect_scanner(*, sql: str, syntax: SqlLexicalSyntax) -> _native.SqlCodeScanner: + """A native scanner over `sql` under one dialect's lexical rules.""" + + return _native.SqlCodeScanner(sql, syntax.native_mapping, "SQL") diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index 2d45c8472c..850e3434e6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -2,14 +2,8 @@ import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis._helpers.scanning import ( - dialect_non_code_end_impl, - dialect_quoted_text_end_impl, - find_matching_paren_impl, - iter_code_positions_impl, - skip_quoted_text_impl, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( DialectMatchingParenTestCase, @@ -17,11 +11,14 @@ DialectNonCodeSuccessTestCase, DialectQuotedTextErrorTestCase, DialectQuotedTextSuccessTestCase, - IterCodePositionsTestCase, RequiresDialectScanTestCase, SkipQuotedTextErrorTestCase, SkipQuotedTextSuccessTestCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers.helpers import ( + dialect_scanner, + generic_scanner, +) @pytest.mark.parametrize( @@ -75,7 +72,7 @@ def test_given_quoted_text_when_skipping_then_returns_end_position( ) -> None: sql: str = f"{test_case.quoted_sql} suffix" - result: int = skip_quoted_text_impl(sql=sql, start=0) + result: int | None = generic_scanner(sql=sql).non_code_end(0) assert result == test_case.expected_end @@ -102,7 +99,7 @@ def test_given_unclosed_quote_when_skipping_then_raises_contextual_error( test_case: SkipQuotedTextErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - skip_quoted_text_impl(sql=test_case.sql, start=0, context=test_case.context) + generic_scanner(sql=test_case.sql, context=test_case.context).non_code_end(0) ANSI_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() @@ -197,8 +194,8 @@ def test_given_unclosed_quote_when_skipping_then_raises_contextual_error( def test_given_dialect_quoted_text_when_scanning_then_returns_end_position( test_case: DialectQuotedTextSuccessTestCase, ) -> None: - result: int | None = dialect_quoted_text_end_impl( - sql=test_case.sql, start=test_case.start, syntax=test_case.syntax + result: int | None = dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end( + test_case.start ) assert result == test_case.expected_end @@ -226,7 +223,7 @@ def test_given_unclosed_dialect_quoted_text_when_scanning_then_raises( test_case: DialectQuotedTextErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - dialect_quoted_text_end_impl(sql=test_case.sql, start=0, syntax=test_case.syntax) + dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) NESTED_COMMENT_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax(nested_block_comments=True) @@ -295,9 +292,7 @@ def test_given_unclosed_dialect_quoted_text_when_scanning_then_raises( def test_given_dialect_sql_when_scanning_non_code_then_returns_end_position( test_case: DialectNonCodeSuccessTestCase, ) -> None: - result: int | None = dialect_non_code_end_impl( - sql=test_case.sql, start=0, syntax=test_case.syntax - ) + result: int | None = dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) assert result == test_case.expected_end @@ -324,44 +319,7 @@ def test_given_unclosed_dialect_comment_when_scanning_then_raises( test_case: DialectNonCodeErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - dialect_non_code_end_impl(sql=test_case.sql, start=0, syntax=test_case.syntax) - - -@pytest.mark.parametrize( - "test_case", - ( - IterCodePositionsTestCase( - description="parentheses change depth and are not yielded", - sql="a(b)c", - expected_positions=((0, 0), (2, 1), (4, 0)), - ), - IterCodePositionsTestCase( - description="quotes and comments are skipped", - sql="a'(b'`c`--d\n/*e*/f", - expected_positions=((0, 0), (17, 0)), - ), - IterCodePositionsTestCase( - description="dollar-quoted text is skipped", - sql="a $$'(--$$b", - expected_positions=((0, 0), (1, 0), (10, 0)), - ), - IterCodePositionsTestCase( - description="dollars inside identifiers and parameters remain code", - sql="a$$b $1", - expected_positions=tuple((index, 0) for index in range(7)), - ), - IterCodePositionsTestCase( - description="unbalanced close parenthesis goes below zero depth", - sql=")a", - expected_positions=((1, -1),), - ), - ), - ids=lambda case: case.description, -) -def test_given_sql_when_iterating_code_positions_then_yields_code_offsets_with_depth( - test_case: IterCodePositionsTestCase, -) -> None: - assert tuple(iter_code_positions_impl(sql=test_case.sql)) == test_case.expected_positions + dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) @pytest.mark.parametrize( @@ -431,19 +389,19 @@ def test_given_sql_when_checking_dialect_sensitivity_then_reports_differences( ( DialectMatchingParenTestCase( description="backslash-escaped quote hides a parenthesis", - syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + native_syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX.native_mapping, sql="('O\\') ' AS name) tail", expected_index=len("('O\\') ' AS name"), ), DialectMatchingParenTestCase( description="hash comment hides a parenthesis", - syntax=HASH_COMMENT_SYNTAX, + native_syntax=HASH_COMMENT_SYNTAX.native_mapping, sql="(1 # )\n) tail", expected_index=len("(1 # )\n"), ), DialectMatchingParenTestCase( description="generic scanning keeps the backslash literal", - syntax=None, + native_syntax=None, sql="('C:\\') tail", expected_index=len("('C:\\'"), ), @@ -454,6 +412,6 @@ def test_given_dialect_sql_when_matching_parenthesis_then_follows_dialect_rules( test_case: DialectMatchingParenTestCase, ) -> None: assert ( - find_matching_paren_impl(sql=test_case.sql, open_paren_index=0, syntax=test_case.syntax) + _native.SqlCodeScanner(test_case.sql, test_case.native_syntax, "SQL").matching_paren(0) == test_case.expected_index ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index efd7dcee2a..e5f22cb9dc 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -1,7 +1,15 @@ """Helpers for SQL analysis public-entry tests.""" +from typing import cast + +import sqlbuild._native as _native from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog -from sqlbuild.compiler.sql_analysis.types import NativeProjectCatalog +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult +from sqlbuild.compiler.sql_analysis.types import ( + NativeDiagnosticRow, + NativePositionsModule, + NativeProjectCatalog, +) def nested_coalesce_sql(*, function_depth: int) -> str: @@ -27,3 +35,14 @@ def without_catalog(dialect: str) -> None: """Return no catalog, which normalizes batches in turn.""" _ = dialect + + +def mapped_binding_result( + *, sql: str, dialect: str, rows: list[NativeDiagnosticRow] +) -> SqlBindingResult: + """Native polyglot diagnostic rows mapped to SQLBuild binding diagnostics for `sql`.""" + + mapped: list[NativeDiagnosticRow] = cast(NativePositionsModule, _native).binding_diagnostics( + sql=sql, dialect=dialect, rows=rows + ) + return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in mapped)) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py index 528eb27bdc..decef34399 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py @@ -7,12 +7,14 @@ import pytest -import sqlbuild._native as native -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest +from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( SemanticDiagnosticMappingCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import mapped_binding_result + +DIALECTS: tuple[str, ...] = ("duckdb", "snowflake", "postgresql", "bigquery") @pytest.mark.parametrize( @@ -48,7 +50,6 @@ ids=lambda case: case.description, ) def test_given_native_semantic_errors_when_validating_then_codes_and_locations_survive( - monkeypatch: pytest.MonkeyPatch, test_case: SemanticDiagnosticMappingCase, ) -> None: codes: dict[str, str] = {"E202": "B101", "E203": "B102", "E222": "B004", "E223": "B005"} @@ -56,30 +57,37 @@ def test_given_native_semantic_errors_when_validating_then_codes_and_locations_s {f"E{number}": f"B{number}" for number in (*range(210, 220), 230, 231, 232, 233, 234)} ) codes.update({"W210": "B217", "W213": "B218"}) - - def validate(payload: str) -> str: - requests: list[dict[str, Any]] = json.loads(payload) - assert all(request["options"]["semantic"] for request in requests) - assert all(request["options"]["check_types"] for request in requests) - errors: list[dict[str, object]] = [ - { - "code": code, - "severity": {True: "warning", False: "error"}[code.startswith("W")], - "message": "Invalid expression", - "line": 2, - "column": 8, - } - for code in codes - ] - return json.dumps([{"errors": errors}] * len(requests)) - - monkeypatch.setattr(native, "validate_sql_with_schemas_json", validate) - results: tuple[SqlBindingResult, ...] = get_schema_validations( - requests=tuple( - SqlSchemaValidationRequest(sql="SELECT\n invalid", dialect=dialect, schema={}) - for dialect in ("duckdb", "snowflake", "postgres", "bigquery") + rows: list[tuple[str, str, int | None, int | None, int | None, int | None, str]] = [ + ( + code, + "Invalid expression", + 2, + 8, + None, + None, + {True: "warning", False: "error"}[code.startswith("W")], ) - ) + for code in codes + ] + payloads: list[dict[str, Any]] = [ + json.loads( + create_binding_catalog( + dialect=dialect, + quoted_ignore_case=False, + known_functions=(), + known_types=(), + relations={}, + ).native.validation_payload([("SELECT\n invalid", [], {})]) + )[0] + for dialect in DIALECTS + ] + results: list[SqlBindingResult] = [ + mapped_binding_result(sql="SELECT\n invalid", dialect=dialect, rows=rows) + for dialect in DIALECTS + ] + + assert all(payload["options"]["semantic"] for payload in payloads) + assert all(payload["options"]["check_types"] for payload in payloads) for result in results: assert ( tuple(diagnostic.code for diagnostic in result.diagnostics) == test_case.expected_codes diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 490313af2a..4febaed46a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild._native import assemble_project from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -440,7 +440,7 @@ def build_compiled_project_with_model_specs( references=extract_sql_references(sql=sql, syntax=_GENERIC_SQL_SYNTAX), ) ) - return assemble_compiled_project( + return assemble_project( inputs=CompileProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index c9f239e882..09c7298277 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint._helpers.project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 6c42dc005f..5577218e17 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -10,8 +10,8 @@ import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py index 7981434951..e2e0395869 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py @@ -19,7 +19,7 @@ imported_module="sqlbuild.rules", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.pipeline.main.project", + "sqlbuild.compiler.discovery._helpers.filesystem.core", "sqlbuild.adapters.duckdb.classes.duckdb_adapter", ), expected_loaded_modules=(), @@ -29,7 +29,7 @@ imported_module="sqlbuild.rule_engine._helpers.host.custom_host", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.pipeline.main.project", + "sqlbuild.compiler.discovery._helpers.filesystem.core", ), expected_loaded_modules=(), ), diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index d67cdd7500..145414cc0e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -4,10 +4,9 @@ import pytest -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.rule_engine.main.load_config import load_rules_config