From 09c5e610d94f53cc9ee202d6cb117045bd83ba42 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:23:22 +0100 Subject: [PATCH 01/26] refactor: build the DAG artifact natively Build the static DAG JSON in sqlbuild-analysis from plain inputs the binding reads off the compiled project views and the discovered Python node graph, reproducing dataclasses.asdict, the None and empty-field pruning and json.dumps(indent=2) byte for byte. The compile driver and sqb dag call it directly; the Python artifact builder is removed. --- .../src/dag_artifact/_helpers/checks.rs | 278 ++++++++ .../src/dag_artifact/_helpers/edges.rs | 72 +++ .../src/dag_artifact/_helpers/mod.rs | 4 + .../src/dag_artifact/_helpers/nodes.rs | 435 +++++++++++++ .../src/dag_artifact/_helpers/values.rs | 117 ++++ .../src/dag_artifact/constants.rs | 29 + .../src/dag_artifact/errors.rs | 16 + .../src/dag_artifact/main/build_dag_json.rs | 64 ++ .../src/dag_artifact/main/mod.rs | 1 + .../sqlbuild-analysis/src/dag_artifact/mod.rs | 9 + .../src/dag_artifact/models.rs | 236 +++++++ .../src/dag_artifact/tests.rs | 6 + .../src/dag_artifact/tests/helpers.rs | 97 +++ .../dag_artifact/tests/test_build_dag_json.rs | 173 +++++ .../src/dag_artifact/tests/test_types.rs | 11 + .../src/dag_artifact/types.rs | 7 + crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/compile_driver/phases.rs | 24 +- .../_helpers/compile_outputs/json_report.rs | 2 +- .../_helpers/dag_artifact/artifact.rs | 57 ++ .../src/bindings/_helpers/dag_artifact/mod.rs | 4 + .../_helpers/dag_artifact/registration.rs | 9 + .../bindings/_helpers/dag_artifact/rows.rs | 456 +++++++++++++ .../bindings/_helpers/dag_artifact/values.rs | 135 ++++ .../bindings/_helpers/graph/project_graph.rs | 5 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 6 +- fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 4 + .../cli/commands/main/project/_dag.py | 7 +- .../compiler/dag/_helpers/__init__.py | 1 - .../compiler/dag/_helpers/artifact.py | 610 ------------------ src/sqlbuild/compiler/dag/constants.py | 3 - src/sqlbuild/compiler/dag/main/build.py | 21 - src/sqlbuild/compiler/dag/models.py | 114 ---- .../main/test_compile_startup_imports.py | 2 - .../sqlbuild/compiler/dag/main/test_build.py | 2 +- 38 files changed, 2253 insertions(+), 773 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/constants.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/errors.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/mod.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/models.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/tests.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs delete mode 100644 src/sqlbuild/compiler/dag/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/dag/_helpers/artifact.py delete mode 100644 src/sqlbuild/compiler/dag/constants.py delete mode 100644 src/sqlbuild/compiler/dag/main/build.py delete mode 100644 src/sqlbuild/compiler/dag/models.py diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs new file mode 100644 index 0000000000..3be61a3c2f --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs @@ -0,0 +1,278 @@ +//! `DagCheck` values of SQL tests, audits, scenarios and Python checks, sorted. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::edges::python_kinds; +use crate::dag_artifact::_helpers::nodes::{node_id, python_node_id}; +use crate::dag_artifact::_helpers::values::{integer, object, optional_text, text, texts}; +use crate::dag_artifact::constants::{ + AUDIT_KIND, CHECK_KIND, MODEL_KIND, PYTHON_CHECK_KIND, SCENARIO_KIND, SQL_TEST_KIND, +}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{ + DagAuditInput, DagProjectInput, DagPythonGraphInput, DagPythonNodeInput, DagScenarioInput, + DagSqlTestInput, DagValue, +}; + +/// Every `DagCheck` field, `None` or empty where a check kind leaves it out. +#[derive(Default)] +struct CheckFields { + id: String, + kind: String, + name: String, + checked_asset_ids: Vec, + path: Option, + description: Option, + severity: Option, + tags: Vec, + group: Option, + meta: Option, + attachment_kind: Option, + attached_target_name: Option, + attached_column_name: Option, + mode: Option, + assertion_names: Vec, + expected_model_names: Vec, + graph_asset_ids: Vec, + fixture_refs: Vec, +} + +fn strings(values: &[String]) -> DagValue { + texts(values.iter().map(String::as_str)) +} + +impl CheckFields { + fn value(self) -> DagValue { + object(vec![ + ("id", text(&self.id)), + ("kind", text(&self.kind)), + ("name", text(&self.name)), + ("checked_asset_ids", strings(&self.checked_asset_ids)), + ("path", optional_text(self.path.as_deref())), + ("description", optional_text(self.description.as_deref())), + ("severity", optional_text(self.severity.as_deref())), + ("tags", strings(&self.tags)), + ("group", optional_text(self.group.as_deref())), + ("meta", self.meta.unwrap_or(DagValue::Object(Vec::new()))), + ( + "attachment_kind", + optional_text(self.attachment_kind.as_deref()), + ), + ( + "attached_target_name", + optional_text(self.attached_target_name.as_deref()), + ), + ( + "attached_column_name", + optional_text(self.attached_column_name.as_deref()), + ), + ("mode", optional_text(self.mode.as_deref())), + ("assertion_names", strings(&self.assertion_names)), + ("expected_model_names", strings(&self.expected_model_names)), + ("graph_asset_ids", strings(&self.graph_asset_ids)), + ("fixture_refs", strings(&self.fixture_refs)), + ]) + } +} + +fn optional_integer(value: Option) -> DagValue { + value.map_or(DagValue::Null, integer) +} + +fn sql_test_meta(test: &DagSqlTestInput) -> Result { + let Some(case) = &test.case else { + return Ok(DagValue::Object(Vec::new())); + }; + let types: HashMap<&str, &str> = case + .parameter_schema + .iter() + .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) + .collect(); + let parameters: Vec = case + .parameters + .iter() + .map(|(name, value)| { + let value_type: &str = types + .get(name.as_str()) + .ok_or_else(|| DagArtifactError::MissingParameterType(name.clone()))?; + Ok(object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("value", value.clone()), + ])) + }) + .collect::>()?; + Ok(object(vec![ + ("source_path", text(&test.source_path)), + ("block_index", optional_integer(case.block_index)), + ("parent_name", optional_text(case.parent_name.as_deref())), + ("case_name", text(&case.case_name)), + ("case_index", optional_integer(case.case_index)), + ( + "case_fingerprint", + optional_text(case.case_fingerprint.as_deref()), + ), + ( + "parameter_schema", + DagValue::Array( + case.parameter_schema + .iter() + .map(|(name, value_type, nullable)| { + object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("nullable", DagValue::Bool(*nullable)), + ]) + }) + .collect(), + ), + ), + ("parameters", DagValue::Array(parameters)), + ])) +} + +fn sql_test_check(test: &DagSqlTestInput) -> Result<(String, String, DagValue), DagArtifactError> { + let id: String = match &test.case { + None => node_id(&test.key), + Some(case) => { + let block_index: String = case + .block_index + .map_or_else(|| String::from("None"), |index| index.to_string()); + format!( + "{SQL_TEST_KIND}:{}:{block_index}:{}", + test.source_path, case.case_name + ) + } + }; + let value: DagValue = CheckFields { + id, + kind: SQL_TEST_KIND.to_owned(), + name: test.name.clone(), + checked_asset_ids: test.scope_deps.iter().map(node_id).collect(), + path: Some(test.path.clone()), + mode: Some(test.mode.clone()), + meta: Some(sql_test_meta(test)?), + ..CheckFields::default() + } + .value(); + Ok((SQL_TEST_KIND.to_owned(), test.name.clone(), value)) +} + +fn audit_check(audit: &DagAuditInput) -> (String, String, DagValue) { + let checked_asset_ids: Vec = + match (&audit.attached_target_kind, &audit.attached_target_name) { + (Some(kind), Some(name)) => vec![node_id(&(kind.clone(), name.clone()))], + _ => audit.scope_deps.iter().map(node_id).collect(), + }; + let id: String = [ + Some(node_id(&audit.key)), + audit.attached_target_kind.clone(), + audit.attached_target_name.clone(), + audit.attached_column_name.clone(), + ] + .into_iter() + .flatten() + .filter(|part| !part.is_empty()) + .collect::>() + .join(":"); + let value: DagValue = CheckFields { + id, + kind: AUDIT_KIND.to_owned(), + name: audit.name.clone(), + checked_asset_ids, + path: Some(audit.path.clone()), + severity: audit.severity.clone(), + attachment_kind: audit.attached_target_kind.clone(), + attached_target_name: audit.attached_target_name.clone(), + attached_column_name: audit.attached_column_name.clone(), + ..CheckFields::default() + } + .value(); + (AUDIT_KIND.to_owned(), audit.name.clone(), value) +} + +fn model_id(name: &str) -> String { + node_id(&(MODEL_KIND.to_owned(), name.to_owned())) +} + +fn scenario_check(scenario: &DagScenarioInput) -> (String, String, DagValue) { + let value: DagValue = CheckFields { + id: node_id(&scenario.key), + kind: SCENARIO_KIND.to_owned(), + name: scenario.name.clone(), + checked_asset_ids: scenario + .expected_model_names + .iter() + .map(|name| model_id(name)) + .collect(), + path: Some(scenario.path.clone()), + assertion_names: scenario.assertion_names.clone(), + expected_model_names: scenario.expected_model_names.clone(), + graph_asset_ids: scenario + .authored_cte_names + .iter() + .map(|name| model_id(name)) + .collect(), + fixture_refs: scenario.fixture_names.concat(), + ..CheckFields::default() + } + .value(); + (SCENARIO_KIND.to_owned(), scenario.name.clone(), value) +} + +fn python_check( + node: &DagPythonNodeInput, + python: &DagPythonGraphInput, + kinds: &HashMap<&str, &str>, +) -> (String, String, DagValue) { + let mut checked_asset_ids: Vec = python + .dependency_edges + .iter() + .filter(|(_, downstream)| *downstream == node.name) + .map(|(upstream, _)| python_node_id(kinds[upstream.as_str()], upstream)) + .collect(); + checked_asset_ids.extend(node.sql_deps.iter().map(node_id)); + let value: DagValue = CheckFields { + id: python_node_id(&node.kind, &node.name), + kind: PYTHON_CHECK_KIND.to_owned(), + name: node.name.clone(), + checked_asset_ids, + path: Some(node.path.clone()), + description: node.description.clone(), + severity: node.check_severity.clone(), + tags: node.tags.clone(), + group: node.group.clone(), + meta: Some(node.meta.clone()), + ..CheckFields::default() + } + .value(); + (PYTHON_CHECK_KIND.to_owned(), node.name.clone(), value) +} + +/// Every check, sorted by `(kind, name)`. +pub(crate) fn checks( + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> Result { + let mut checks: Vec<(String, String, DagValue)> = project + .sql_tests + .iter() + .map(sql_test_check) + .collect::>()?; + checks.extend(project.audits.iter().map(audit_check)); + checks.extend(project.scenarios.iter().map(scenario_check)); + if let Some(python) = python { + let kinds: HashMap<&str, &str> = python_kinds(python); + checks.extend( + python + .nodes + .iter() + .filter(|node| node.kind == CHECK_KIND) + .map(|node| python_check(node, python, &kinds)), + ); + } + checks.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + Ok(DagValue::Array( + checks.into_iter().map(|(_, _, value)| value).collect(), + )) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs new file mode 100644 index 0000000000..1c73520b42 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs @@ -0,0 +1,72 @@ +//! `DagEdge` values: lineage, loader dependency and Python node edges, sorted. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::nodes::{ + loader_or_source_id, node_id, python_node_id, sources_by_loader, +}; +use crate::dag_artifact::_helpers::values::{object, text}; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::graph::models::ProjectGraph; + +/// Each Python node name's kind, as `nodes_by_name` maps it. +pub(crate) fn python_kinds(python: &DagPythonGraphInput) -> HashMap<&str, &str> { + python + .kinds_by_name + .iter() + .map(|(name, kind)| (name.as_str(), kind.as_str())) + .collect() +} + +fn python_edges(python: &DagPythonGraphInput) -> Vec<(String, String)> { + let kinds: HashMap<&str, &str> = python_kinds(python); + let mut edges: Vec<(String, String)> = python + .dependency_edges + .iter() + .map(|(upstream, downstream)| { + ( + python_node_id(kinds[upstream.as_str()], upstream), + python_node_id(kinds[downstream.as_str()], downstream), + ) + }) + .collect(); + for node in &python.nodes { + for dependency in &node.sql_deps { + edges.push((node_id(dependency), python_node_id(&node.kind, &node.name))); + } + } + edges +} + +/// Every edge, sorted by `(from_id, to_id)`. +pub(crate) fn edges( + graph: &ProjectGraph, + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> DagValue { + let mut edges: Vec<(String, String)> = Vec::new(); + for (key, deps) in &graph.upstream { + for dep in deps { + edges.push((node_id(dep), node_id(key))); + } + } + let sources: HashMap<&str, &str> = sources_by_loader(project); + for loader in &project.loaders { + for dependency in &loader.dependency_names { + edges.push(( + loader_or_source_id(dependency, &sources), + loader_or_source_id(&loader.name, &sources), + )); + } + } + if let Some(python) = python { + edges.extend(python_edges(python)); + } + edges.sort(); + DagValue::Array( + edges + .iter() + .map(|(from, to)| object(vec![("from_id", text(from)), ("to_id", text(to))])) + .collect(), + ) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs new file mode 100644 index 0000000000..7dec7aa968 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod checks; +pub(crate) mod edges; +pub(crate) mod nodes; +pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs new file mode 100644 index 0000000000..312b15f3fd --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs @@ -0,0 +1,435 @@ +//! `DagNode` values of sources, loaders, seeds, functions, models and Python nodes. + +use sqlbuild_core::text::main::python_strip::python_strip; +use std::collections::HashMap; + +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::_helpers::values::{ + column, object, optional_text, text, texts, typed_column, +}; +use crate::dag_artifact::constants::{ + ASSET_KIND, DEFAULT_MATERIALIZATION_PYTHON_ASSET, LOADER_DESTINATION_PREFIX, LOADER_KIND, + LOADER_MAX_PARTS, LOADER_NODE_META_KEY, MODEL_KIND, SCALAR_RETURN_KIND, SEED_KIND, SOURCE_KIND, + TABLE_FUNCTION_KIND, TABLE_RETURN_KIND, UDF_KIND, +}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{ + DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, DagProjectInput, + DagPythonNodeInput, DagRelationInput, DagSeedInput, DagSourceEntryInput, DagSourceInput, + DagValue, +}; +use crate::dag_artifact::types::DagColumnLineage; + +/// Every `DagNode` field, `None` or empty where a node kind leaves it out. +#[derive(Default)] +pub(crate) struct NodeFields { + pub(crate) id: String, + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) asset_key: Vec, + pub(crate) target: Option, + pub(crate) path: Option, + pub(crate) description: Option, + pub(crate) sql: Option, + pub(crate) tags: Vec, + pub(crate) group: Option, + pub(crate) meta: Option, + pub(crate) columns: Vec, + pub(crate) column_lineage: DagColumnLineage, + pub(crate) materialization_type: Option, + pub(crate) language: Option, + pub(crate) return_kind: Option, + pub(crate) returns: Option, + pub(crate) loader: Option, + pub(crate) arguments: Vec<(String, String)>, +} + +impl NodeFields { + /// The node as `dataclasses.asdict` writes it. + pub(crate) fn value(self) -> DagValue { + object(vec![ + ("id", text(&self.id)), + ("kind", text(&self.kind)), + ("name", text(&self.name)), + ( + "asset_key", + texts(self.asset_key.iter().map(String::as_str)), + ), + ("target", self.target.unwrap_or(DagValue::Null)), + ("path", optional_text(self.path.as_deref())), + ("description", optional_text(self.description.as_deref())), + ("sql", optional_text(self.sql.as_deref())), + ("tags", texts(self.tags.iter().map(String::as_str))), + ("group", optional_text(self.group.as_deref())), + ("meta", self.meta.unwrap_or(DagValue::Object(Vec::new()))), + ("columns", DagValue::Array(self.columns)), + ( + "column_lineage", + DagValue::Object( + self.column_lineage + .iter() + .map(|(name, refs)| (Ok(name.clone()), lineage_refs(refs))) + .collect(), + ), + ), + ( + "materialization_type", + optional_text(self.materialization_type.as_deref()), + ), + ("language", optional_text(self.language.as_deref())), + ("return_kind", optional_text(self.return_kind.as_deref())), + ("returns", optional_text(self.returns.as_deref())), + ("loader", optional_text(self.loader.as_deref())), + ( + "arguments", + DagValue::Array( + self.arguments + .iter() + .map(|(name, data_type)| { + object(vec![("name", text(name)), ("type", text(data_type))]) + }) + .collect(), + ), + ), + ]) + } +} + +fn lineage_refs(refs: &[(String, String)]) -> DagValue { + DagValue::Array( + refs.iter() + .map(|(node, column)| object(vec![("node", text(node)), ("column", text(column))])) + .collect(), + ) +} + +/// `{resource type}:{name}`. +pub(crate) fn node_id((kind, name): &ObjectKey) -> String { + format!("{kind}:{name}") +} + +/// `loader:{name}`. +pub(crate) fn loader_node_id(name: &str) -> String { + format!("{LOADER_KIND}:{name}") +} + +/// A Python node's id: loaders share the loader namespace. +pub(crate) fn python_node_id(kind: &str, name: &str) -> String { + if kind == LOADER_KIND { + return loader_node_id(name); + } + format!("{kind}:{name}") +} + +fn present(parts: &[Option<&str>]) -> Vec { + parts + .iter() + .flatten() + .filter(|part| !part.is_empty()) + .map(|part| (*part).to_owned()) + .collect() +} + +fn entry_target(entry: (Option<&str>, Option<&str>, &str)) -> DagValue { + let (database, schema, name) = entry; + object(vec![ + ("database", optional_text(database)), + ("schema", optional_text(schema)), + ("name", text(name)), + ( + "qualified_name", + text(&present(&[database, schema, Some(name)]).join(".")), + ), + ("logical_database", DagValue::Null), + ("logical_schema", DagValue::Null), + ]) +} + +fn relation_target(target: &DagRelationInput) -> DagValue { + object(vec![ + ("database", optional_text(target.database.as_deref())), + ("schema", optional_text(target.schema.as_deref())), + ("name", optional_text(target.name.as_deref())), + ( + "qualified_name", + optional_text(target.qualified_name.as_deref()), + ), + ( + "logical_database", + optional_text(target.logical_database.as_deref()), + ), + ( + "logical_schema", + optional_text(target.logical_schema.as_deref()), + ), + ]) +} + +fn relation_asset_key(target: &DagRelationInput) -> Vec { + let database: Option<&str> = target + .logical_database + .as_deref() + .filter(|part| !part.is_empty()) + .or(target.database.as_deref()); + let schema: Option<&str> = target + .logical_schema + .as_deref() + .filter(|part| !part.is_empty()) + .or(target.schema.as_deref()); + present(&[database, schema, target.name.as_deref()]) +} + +fn entry_table(entry: &DagSourceEntryInput) -> &str { + entry + .table + .as_deref() + .filter(|table| !table.is_empty()) + .unwrap_or(&entry.name) +} + +fn columns(columns: &[DagColumnInput]) -> Vec { + columns.iter().map(column).collect() +} + +/// The source node. +pub(crate) fn source_node(source: &DagSourceInput) -> DagValue { + let entry: &DagSourceEntryInput = &source.entry; + let table: &str = entry_table(entry); + let mut asset_key: Vec = present(&[ + entry.database.as_deref(), + entry.schema.as_deref(), + Some(table), + ]); + if asset_key.is_empty() { + asset_key.push(entry.name.clone()); + } + NodeFields { + id: node_id(&source.key), + kind: SOURCE_KIND.to_owned(), + name: source.name.clone(), + asset_key, + target: Some(entry_target(( + entry.database.as_deref(), + entry.schema.as_deref(), + table, + ))), + path: Some(source.path.clone()), + description: entry.description.clone(), + meta: Some(entry.meta.clone()), + columns: columns(&entry.columns), + ..NodeFields::default() + } + .value() +} + +/// `loader_destination_parts`: `(database, schema, table)` of a one- to three-part destination. +fn destination_parts( + destination: &str, + defaults: (Option<&str>, Option<&str>), +) -> Result<(Option, Option, String), DagArtifactError> { + let parts: Vec<&str> = destination.split('.').collect(); + if parts.len() > LOADER_MAX_PARTS || parts.iter().any(|part| python_strip(part).is_empty()) { + return Err(DagArtifactError::LoaderDestination(format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ))); + } + let owned = |part: Option<&str>| part.map(str::to_owned); + Ok(match parts.as_slice() { + [table] => (owned(defaults.0), owned(defaults.1), (*table).to_owned()), + [schema, table] => ( + owned(defaults.0), + Some((*schema).to_owned()), + (*table).to_owned(), + ), + _ => ( + Some(parts[0].to_owned()), + Some(parts[1].to_owned()), + parts[2].to_owned(), + ), + }) +} + +/// The node of a loader no source declares. +pub(crate) fn loader_node( + loader: &DagLoaderInput, + project: &DagProjectInput, +) -> Result { + let destination: String = loader + .destination + .clone() + .unwrap_or_else(|| format!("{LOADER_DESTINATION_PREFIX}{}", loader.name)); + let (database, schema, table) = destination_parts( + &destination, + ( + project.target_database.as_deref(), + project.target_schema.as_deref(), + ), + )?; + let table: &str = if table.is_empty() { + &loader.name + } else { + &table + }; + Ok(NodeFields { + id: loader_node_id(&loader.name), + kind: LOADER_KIND.to_owned(), + name: loader.name.clone(), + asset_key: vec![loader.name.clone()], + target: Some(entry_target(( + database.as_deref(), + schema.as_deref(), + table, + ))), + path: Some(loader.path.clone()), + meta: Some(DagValue::Object(vec![( + Ok(LOADER_NODE_META_KEY.to_owned()), + DagValue::Bool(true), + )])), + columns: columns(&loader.columns), + loader: Some(loader.name.clone()), + ..NodeFields::default() + } + .value()) +} + +/// The seed node. +pub(crate) fn seed_node(seed: &DagSeedInput) -> DagValue { + NodeFields { + id: node_id(&seed.key), + kind: SEED_KIND.to_owned(), + name: seed.name.clone(), + asset_key: relation_asset_key(&seed.destination), + target: Some(relation_target(&seed.destination)), + path: Some(seed.path.clone()), + description: seed.description.clone(), + tags: seed.tags.clone(), + meta: Some(seed.meta.clone()), + columns: columns(&seed.columns), + ..NodeFields::default() + } + .value() +} + +/// The UDF or table function node. +pub(crate) fn function_node(function: &DagFunctionInput) -> DagValue { + let (kind, return_kind) = if function.return_columns.is_empty() { + (UDF_KIND, SCALAR_RETURN_KIND) + } else { + (TABLE_FUNCTION_KIND, TABLE_RETURN_KIND) + }; + NodeFields { + id: format!("{kind}:{}", function.name), + kind: kind.to_owned(), + name: function.name.clone(), + asset_key: relation_asset_key(&function.destination), + target: Some(relation_target(&function.destination)), + path: Some(function.path.clone()), + sql: Some(function.body_sql.clone()), + language: Some(function.language.clone()), + tags: function.tags.clone(), + return_kind: Some(return_kind.to_owned()), + returns: Some(function.returns.clone()), + columns: function.return_columns.iter().map(typed_column).collect(), + arguments: function.arguments.clone(), + ..NodeFields::default() + } + .value() +} + +fn model_tags(model: &DagModelInput) -> Vec { + let schema_tags: Vec = model + .schema + .as_ref() + .map_or_else(Vec::new, |schema| schema.tags.clone()); + let Some(config_tags) = &model.config_tags else { + return schema_tags; + }; + let mut tags: Vec = Vec::new(); + for tag in schema_tags.iter().chain(config_tags) { + if !tags.contains(tag) { + tags.push(tag.clone()); + } + } + tags +} + +fn model_columns(model: &DagModelInput) -> Vec { + if let Some(schema) = model + .schema + .as_ref() + .filter(|schema| !schema.columns.is_empty()) + { + return columns(&schema.columns); + } + model + .inferred_columns + .as_ref() + .map_or_else(Vec::new, |inferred| { + inferred.iter().map(typed_column).collect() + }) +} + +/// The model node. +pub(crate) fn model_node(model: &DagModelInput) -> DagValue { + NodeFields { + id: node_id(&model.key), + kind: MODEL_KIND.to_owned(), + name: model.name.clone(), + asset_key: relation_asset_key(&model.destination), + target: Some(relation_target(&model.destination)), + path: Some(model.path.clone()), + description: model + .schema + .as_ref() + .and_then(|schema| schema.description.clone()), + sql: Some(model.authored_sql.clone()), + tags: model_tags(model), + meta: model.schema.as_ref().map(|schema| schema.meta.clone()), + columns: model_columns(model), + materialization_type: Some(model.materialization.clone()), + ..NodeFields::default() + } + .value() +} + +/// The node of a Python task, asset or check. +pub(crate) fn python_node(node: &DagPythonNodeInput) -> DagValue { + let mut fields: NodeFields = NodeFields { + id: python_node_id(&node.kind, &node.name), + kind: node.kind.clone(), + name: node.name.clone(), + asset_key: vec![node.kind.clone(), node.name.clone()], + path: Some(node.path.clone()), + description: node.description.clone(), + tags: node.tags.clone(), + group: node.group.clone(), + meta: Some(node.meta.clone()), + ..NodeFields::default() + }; + if let Some(asset) = node.asset.as_ref().filter(|_| node.kind == ASSET_KIND) { + fields.columns = columns(&asset.columns); + fields.column_lineage = asset.column_lineage.clone().unwrap_or_default(); + fields.materialization_type = Some(DEFAULT_MATERIALIZATION_PYTHON_ASSET.to_owned()); + } + fields.value() +} + +/// Each loader's declaring source entry name, the last source naming a loader winning. +pub(crate) fn sources_by_loader(project: &DagProjectInput) -> HashMap<&str, &str> { + let mut sources: HashMap<&str, &str> = HashMap::new(); + for source in &project.sources { + if let Some(loader) = source.entry.loader.as_deref() { + sources.insert(loader, source.entry.name.as_str()); + } + } + sources +} + +/// The source node id of a loader a source declares, or its loader node id. +pub(crate) fn loader_or_source_id(name: &str, sources: &HashMap<&str, &str>) -> String { + sources.get(name).map_or_else( + || loader_node_id(name), + |source| node_id(&(SOURCE_KIND.to_owned(), (*source).to_owned())), + ) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs new file mode 100644 index 0000000000..d9c45cb611 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs @@ -0,0 +1,117 @@ +//! Building artifact values, dropping `None` and empty optional fields, and encoding them. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::dag_artifact::constants::REQUIRED_COLLECTION_KEYS; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagColumnInput, DagValue}; +use crate::dag_artifact::types::DagKey; + +/// An object whose keys are all field names. +pub(crate) fn object(entries: Vec<(&str, DagValue)>) -> DagValue { + DagValue::Object( + entries + .into_iter() + .map(|(key, value)| (Ok(key.to_owned()), value)) + .collect(), + ) +} + +/// A string field. +pub(crate) fn text(value: &str) -> DagValue { + DagValue::String(value.to_owned()) +} + +/// An optional string field. +pub(crate) fn optional_text(value: Option<&str>) -> DagValue { + value.map_or(DagValue::Null, text) +} + +/// A list of strings. +pub(crate) fn texts<'a>(values: impl IntoIterator) -> DagValue { + DagValue::Array(values.into_iter().map(text).collect()) +} + +/// An integer field. +pub(crate) fn integer(value: i64) -> DagValue { + DagValue::Integer(JsonInteger::from(value)) +} + +/// A `DagColumn` of an authored column. +pub(crate) fn column(column: &DagColumnInput) -> DagValue { + object(vec![ + ("name", text(&column.name)), + ("type", optional_text(column.data_type.as_deref())), + ( + "nullable", + column.nullable.map_or(DagValue::Null, DagValue::Bool), + ), + ("description", optional_text(column.description.as_deref())), + ("meta", column.meta.clone()), + ]) +} + +/// A `DagColumn` of only a name and type. +pub(crate) fn typed_column((name, data_type): &(String, Option)) -> DagValue { + object(vec![ + ("name", text(name)), + ("type", optional_text(data_type.as_deref())), + ("nullable", DagValue::Null), + ("description", DagValue::Null), + ("meta", DagValue::Object(Vec::new())), + ]) +} + +fn is_required_collection(key: &DagKey) -> bool { + key.as_ref() + .is_ok_and(|key| REQUIRED_COLLECTION_KEYS.contains(&key.as_str())) +} + +fn is_empty_collection(value: &DagValue) -> bool { + match value { + DagValue::Array(items) => items.is_empty(), + DagValue::Object(entries) => entries.is_empty(), + _ => false, + } +} + +/// `_drop_none`: drop `None` fields, then empty optional collections, at every depth. +pub(crate) fn drop_none(value: DagValue) -> DagValue { + match value { + DagValue::Array(items) => DagValue::Array(items.into_iter().map(drop_none).collect()), + DagValue::Object(entries) => DagValue::Object( + entries + .into_iter() + .filter(|(_, item)| *item != DagValue::Null) + .map(|(key, item)| (key, drop_none(item))) + .filter(|(key, item)| is_required_collection(key) || !is_empty_collection(item)) + .collect(), + ), + other => other, + } +} + +/// The value `json.dumps` encodes, or the `TypeError` it raises first in output order. +pub(crate) fn encoded(value: DagValue) -> Result { + Ok(match value { + DagValue::Null => JsonValue::Null, + DagValue::Bool(flag) => JsonValue::Bool(flag), + DagValue::Integer(number) => JsonValue::Integer(number), + DagValue::Float(number) => JsonValue::Float(number), + DagValue::String(text) => JsonValue::String(text), + DagValue::Utf16(units) => JsonValue::Utf16(units), + DagValue::Array(items) => { + JsonValue::Array(items.into_iter().map(encoded).collect::>()?) + } + DagValue::Object(entries) => JsonValue::Object( + entries + .into_iter() + .map(|(key, item)| { + let key: String = key.map_err(DagArtifactError::Unencodable)?; + Ok((key, encoded(item)?)) + }) + .collect::>()?, + ), + DagValue::Unencodable(message) => return Err(DagArtifactError::Unencodable(message)), + }) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs new file mode 100644 index 0000000000..8b877efc20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -0,0 +1,29 @@ +//! Versions, kinds and keys of the static DAG artifact. + +/// The artifact's `version`. +pub(crate) const DAG_VERSION: i64 = 1; +/// Keys whose empty collections are written instead of dropped. +pub(crate) const REQUIRED_COLLECTION_KEYS: [&str; 3] = ["nodes", "edges", "checks"]; +/// `json.dumps(..., indent=2)`. +pub(crate) const DAG_INDENT: usize = 2; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const LOADER_KIND: &str = "loader"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub(crate) const ASSET_KIND: &str = "asset"; +pub(crate) const CHECK_KIND: &str = "check"; +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; +pub(crate) const AUDIT_KIND: &str = "audit"; +pub(crate) const SCENARIO_KIND: &str = "scenario"; +pub(crate) const PYTHON_CHECK_KIND: &str = "python_check"; +pub(crate) const DEFAULT_MATERIALIZATION_PYTHON_ASSET: &str = "python_asset"; +pub(crate) const TABLE_RETURN_KIND: &str = "table"; +pub(crate) const SCALAR_RETURN_KIND: &str = "scalar"; +/// The meta of a loader node no source declares. +pub(crate) const LOADER_NODE_META_KEY: &str = "sqlbuild_loader_node"; +/// The destination of a loader without one. +pub(crate) const LOADER_DESTINATION_PREFIX: &str = "__loader__"; +/// The most parts a loader destination may have. +pub(crate) const LOADER_MAX_PARTS: usize = 3; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs new file mode 100644 index 0000000000..b37aa7a6eb --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs @@ -0,0 +1,16 @@ +//! Why the DAG artifact could not be built, named after the Python error each reproduces. + +use sqlbuild_core::json::errors::JsonEmitError; + +/// A failure building or encoding the DAG artifact. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DagArtifactError { + /// `SpecConfigError` for a loader destination that is not 1 to 3 non-empty parts. + LoaderDestination(String), + /// `KeyError` for a bound SQL test parameter missing from the parameter schema. + MissingParameterType(String), + /// `TypeError` for the first value `json.dumps` cannot encode. + Unencodable(String), + /// The emitter refused the encoded artifact. + Emit(JsonEmitError), +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs new file mode 100644 index 0000000000..0e2c34a039 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs @@ -0,0 +1,64 @@ +//! The static DAG artifact JSON of a compiled project and its Python node graph. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::checks::checks; +use crate::dag_artifact::_helpers::edges::edges; +use crate::dag_artifact::_helpers::nodes::{ + function_node, loader_node, model_node, python_node, seed_node, source_node, sources_by_loader, +}; +use crate::dag_artifact::_helpers::values::{drop_none, encoded, integer, object, text}; +use crate::dag_artifact::constants::{DAG_INDENT, DAG_VERSION, LOADER_KIND}; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::graph::models::ProjectGraph; + +fn nodes( + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, +) -> Result, DagArtifactError> { + let declared: HashMap<&str, &str> = sources_by_loader(project); + let mut nodes: Vec = project.sources.iter().map(source_node).collect(); + for loader in &project.loaders { + if !declared.contains_key(loader.name.as_str()) { + nodes.push(loader_node(loader, project)?); + } + } + nodes.extend(project.seeds.iter().map(seed_node)); + nodes.extend(project.functions.iter().map(function_node)); + nodes.extend(project.models.iter().map(model_node)); + if let Some(python) = python { + nodes.extend( + python + .nodes + .iter() + .filter(|node| node.kind != LOADER_KIND) + .map(python_node), + ); + } + Ok(nodes) +} + +/// `json.dumps` of the artifact with `None` and empty optional fields dropped, indented by two. +pub fn build_dag_json( + graph: &ProjectGraph, + project: &DagProjectInput, + python: Option<&DagPythonGraphInput>, + project_name: &str, +) -> Result { + let artifact: DagValue = object(vec![ + ("version", integer(DAG_VERSION)), + ("project_name", text(project_name)), + ("nodes", DagValue::Array(nodes(project, python)?)), + ("edges", edges(graph, project, python)), + ("checks", checks(project, python)?), + ]); + let value: JsonValue = encoded(drop_none(artifact))?; + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(DAG_INDENT))), + ) + .map_err(DagArtifactError::Emit) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs new file mode 100644 index 0000000000..60780629b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -0,0 +1 @@ +pub mod build_dag_json; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/mod.rs new file mode 100644 index 0000000000..38a9736573 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs new file mode 100644 index 0000000000..157309cb8e --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -0,0 +1,236 @@ +//! The compiled project, Python node graph and authored values the DAG artifact is built from. + +use sqlbuild_core::json::models::JsonInteger; + +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::types::{DagColumnLineage, DagKey}; + +/// A Python value as `dataclasses.asdict` copies it, before `json.dumps` encodes it. +#[derive(Clone, Debug, PartialEq)] +pub enum DagValue { + Null, + Bool(bool), + Integer(JsonInteger), + Float(f64), + String(String), + /// A `str` holding lone surrogates, as UTF-16 code units. + Utf16(Vec), + Array(Vec), + Object(Vec<(DagKey, DagValue)>), + /// A value `json.dumps` refuses, with the message of its `TypeError`. + Unencodable(String), +} + +/// One authored column: name, type, nullability, description and meta. +#[derive(Clone, Debug, PartialEq)] +pub struct DagColumnInput { + pub name: String, + pub data_type: Option, + pub nullable: Option, + pub description: Option, + pub meta: DagValue, +} + +/// A compiled relation location. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagRelationInput { + pub database: Option, + pub schema: Option, + pub name: Option, + pub qualified_name: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +/// A source declaration's entry. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSourceEntryInput { + pub name: String, + pub database: Option, + pub schema: Option, + pub table: Option, + pub description: Option, + pub loader: Option, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled source. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSourceInput { + pub key: ObjectKey, + pub name: String, + /// `str(source_file.relative_path)`. + pub path: String, + pub entry: DagSourceEntryInput, +} + +/// A discovered loader function. +#[derive(Clone, Debug, PartialEq)] +pub struct DagLoaderInput { + pub name: String, + pub path: String, + /// The authored destination, if any. + pub destination: Option, + pub columns: Vec, + /// Names of the project loaders this loader depends on, in dependency order. + pub dependency_names: Vec, +} + +/// A compiled seed. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSeedInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub description: Option, + pub tags: Vec, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled SQL or Python function. +#[derive(Clone, Debug, PartialEq)] +pub struct DagFunctionInput { + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub body_sql: String, + pub language: String, + pub tags: Vec, + pub returns: String, + /// `(name, type)` of each returned column; a table function has at least one. + pub return_columns: Vec<(String, Option)>, + pub arguments: Vec<(String, String)>, +} + +/// The parts of a model's schema entry the artifact reads. +#[derive(Clone, Debug, PartialEq)] +pub struct DagModelSchemaInput { + pub description: Option, + pub tags: Vec, + pub meta: DagValue, + pub columns: Vec, +} + +/// A compiled model. +#[derive(Clone, Debug, PartialEq)] +pub struct DagModelInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub destination: DagRelationInput, + pub authored_sql: String, + pub schema: Option, + /// `str()` of each configured tag, when the `tags` config value is a list or tuple. + pub config_tags: Option>, + /// `str()` of the `materialized` config value, `view` by default. + pub materialization: String, + pub inferred_columns: Option)>>, +} + +/// A Python asset's declared columns and column lineage. +#[derive(Clone, Debug, PartialEq)] +pub struct DagPythonAssetInput { + pub columns: Vec, + /// Column to `(upstream node, upstream column)` references, when declared. + pub column_lineage: Option, +} + +/// A discovered Python node. +#[derive(Clone, Debug, PartialEq)] +pub struct DagPythonNodeInput { + pub kind: String, + pub name: String, + pub path: String, + pub description: Option, + pub tags: Vec, + pub group: Option, + /// `node.meta or {}`. + pub meta: DagValue, + pub asset: Option, + /// The `AuditSeverity` value of a check's severity. + pub check_severity: Option, + /// Graph keys of the SQL resources the node reads. + pub sql_deps: Vec, +} + +/// The discovered Python node graph. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DagPythonGraphInput { + pub nodes: Vec, + /// `(upstream name, downstream name)` of each dependency edge. + pub dependency_edges: Vec<(String, String)>, + /// Each name's node kind, as `nodes_by_name` maps it. + pub kinds_by_name: Vec<(String, String)>, +} + +/// One case of a parameterized SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSqlTestCaseInput { + pub case_name: String, + pub block_index: Option, + pub parent_name: Option, + pub case_index: Option, + pub case_fingerprint: Option, + /// `(name, value type, nullable)` of each declared parameter. + pub parameter_schema: Vec<(String, String, bool)>, + /// `(name, value)` of each bound parameter, decimals already as text. + pub parameters: Vec<(String, DagValue)>, +} + +/// A compiled SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct DagSqlTestInput { + pub key: ObjectKey, + pub name: String, + /// `str(test_file.relative_path)`. + pub path: String, + /// `(source_path or test_file.relative_path).as_posix()`. + pub source_path: String, + pub mode: String, + pub scope_deps: Vec, + pub case: Option, +} + +/// A compiled audit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DagAuditInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub scope_deps: Vec, + pub attached_target_kind: Option, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub severity: Option, +} + +/// A compiled SQL scenario. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DagScenarioInput { + pub key: ObjectKey, + pub name: String, + pub path: String, + pub expected_model_names: Vec, + pub authored_cte_names: Vec, + pub assertion_names: Vec, + /// Source, ref, seed and dbt ref fixture names, each group in authored order. + pub fixture_names: [Vec; 4], +} + +/// The project's compiled resources and target defaults. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DagProjectInput { + pub sources: Vec, + pub loaders: Vec, + pub seeds: Vec, + pub functions: Vec, + pub models: Vec, + pub sql_tests: Vec, + pub audits: Vec, + pub scenarios: Vec, + pub target_database: Option, + pub target_schema: Option, +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs new file mode 100644 index 0000000000..e7bae12d29 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_dag_json.rs"] +mod build_dag_json; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs new file mode 100644 index 0000000000..ec04a5e978 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -0,0 +1,97 @@ +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::main::build_dag_json::build_dag_json; +use crate::dag_artifact::models::{ + DagColumnInput, DagLoaderInput, DagPythonNodeInput, DagSourceEntryInput, DagSourceInput, + DagValue, +}; +use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::graph::models::ProjectGraph; + +/// A `(resource type, name)` key. +pub(crate) fn key(kind: &str, name: &str) -> ObjectKey { + (kind.to_owned(), name.to_owned()) +} + +/// A typed column without meta. +pub(crate) fn column(name: &str, data_type: &str) -> DagColumnInput { + DagColumnInput { + name: name.to_owned(), + data_type: Some(data_type.to_owned()), + nullable: None, + description: None, + meta: DagValue::Object(Vec::new()), + } +} + +/// A source in `raw` loaded by `loader`, when given. +pub(crate) fn source(name: &str, loader: Option<&str>, meta: DagValue) -> DagSourceInput { + DagSourceInput { + key: key("source", name), + name: name.to_owned(), + path: "sources/raw.yml".to_owned(), + entry: DagSourceEntryInput { + name: name.to_owned(), + database: None, + schema: Some("raw".to_owned()), + table: None, + description: None, + loader: loader.map(str::to_owned), + meta, + columns: vec![column("id", "INTEGER")], + }, + } +} + +/// A loader at `destination` depending on `dependencies`. +pub(crate) fn loader( + name: &str, + destination: Option<&str>, + dependencies: &[&str], +) -> DagLoaderInput { + DagLoaderInput { + name: name.to_owned(), + path: "python/loaders.py".to_owned(), + destination: destination.map(str::to_owned), + columns: Vec::new(), + dependency_names: dependencies.iter().map(|name| (*name).to_owned()).collect(), + } +} + +/// A Python node of `kind` reading the `sql_deps` keys. +pub(crate) fn python_node(kind: &str, name: &str, sql_deps: Vec) -> DagPythonNodeInput { + DagPythonNodeInput { + kind: kind.to_owned(), + name: name.to_owned(), + path: "python/nodes.py".to_owned(), + description: None, + tags: Vec::new(), + group: None, + meta: DagValue::Object(Vec::new()), + asset: None, + check_severity: (kind == "check").then(|| "error".to_owned()), + sql_deps, + } +} + +/// The artifact JSON of a test case. +pub(crate) fn built(test_case: &BuildDagJsonTestCase) -> Result { + let graph: ProjectGraph = ProjectGraph { + upstream: test_case.upstream.clone(), + ..ProjectGraph::default() + }; + build_dag_json( + &graph, + &test_case.project, + test_case.python.as_ref(), + "orders_project", + ) +} + +/// Source meta with a `None` owner and the given team. +pub(crate) fn team_meta(value: DagValue) -> DagValue { + DagValue::Object(vec![ + (Ok("owner".to_owned()), DagValue::Null), + (Ok("team".to_owned()), value), + ]) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs new file mode 100644 index 0000000000..003d133324 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -0,0 +1,173 @@ +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::dag_artifact::tests::helpers::{built, key, loader, python_node, source, team_meta}; +use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; + +const EMPTY_JSON: &str = "{ + \"version\": 1, + \"project_name\": \"orders_project\", + \"nodes\": [], + \"edges\": [], + \"checks\": [] +}"; + +const LOADED_JSON: &str = "{ + \"version\": 1, + \"project_name\": \"orders_project\", + \"nodes\": [ + { + \"id\": \"source:orders\", + \"kind\": \"source\", + \"name\": \"orders\", + \"asset_key\": [ + \"raw\", + \"orders\" + ], + \"target\": { + \"schema\": \"raw\", + \"name\": \"orders\", + \"qualified_name\": \"raw.orders\" + }, + \"path\": \"sources/raw.yml\", + \"meta\": { + \"team\": \"ops\" + }, + \"columns\": [ + { + \"name\": \"id\", + \"type\": \"INTEGER\" + } + ] + }, + { + \"id\": \"loader:load_customers\", + \"kind\": \"loader\", + \"name\": \"load_customers\", + \"asset_key\": [ + \"load_customers\" + ], + \"target\": { + \"schema\": \"main\", + \"name\": \"__loader__load_customers\", + \"qualified_name\": \"main.__loader__load_customers\" + }, + \"path\": \"python/loaders.py\", + \"meta\": { + \"sqlbuild_loader_node\": true + }, + \"loader\": \"load_customers\" + }, + { + \"id\": \"check:orders_check\", + \"kind\": \"check\", + \"name\": \"orders_check\", + \"asset_key\": [ + \"check\", + \"orders_check\" + ], + \"path\": \"python/nodes.py\" + } + ], + \"edges\": [ + { + \"from_id\": \"source:orders\", + \"to_id\": \"check:orders_check\" + }, + { + \"from_id\": \"source:orders\", + \"to_id\": \"loader:load_customers\" + } + ], + \"checks\": [ + { + \"id\": \"check:orders_check\", + \"kind\": \"python_check\", + \"name\": \"orders_check\", + \"checked_asset_ids\": [ + \"source:orders\" + ], + \"path\": \"python/nodes.py\", + \"severity\": \"error\" + } + ] +}"; + +#[test] +fn given_project_when_building_dag_json_then_python_artifact_agrees() { + let test_cases = [ + BuildDagJsonTestCase { + description: "an empty project keeps its required collections", + project: DagProjectInput::default(), + python: None, + upstream: Vec::new(), + expected_json: Ok(EMPTY_JSON), + }, + BuildDagJsonTestCase { + description: "declared loaders become sources and None and empty fields are dropped", + project: DagProjectInput { + sources: vec![source( + "orders", + Some("load_orders"), + team_meta(DagValue::String("ops".to_owned())), + )], + loaders: vec![ + loader("load_orders", None, &[]), + loader("load_customers", None, &["load_orders"]), + ], + target_schema: Some("main".to_owned()), + ..DagProjectInput::default() + }, + python: Some(DagPythonGraphInput { + nodes: vec![python_node( + "check", + "orders_check", + vec![key("source", "orders")], + )], + dependency_edges: Vec::new(), + kinds_by_name: vec![("orders_check".to_owned(), "check".to_owned())], + }), + upstream: Vec::new(), + expected_json: Ok(LOADED_JSON), + }, + BuildDagJsonTestCase { + description: "a loader destination with an empty part is refused", + project: DagProjectInput { + loaders: vec![loader("load_orders", Some("raw..orders"), &[])], + ..DagProjectInput::default() + }, + python: None, + upstream: Vec::new(), + expected_json: Err(DagArtifactError::LoaderDestination( + "Invalid loader destination 'raw..orders': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + .to_owned(), + )), + }, + BuildDagJsonTestCase { + description: "a meta value json cannot encode raises its TypeError", + project: DagProjectInput { + sources: vec![source( + "orders", + None, + team_meta(DagValue::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + )), + )], + ..DagProjectInput::default() + }, + python: None, + upstream: Vec::new(), + expected_json: Err(DagArtifactError::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + built(&test_case), + test_case.expected_json.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs new file mode 100644 index 0000000000..529ebbda57 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs @@ -0,0 +1,11 @@ +use crate::assembly::project::types::ObjectKey; +use crate::dag_artifact::errors::DagArtifactError; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; + +pub(crate) struct BuildDagJsonTestCase { + pub(crate) description: &'static str, + pub(crate) project: DagProjectInput, + pub(crate) python: Option, + pub(crate) upstream: Vec<(ObjectKey, Vec)>, + pub(crate) expected_json: Result<&'static str, DagArtifactError>, +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/types.rs b/crates/sqlbuild-analysis/src/dag_artifact/types.rs new file mode 100644 index 0000000000..3154b438ad --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/types.rs @@ -0,0 +1,7 @@ +//! Keys and lineage shapes of the DAG artifact inputs. + +/// A mapping key as `json.dumps` writes it, or the message of the `TypeError` it raises. +pub type DagKey = Result; + +/// Column to `(upstream node, upstream column)` references. +pub type DagColumnLineage = Vec<(String, Vec<(String, String)>)>; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index b474f08964..5e0d31c0c9 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -9,6 +9,7 @@ pub mod compiled_project; pub mod compiler; mod constants; pub mod contracts; +pub mod dag_artifact; pub mod explicit_references; pub mod graph; pub mod lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index ee36509c2c..a15255bdf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -2,13 +2,13 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyTuple}; +use pyo3::types::{PyBool, PyBytes, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; +use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const DAG_MODULE: &str = "sqlbuild.compiler.dag.main.build"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; const ARTIFACT_BATCH_MODULE: &str = "sqlbuild.cli.commands.classes.native_artifact_batch"; const TARGET_WRITER_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.target_writer"; @@ -120,16 +120,14 @@ pub(crate) fn write_dag_artifact<'py>( &[("discovered_inputs", discovered.clone())], )?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; - let dag_json: Bound<'py, PyAny> = call_with( - &imported(py, DAG_MODULE, "build_dag_json")?, - &[ - ("graph", analysis.getattr("graph")?), - ( - "project_name", - discovered.getattr("project_config")?.getattr("name")?, - ), - ("python_graph", python_graph), - ], + let project_name: String = discovered + .getattr("project_config")? + .getattr("name")? + .extract()?; + let dag_json: String = build_dag_json( + &analysis.getattr("graph")?, + &project_name, + Some(&python_graph), )?; let batch: Bound<'py, PyAny> = call_with( &imported(py, ARTIFACT_BATCH_MODULE, "NativeArtifactBatch")?, @@ -142,7 +140,7 @@ pub(crate) fn write_dag_artifact<'py>( &batch.getattr("queue")?, &[ ("path", path), - ("contents", dag_json.call_method1("encode", ("utf-8",))?), + ("contents", PyBytes::new(py, dag_json.as_bytes()).into_any()), ], )?; batch.call_method0("flush")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index d96a085d1f..c502f0aaef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -220,7 +220,7 @@ impl Conversion { } /// `json.dumps` writes a float key as `float.__repr__`, or as `NaN` / `Infinity`. -fn float_key(key: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn float_key(key: &Bound<'_, PyAny>) -> PyResult { let number: f64 = key.extract::()?; if number.is_nan() { return Ok("NaN".to_owned()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs new file mode 100644 index 0000000000..a85b7a51d6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -0,0 +1,57 @@ +//! `build_dag_json`: the static DAG artifact of a compiled project graph. + +use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::{PyRef, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; +use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; +use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; + +const SPEC_EXCEPTIONS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; + +fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { + match error { + DagArtifactError::LoaderDestination(message) => py + .import(SPEC_EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("SpecConfigError")) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + DagArtifactError::MissingParameterType(name) => PyKeyError::new_err(name), + DagArtifactError::Unencodable(message) => PyTypeError::new_err(message), + DagArtifactError::Emit(JsonEmitError::NestingTooDeep) => { + PyRecursionError::new_err("maximum recursion depth exceeded") + } + DagArtifactError::Emit(error) => PyValueError::new_err(format!("{error:?}")), + } +} + +/// The DAG JSON of `graph` (a `ProjectGraph`) and the optional discovered Python node graph. +#[pyfunction] +#[pyo3(signature = (*, graph, project_name, python_graph=None))] +pub(crate) fn build_dag_json( + graph: &Bound<'_, PyAny>, + project_name: &str, + python_graph: Option<&Bound<'_, PyAny>>, +) -> PyResult { + compiler_guard(|| { + let py = graph.py(); + let project: DagProjectInput = project_input(&graph.getattr("project")?)?; + let python: Option = python_graph + .filter(|python_graph| !python_graph.is_none()) + .map(python_graph_input) + .transpose()?; + let native: Bound<'_, PyAny> = graph.getattr("native")?; + let native: PyRef<'_, NativeProjectGraph> = native.extract()?; + build(native.graph(), &project, python.as_ref(), project_name) + .map_err(|error| python_error(error, py)) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_dag_json, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs new file mode 100644 index 0000000000..c3b3b90372 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod artifact; +pub(crate) mod registration; +pub(crate) mod rows; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs new file mode 100644 index 0000000000..29c2baeab9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -0,0 +1,9 @@ +//! Register the DAG artifact builder. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dag_artifact::artifact; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + artifact::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs new file mode 100644 index 0000000000..0509429a49 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs @@ -0,0 +1,456 @@ +//! Read the compiled project views and the Python node graph into DAG artifact inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::dag_artifact::models::{ + DagAuditInput, DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, + DagModelSchemaInput, DagProjectInput, DagPythonAssetInput, DagPythonGraphInput, + DagPythonNodeInput, DagRelationInput, DagScenarioInput, DagSeedInput, DagSourceEntryInput, + DagSourceInput, DagSqlTestCaseInput, DagSqlTestInput, DagValue, +}; + +use sqlbuild_analysis::dag_artifact::types::DagColumnLineage; + +use crate::bindings::_helpers::dag_artifact::values::dag_value; + +const AUDIT_SEVERITY_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const DECIMAL_KIND: &str = "decimal"; +const ASSET_KIND: &str = "asset"; +const CHECK_KIND: &str = "check"; +const DEFAULT_MATERIALIZATION: &str = "view"; + +fn attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attribute(value, name)?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)?.extract() +} + +/// `str(value.)`. +fn path_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(attribute(value, name)?.str()?.to_string()) +} + +/// The `.value` of an optional enum attribute. +fn enum_value(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let member: Bound<'_, PyAny> = attribute(value, name)?; + if member.is_none() { + return Ok(None); + } + member.getattr("value")?.extract().map(Some) +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + attribute(value, name)?.try_iter()?.collect() +} + +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok((text(key, "resource_type")?, text(key, "name")?)) +} + +fn keys(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(object_key).collect() +} + +fn column(column: &Bound<'_, PyAny>) -> PyResult { + Ok(DagColumnInput { + name: text(column, "name")?, + data_type: optional_text(column, "type")?, + nullable: attribute(column, "nullable")?.extract()?, + description: optional_text(column, "description")?, + meta: dag_value(&attribute(column, "meta")?)?, + }) +} + +fn columns(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(column).collect() +} + +fn named_types(value: &Bound<'_, PyAny>, name: &str) -> PyResult)>> { + items(value, name)? + .iter() + .map(|item| Ok((text(item, "name")?, optional_text(item, "type")?))) + .collect() +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(DagRelationInput { + database: optional_text(value, "database")?, + schema: optional_text(value, "schema")?, + name: optional_text(value, "name")?, + qualified_name: optional_text(value, "qualified_name")?, + logical_database: optional_text(value, "logical_database")?, + logical_schema: optional_text(value, "logical_schema")?, + }) +} + +fn source(source: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(source, "source_entry")?; + Ok(DagSourceInput { + key: object_key(&attribute(source, "key")?)?, + name: text(source, "name")?, + path: path_text(&attribute(source, "source_file")?, "relative_path")?, + entry: DagSourceEntryInput { + name: text(&entry, "name")?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?, + table: optional_text(&entry, "table")?, + description: optional_text(&entry, "description")?, + loader: optional_text(&entry, "loader")?, + meta: dag_value(&attribute(&entry, "meta")?)?, + columns: columns(&entry, "columns")?, + }, + }) +} + +fn loaders(project: &Bound<'_, PyAny>) -> PyResult> { + let loaders: Vec> = items(project, "loader_functions")?; + let names: Bound<'_, PyDict> = PyDict::new(project.py()); + for loader in &loaders { + names.set_item(attribute(loader, "function")?, attribute(loader, "name")?)?; + } + loaders + .iter() + .map(|loader| { + let mut dependency_names: Vec = Vec::new(); + for dependency in items(loader, "depends_on")? { + if let Some(name) = names.get_item(dependency)? + && !name.is_none() + { + dependency_names.push(name.extract()?); + } + } + Ok(DagLoaderInput { + name: text(loader, "name")?, + path: path_text(loader, "relative_path")?, + destination: optional_text(loader, "destination")?, + columns: columns(loader, "columns")?, + dependency_names, + }) + }) + .collect() +} + +fn seed(seed: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(seed, "schema_entry")?; + Ok(DagSeedInput { + key: object_key(&attribute(seed, "key")?)?, + name: text(seed, "name")?, + path: path_text(&attribute(seed, "seed_file")?, "relative_path")?, + destination: relation(&attribute(seed, "destination")?)?, + description: optional_text(&entry, "description")?, + tags: texts(&entry, "tags")?, + meta: dag_value(&attribute(&entry, "meta")?)?, + columns: columns(&entry, "columns")?, + }) +} + +fn function(function: &Bound<'_, PyAny>) -> PyResult { + Ok(DagFunctionInput { + name: text(function, "name")?, + path: path_text(function, "relative_path")?, + destination: relation(&attribute(function, "destination")?)?, + body_sql: text(function, "body_sql")?, + language: enum_value(function, "language")?.unwrap_or_default(), + tags: texts(function, "tags")?, + returns: text(function, "returns")?, + return_columns: named_types(function, "return_columns")?, + arguments: items(function, "arguments")? + .iter() + .map(|argument| Ok((text(argument, "name")?, text(argument, "type")?))) + .collect::>()?, + }) +} + +fn model_schema(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + Ok(Some(DagModelSchemaInput { + description: optional_text(entry, "description")?, + tags: texts(entry, "tags")?, + meta: dag_value(&attribute(entry, "meta")?)?, + columns: columns(entry, "columns")?, + })) +} + +/// `str()` of each configured tag when the `tags` value is a list or tuple. +fn config_tags(values: &Bound<'_, PyAny>) -> PyResult>> { + let tags: Bound<'_, PyAny> = + values.call_method1("get", ("tags", PyTuple::empty(values.py())))?; + if !(tags.is_instance_of::() || tags.is_instance_of::()) { + return Ok(None); + } + tags.try_iter()? + .map(|tag| Ok(tag?.str()?.to_string())) + .collect::>() + .map(Some) +} + +fn model(model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = attribute(model, "config")?.getattr("values")?; + let inferred: Bound<'_, PyAny> = attribute(model, "inferred_columns")?; + Ok(DagModelInput { + key: object_key(&attribute(model, "key")?)?, + name: text(model, "name")?, + path: path_text(model, "relative_path")?, + destination: relation(&attribute(model, "destination")?)?, + authored_sql: text(model, "authored_sql")?, + schema: model_schema(&attribute(model, "schema_entry")?)?, + config_tags: config_tags(&values)?, + materialization: values + .call_method1("get", ("materialized", DEFAULT_MATERIALIZATION))? + .str()? + .to_string(), + inferred_columns: if inferred.is_none() { + None + } else { + Some(named_types(model, "inferred_columns")?) + }, + }) +} + +fn sql_test_case(test: &Bound<'_, PyAny>) -> PyResult> { + let Some(case_name) = optional_text(test, "case_name")? else { + return Ok(None); + }; + let parameter_schema: Vec<(String, String, bool)> = items(test, "parameter_schema")? + .iter() + .map(|parameter| { + Ok(( + text(parameter, "name")?, + enum_value(parameter, "value_type")?.unwrap_or_default(), + attribute(parameter, "nullable")?.extract()?, + )) + }) + .collect::>()?; + let mut parameters: Vec<(String, DagValue)> = Vec::new(); + for pair in items(test, "parameter_values")? { + let (name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let raw: Bound<'_, PyAny> = attribute(&value, "value")?; + let is_decimal: bool = enum_value(&value, "kind")?.as_deref() == Some(DECIMAL_KIND); + let converted: DagValue = if is_decimal { + DagValue::String(raw.str()?.to_string()) + } else { + dag_value(&raw)? + }; + parameters.push((name, converted)); + } + Ok(Some(DagSqlTestCaseInput { + case_name, + block_index: attribute(test, "block_index")?.extract()?, + parent_name: optional_text(test, "parent_name")?, + case_index: attribute(test, "case_index")?.extract()?, + case_fingerprint: optional_text(test, "case_fingerprint")?, + parameter_schema, + parameters, + })) +} + +fn sql_test(test: &Bound<'_, PyAny>) -> PyResult { + let test_file_path: Bound<'_, PyAny> = + attribute(test, "test_file")?.getattr("relative_path")?; + let source_path: Bound<'_, PyAny> = attribute(test, "source_path")?; + let source_path: Bound<'_, PyAny> = if source_path.is_none() { + test_file_path.clone() + } else { + source_path + }; + Ok(DagSqlTestInput { + key: object_key(&attribute(test, "key")?)?, + name: text(test, "name")?, + path: test_file_path.str()?.to_string(), + source_path: source_path.call_method0("as_posix")?.extract()?, + mode: enum_value(test, "mode")?.unwrap_or_default(), + scope_deps: keys(test, "scope_deps")?, + case: sql_test_case(test)?, + }) +} + +fn audit(audit: &Bound<'_, PyAny>) -> PyResult { + Ok(DagAuditInput { + key: object_key(&attribute(audit, "key")?)?, + name: text(audit, "name")?, + path: path_text(&attribute(audit, "audit_file")?, "relative_path")?, + scope_deps: keys(audit, "scope_deps")?, + attached_target_kind: enum_value(audit, "attached_target_kind")?, + attached_target_name: optional_text(audit, "attached_target_name")?, + attached_column_name: optional_text(audit, "attached_column_name")?, + severity: enum_value(audit, "severity")?, + }) +} + +fn scenario(scenario: &Bound<'_, PyAny>) -> PyResult { + Ok(DagScenarioInput { + key: object_key(&attribute(scenario, "key")?)?, + name: text(scenario, "name")?, + path: path_text(&attribute(scenario, "scenario_file")?, "relative_path")?, + expected_model_names: texts(scenario, "expected_model_names")?, + authored_cte_names: items(scenario, "authored_ctes")? + .iter() + .map(|cte| text(cte, "name")) + .collect::>()?, + assertion_names: texts(scenario, "assertion_names")?, + fixture_names: [ + texts(scenario, "source_fixture_names")?, + texts(scenario, "ref_fixture_names")?, + texts(scenario, "seed_fixture_names")?, + texts(scenario, "dbt_ref_fixture_names")?, + ], + }) +} + +/// The compiled project's resources and target defaults. +pub(crate) fn project_input(project: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectInput { + sources: items(project, "sources")? + .iter() + .map(source) + .collect::>()?, + loaders: loaders(project)?, + seeds: items(project, "seeds")? + .iter() + .map(seed) + .collect::>()?, + functions: items(project, "functions")? + .iter() + .map(function) + .collect::>()?, + models: items(project, "models")? + .iter() + .map(model) + .collect::>()?, + sql_tests: items(project, "sql_tests")? + .iter() + .map(sql_test) + .collect::>()?, + audits: items(project, "audits")? + .iter() + .map(audit) + .collect::>()?, + scenarios: items(project, "sql_scenarios")? + .iter() + .map(scenario) + .collect::>()?, + target_database: optional_text(project, "effective_target_database")?, + target_schema: optional_text(project, "effective_target_schema")?, + }) +} + +fn python_asset(node: &Bound<'_, PyAny>) -> PyResult> { + let asset: Bound<'_, PyAny> = attribute(node, "asset")?; + if asset.is_none() { + return Ok(None); + } + let lineage: Bound<'_, PyAny> = attribute(&asset, "column_lineage")?; + let column_lineage: Option = if lineage.is_none() { + None + } else { + let mut entries: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for (column, refs) in lineage.downcast::()?.iter() { + let refs: Vec<(String, String)> = refs + .try_iter()? + .map(|reference| { + let reference: Bound<'_, PyAny> = reference?; + Ok((text(&reference, "node")?, text(&reference, "column")?)) + }) + .collect::>()?; + entries.push((column.extract()?, refs)); + } + Some(entries) + }; + Ok(Some(DagPythonAssetInput { + columns: columns(&asset, "columns")?, + column_lineage, + })) +} + +fn check_severity(node: &Bound<'_, PyAny>) -> PyResult> { + let check: Bound<'_, PyAny> = attribute(node, "check")?; + if check.is_none() { + return Ok(None); + } + let severity: Bound<'_, PyAny> = check.getattr("severity")?.getattr("value")?; + node.py() + .import(AUDIT_SEVERITY_MODULE)? + .getattr("AuditSeverity")? + .call1((severity,))? + .getattr("value")? + .extract() + .map(Some) +} + +fn python_node(node: &Bound<'_, PyAny>) -> PyResult { + let kind: String = enum_value(node, "kind")?.unwrap_or_default(); + let meta: Bound<'_, PyAny> = attribute(node, "meta")?; + Ok(DagPythonNodeInput { + asset: if kind == ASSET_KIND { + python_asset(node)? + } else { + None + }, + check_severity: if kind == CHECK_KIND { + check_severity(node)? + } else { + None + }, + kind, + name: text(node, "name")?, + path: path_text(node, "relative_path")?, + description: optional_text(node, "description")?, + tags: texts(node, "tags")?, + group: optional_text(node, "group")?, + meta: if meta.is_truthy()? { + dag_value(&meta)? + } else { + DagValue::Object(Vec::new()) + }, + sql_deps: items(node, "sql_deps")? + .iter() + .map(|reference| { + Ok(( + enum_value(reference, "kind")?.unwrap_or_default(), + text(reference, "name")?, + )) + }) + .collect::>()?, + }) +} + +/// The discovered Python node graph. +pub(crate) fn python_graph_input(graph: &Bound<'_, PyAny>) -> PyResult { + let mut kinds_by_name: Vec<(String, String)> = Vec::new(); + for (name, node) in attribute(graph, "nodes_by_name")? + .downcast::()? + .iter() + { + kinds_by_name.push(( + name.extract()?, + enum_value(&node, "kind")?.unwrap_or_default(), + )); + } + Ok(DagPythonGraphInput { + nodes: items(graph, "nodes")? + .iter() + .map(python_node) + .collect::>()?, + dependency_edges: items(graph, "dependency_edges")? + .iter() + .map(|edge| Ok((text(edge, "upstream_name")?, text(edge, "downstream_name")?))) + .collect::>()?, + kinds_by_name, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs new file mode 100644 index 0000000000..0c7750f27d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs @@ -0,0 +1,135 @@ +//! Python values as `dataclasses.asdict` copies them and `json.dumps` would encode them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, + PyString, PyStringMethods, PyTuple, PyTypeMethods, +}; +use sqlbuild_analysis::dag_artifact::models::DagValue; +use sqlbuild_analysis::dag_artifact::types::DagKey; +use sqlbuild_core::json::models::JsonInteger; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_outputs::json_report::float_key; + +const DATACLASS_FIELDS: &str = "__dataclass_fields__"; +const SURROGATE_KEY: &str = "a DAG artifact mapping key holds a lone surrogate, which the native \ + encoder does not write"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +fn type_name(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.get_type().name()?.to_string()) +} + +/// A `str`, as UTF-16 code units when it holds lone surrogates. +pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { + if let Ok(text) = text.to_str() { + return Ok(DagValue::String(text.to_owned())); + } + let encoded: Bound<'_, PyBytes> = text + .call_method1("encode", ("utf-16-le", "surrogatepass"))? + .downcast_into::()?; + Ok(DagValue::Utf16( + encoded + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + )) +} + +fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(signed) = value.extract::() { + return Ok(DagValue::Integer(JsonInteger::from(signed))); + } + if let Ok(unsigned) = value.extract::() { + return Ok(DagValue::Integer(JsonInteger::from(unsigned))); + } + let decimal: String = value.str()?.to_str()?.to_owned(); + Ok(JsonInteger::parse(&decimal).map_or(DagValue::Null, DagValue::Integer)) +} + +/// A mapping key as `json.dumps` writes it, or its `TypeError` message. +fn key_value(key: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = key.downcast::() { + return text + .to_str() + .map(|text| Ok(text.to_owned())) + .map_err(|_| NativeCompilerError::new_err(SURROGATE_KEY)); + } + if key.is_none() { + return Ok(Ok("null".to_owned())); + } + if let Ok(flag) = key.downcast::() { + return Ok(Ok(if flag.is_true() { "true" } else { "false" }.to_owned())); + } + if key.is_instance_of::() { + return Ok(Ok(key.str()?.to_str()?.to_owned())); + } + if key.is_instance_of::() { + return float_key(key).map(Ok); + } + Ok(Err(format!( + "keys must be str, int, float, bool or None, not {}", + type_name(key)? + ))) +} + +fn dataclass_value(value: &Bound<'_, PyAny>) -> PyResult { + let fields: Bound<'_, PyAny> = value + .py() + .import(DATACLASSES_MODULE)? + .getattr("fields")? + .call1((value,))?; + let mut entries: Vec<(DagKey, DagValue)> = Vec::new(); + for field in fields.try_iter()? { + let name: Bound<'_, PyAny> = field?.getattr("name")?; + let item: Bound<'_, PyAny> = value.getattr(name.downcast::()?)?; + entries.push((key_value(&name)?, dag_value(&item)?)); + } + Ok(DagValue::Object(entries)) +} + +/// The value `asdict` copies into the artifact; refused leaves keep `json.dumps`'s message. +pub(crate) fn dag_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(DagValue::Null); + } + if value.get_type().hasattr(DATACLASS_FIELDS)? { + return dataclass_value(value); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(DagValue::Array( + value + .try_iter()? + .map(|item| dag_value(&item?)) + .collect::>()?, + )); + } + if let Ok(mapping) = value.downcast::() { + return Ok(DagValue::Object( + mapping + .iter() + .map(|(key, item)| Ok((key_value(&key)?, dag_value(&item)?))) + .collect::>()?, + )); + } + if let Ok(flag) = value.downcast::() { + return Ok(DagValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return integer_value(value); + } + if value.is_instance_of::() { + return Ok(DagValue::Float(value.extract::()?)); + } + if let Ok(text) = value.downcast::() { + return text_value(text); + } + Ok(DagValue::Unencodable(format!( + "Object of type {} is not JSON serializable", + type_name(value)? + ))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index b3783d0db2..ee70abfb5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -68,6 +68,11 @@ fn failure(error: SelectorError) -> FailureRow { } impl NativeProjectGraph { + /// The graph this object wraps. + pub(crate) fn graph(&self) -> &ProjectGraph { + &self.graph + } + /// Wrap a graph another native stage built, such as project assembly. pub(crate) fn new(graph: ProjectGraph) -> Self { Self { graph } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 70c9e1eb58..fac37883a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod contracts; +pub(crate) mod dag_artifact; pub(crate) mod discovery; pub(crate) mod graph; pub(crate) mod lineage; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 98e4ac6e68..09b575c735 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,8 +5,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, discovery, graph, lineage, model_config, project_assembly, - refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, type_system, + compiled_project, contracts, dag_artifact, discovery, graph, lineage, model_config, + project_assembly, refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, + type_system, }; use crate::bindings::models; @@ -57,5 +58,6 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { graph::registration::register(module)?; compiled_project::registration::register(module)?; compile_driver::registration::register(module)?; + dag_artifact::registration::register(module)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index de395eefcc..3bab925188 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,6 +33,11 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/dag/types.py" +reason = "The native DAG builder owns DAG behavior; this package only declares the node kinds integrations read from the artifact." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/adapters/bigquery/__init__.py" diff --git a/python-boundary.toml b/python-boundary.toml index 53b2f8ecfa..e721d87088 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 137187 +remaining_lines = 136437 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4ee7605459..ba24069e0a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -72,6 +72,7 @@ from sqlbuild.compiler.discovery.models import ( ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -782,6 +783,9 @@ def relation_key_row( kind: str, name: str, values: dict[str, object], entry: object ) -> RelationKeyRow: ... def declared_relation_keys(rows: list[RelationKeyRow]) -> dict[str, list[list[str]]]: ... +def build_dag_json( + *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None +) -> str: ... class NativeProjectGraph: @staticmethod diff --git a/src/sqlbuild/cli/commands/main/project/_dag.py b/src/sqlbuild/cli/commands/main/project/_dag.py index f770c9a6da..f42b5d8670 100644 --- a/src/sqlbuild/cli/commands/main/project/_dag.py +++ b/src/sqlbuild/cli/commands/main/project/_dag.py @@ -5,13 +5,12 @@ import json from pathlib import Path -from sqlbuild._native import build_project_graph +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.dag.main.build import build_dag_json from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -42,7 +41,7 @@ def run_dag( ), project_dir=effective_project_dir, ) - graph: ProjectGraph = build_project_graph( + graph: ProjectGraph = _native.build_project_graph( discovered_inputs=discovered_inputs, adapter=adapter, no_sql_validation=no_sql_validation, @@ -55,7 +54,7 @@ def run_dag( python_graph: PythonNodeGraph = build_discovered_python_node_graph( discovered_inputs=discovered_inputs ) - dag_json: str = build_dag_json( + dag_json: str = _native.build_dag_json( graph=graph, project_name=discovered_inputs.project_config.name, python_graph=python_graph, diff --git a/src/sqlbuild/compiler/dag/_helpers/__init__.py b/src/sqlbuild/compiler/dag/_helpers/__init__.py deleted file mode 100644 index 3d14180233..0000000000 --- a/src/sqlbuild/compiler/dag/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Static DAG helper package.""" diff --git a/src/sqlbuild/compiler/dag/_helpers/artifact.py b/src/sqlbuild/compiler/dag/_helpers/artifact.py deleted file mode 100644 index 886342867f..0000000000 --- a/src/sqlbuild/compiler/dag/_helpers/artifact.py +++ /dev/null @@ -1,610 +0,0 @@ -"""Static DAG artifact construction helpers.""" - -from __future__ import annotations - -import json -from dataclasses import asdict -from enum import StrEnum - -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.compiler.dag.constants import REQUIRED_DAG_COLLECTION_KEYS -from sqlbuild.compiler.dag.models import ( - DagArtifact, - DagCheck, - DagColumn, - DagColumnLineageRef, - DagEdge, - DagFunctionArgument, - DagNode, - DagTarget, -) -from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type -from sqlbuild.python_nodes.models import ColumnLineageRef, SqlResourceRef -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts -from sqlbuild.spec.contracts.models import ( - LoaderDestinationParts, - SchemaColumn, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.sql_values.types import SqlValueKind - -_DAG_VERSION: int = 1 - - -def build_dag_artifact( - *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None -) -> DagArtifact: - """Build a static DAG artifact from a compiled project graph.""" - - nodes: tuple[DagNode, ...] = ( - *(_build_source_node(source) for source in graph.project.sources), - *( - _build_loader_node( - project=graph.project, loader=loader, source_by_loader=_source_by_loader(graph) - ) - for loader in graph.project.loader_functions - if loader.name not in _source_by_loader(graph) - ), - *(_build_seed_node(seed) for seed in graph.project.seeds), - *(_build_function_node(function) for function in graph.project.functions), - *(_build_model_node(model) for model in graph.project.models), - *(_build_python_nodes(python_graph) if python_graph is not None else ()), - ) - return DagArtifact( - version=_DAG_VERSION, - project_name=project_name, - nodes=nodes, - edges=_build_edges(graph=graph, python_graph=python_graph), - checks=_build_checks(graph=graph, python_graph=python_graph), - ) - - -def format_dag_json(*, artifact: DagArtifact) -> str: - """Serialize the static DAG artifact to JSON.""" - - return json.dumps(_drop_none(asdict(artifact)), indent=2) - - -def _build_source_node(source: CompiledSourceView) -> DagNode: - entry: SourceEntry = source.source_entry - target: DagTarget = DagTarget( - database=entry.database, - schema=entry.schema, - name=entry.table or entry.name, - qualified_name=_qualified_name( - database=entry.database, schema=entry.schema, name=entry.table or entry.name - ), - ) - return DagNode( - id=_node_id(source.key), - kind=NodeKind.SOURCE.value, - name=source.name, - asset_key=_source_asset_key(entry), - target=target, - path=str(source.source_file.relative_path), - description=entry.description, - meta=entry.meta, - columns=tuple(_source_column(column) for column in entry.columns), - ) - - -def _build_loader_node( - *, - project: CompiledProjectView, - loader: DiscoveredLoaderFunction, - source_by_loader: dict[str, SourceEntry], -) -> DagNode: - entry: SourceEntry = source_by_loader.get(loader.name) or _loader_to_source_entry( - project=project, loader=loader - ) - target: DagTarget = DagTarget( - database=entry.database, - schema=entry.schema, - name=entry.table or entry.name, - qualified_name=_qualified_name( - database=entry.database, schema=entry.schema, name=entry.table or entry.name - ), - ) - return DagNode( - id=_loader_node_id(loader.name), - kind=NodeKind.LOADER.value, - name=loader.name, - asset_key=(loader.name,), - target=target, - path=str(loader.relative_path), - meta=entry.meta, - columns=tuple(_source_column(column) for column in entry.columns), - loader=loader.name, - ) - - -def _build_python_nodes(python_graph: PythonNodeGraph) -> tuple[DagNode, ...]: - return tuple( - _build_python_node(node) - for node in python_graph.nodes - if node.kind != PythonNodeKind.LOADER - ) - - -def _build_python_node(node: DiscoveredPythonNode) -> DagNode: - columns: tuple[DagColumn, ...] = () - column_lineage: dict[str, tuple[DagColumnLineageRef, ...]] = {} - materialization_type: str | None = None - if node.kind == PythonNodeKind.ASSET and node.asset is not None: - columns = tuple(_source_column(column) for column in node.asset.columns) - column_lineage = _python_column_lineage(node.asset.column_lineage) - materialization_type = "python_asset" - return DagNode( - id=_python_node_id(kind=node.kind, node_name=node.name), - kind=NodeKind(node.kind.value).value, - name=node.name, - asset_key=(node.kind.value, node.name), - path=str(node.relative_path), - description=node.description, - tags=node.tags, - group=node.group, - meta=node.meta or {}, - columns=columns, - column_lineage=column_lineage, - materialization_type=materialization_type, - ) - - -def _build_seed_node(seed: CompiledSeedView) -> DagNode: - return DagNode( - id=_node_id(seed.key), - kind=NodeKind.SEED.value, - name=seed.name, - asset_key=_target_asset_key(seed.destination), - target=_dag_target(seed.destination), - path=str(seed.seed_file.relative_path), - description=seed.schema_entry.description, - tags=seed.schema_entry.tags, - meta=seed.schema_entry.meta, - columns=tuple(_schema_column(column) for column in seed.schema_entry.columns), - ) - - -def _build_function_node(function: CompiledFunctionView) -> DagNode: - node_type: str = function_node_type(return_columns=function.return_columns) - return DagNode( - id=f"{node_type}:{function.name}", - kind=NodeKind(node_type).value, - name=function.name, - asset_key=_target_asset_key(function.destination), - target=_dag_target(function.destination), - path=str(function.relative_path), - sql=function.body_sql, - language=function.language.value, - tags=function.tags, - return_kind="table" if function.return_columns else "scalar", - returns=function.returns, - columns=tuple( - DagColumn(name=column.name, type=column.type) for column in function.return_columns - ), - arguments=tuple( - DagFunctionArgument(name=argument.name, type=argument.type) - for argument in function.arguments - ), - ) - - -def _build_model_node(model: CompiledModelView) -> DagNode: - return DagNode( - id=_node_id(model.key), - kind=NodeKind.MODEL.value, - name=model.name, - asset_key=_target_asset_key(model.destination), - target=_dag_target(model.destination), - path=str(model.relative_path), - description=(model.schema_entry.description if model.schema_entry is not None else None), - sql=model.authored_sql, - tags=_model_tags(model), - meta=(model.schema_entry.meta if model.schema_entry is not None else {}), - columns=_model_columns(model), - materialization_type=str(model.config.values.get("materialized", "view")), - ) - - -def _build_edges( - *, graph: ProjectGraph, python_graph: PythonNodeGraph | None = None -) -> tuple[DagEdge, ...]: - edges: list[DagEdge] = [ - DagEdge(from_id=f"{dep_kind}:{dep_name}", to_id=f"{kind}:{name}") - for (dep_kind, dep_name), (kind, name) in graph.native.edges() - ] - edges.extend(_build_loader_edges(graph)) - if python_graph is not None: - edges.extend(_build_python_edges(python_graph)) - return tuple(sorted(edges, key=lambda edge: (edge.from_id, edge.to_id))) - - -def _build_loader_edges(graph: ProjectGraph) -> tuple[DagEdge, ...]: - source_by_loader: dict[str, SourceEntry] = _source_by_loader(graph) - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in graph.project.loader_functions - } - edges: list[DagEdge] = [] - loader: DiscoveredLoaderFunction - for loader in graph.project.loader_functions: - for dependency in loader.depends_on: - dependency_name: str | None = loader_name_by_function.get(dependency) - if dependency_name is None: - continue - edges.append( - DagEdge( - from_id=_loader_or_source_node_id( - loader_name=dependency_name, source_by_loader=source_by_loader - ), - to_id=_loader_or_source_node_id( - loader_name=loader.name, source_by_loader=source_by_loader - ), - ) - ) - return tuple(edges) - - -def _loader_or_source_node_id(*, loader_name: str, source_by_loader: dict[str, SourceEntry]) -> str: - source_entry: SourceEntry | None = source_by_loader.get(loader_name) - if source_entry is not None: - return _node_id(CompiledObjectKey(CompiledResourceType.SOURCE, source_entry.name)) - return _loader_node_id(loader_name) - - -def _build_python_edges(python_graph: PythonNodeGraph) -> tuple[DagEdge, ...]: - edges: list[DagEdge] = [] - for edge in python_graph.dependency_edges: - upstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[edge.upstream_name] - downstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[edge.downstream_name] - edges.append( - DagEdge( - from_id=_python_node_id(kind=upstream_node.kind, node_name=upstream_node.name), - to_id=_python_node_id(kind=downstream_node.kind, node_name=downstream_node.name), - ) - ) - for node in python_graph.nodes: - for sql_dep in node.sql_deps: - edges.append( - DagEdge( - from_id=_sql_ref_node_id(sql_dep), - to_id=_python_node_id(kind=node.kind, node_name=node.name), - ) - ) - return tuple(edges) - - -def _build_checks( - *, graph: ProjectGraph, python_graph: PythonNodeGraph | None = None -) -> tuple[DagCheck, ...]: - checks: list[DagCheck] = [] - checks.extend(_build_sql_test_check(test) for test in graph.project.sql_tests) - checks.extend(_build_audit_check(audit) for audit in graph.project.audits) - checks.extend(_build_scenario_check(scenario) for scenario in graph.project.sql_scenarios) - if python_graph is not None: - checks.extend( - _build_python_check(node=node, python_graph=python_graph) - for node in python_graph.nodes - if node.kind == PythonNodeKind.CHECK - ) - return tuple(sorted(checks, key=lambda check: (check.kind, check.name))) - - -def _build_sql_test_check(test: CompiledSqlTest) -> DagCheck: - return DagCheck( - id=_sql_test_check_id(test), - kind=NodeKind.SQL_TEST.value, - name=test.name, - checked_asset_ids=tuple(_node_id(key) for key in test.scope_deps), - path=str(test.test_file.relative_path), - mode=test.mode.value, - meta=_sql_test_meta(test), - ) - - -def _sql_test_check_id(test: CompiledSqlTest) -> str: - if test.case_name is None: - return _node_id(test.key) - source_path: str = (test.source_path or test.test_file.relative_path).as_posix() - return f"sql_test:{source_path}:{test.block_index}:{test.case_name}" - - -def _sql_test_meta(test: CompiledSqlTest) -> dict[str, object]: - if test.case_name is None: - return {} - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test.parameter_schema - } - return { - "source_path": (test.source_path or test.test_file.relative_path).as_posix(), - "block_index": test.block_index, - "parent_name": test.parent_name, - "case_name": test.case_name, - "case_index": test.case_index, - "case_fingerprint": test.case_fingerprint, - "parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test.parameter_schema - ], - "parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": (str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value), - } - for name, value in test.parameter_values - ], - } - - -def _build_audit_check(audit: CompiledAuditView) -> DagCheck: - checked_asset_ids: tuple[str, ...] = tuple(_node_id(key) for key in audit.scope_deps) - if audit.attached_target_kind is not None and audit.attached_target_name is not None: - resource_type: CompiledResourceType = AttachedAuditTargetKind( - audit.attached_target_kind - ).resource_type - checked_asset_ids = ( - _node_id(CompiledObjectKey(resource_type, audit.attached_target_name)), - ) - return DagCheck( - id=_audit_check_id(audit), - kind=NodeKind.AUDIT.value, - name=audit.name, - checked_asset_ids=checked_asset_ids, - path=str(audit.audit_file.relative_path), - severity=audit.severity, - attachment_kind=( - audit.attached_target_kind.value if audit.attached_target_kind is not None else None - ), - attached_target_name=audit.attached_target_name, - attached_column_name=audit.attached_column_name, - ) - - -def _audit_check_id(audit: CompiledAuditView) -> str: - parts: tuple[str | None, ...] = ( - _node_id(audit.key), - audit.attached_target_kind.value if audit.attached_target_kind is not None else None, - audit.attached_target_name, - audit.attached_column_name, - ) - return ":".join(part for part in parts if part) - - -def _build_scenario_check(scenario: CompiledSqlScenario) -> DagCheck: - target_keys: tuple[CompiledObjectKey, ...] = tuple( - CompiledObjectKey(CompiledResourceType.MODEL, name) - for name in scenario.expected_model_names - ) - graph_asset_ids: tuple[str, ...] = tuple( - _node_id(CompiledObjectKey(CompiledResourceType.MODEL, cte.name)) - for cte in scenario.authored_ctes - ) - return DagCheck( - id=_node_id(scenario.key), - kind=NodeKind.SCENARIO.value, - name=scenario.name, - checked_asset_ids=tuple(_node_id(key) for key in target_keys), - path=str(scenario.scenario_file.relative_path), - assertion_names=scenario.assertion_names, - expected_model_names=scenario.expected_model_names, - graph_asset_ids=graph_asset_ids, - fixture_refs=( - *scenario.source_fixture_names, - *scenario.ref_fixture_names, - *scenario.seed_fixture_names, - *scenario.dbt_ref_fixture_names, - ), - ) - - -def _build_python_check(*, node: DiscoveredPythonNode, python_graph: PythonNodeGraph) -> DagCheck: - checked_asset_ids: tuple[str, ...] = tuple( - _python_node_id( - kind=python_graph.nodes_by_name[edge.upstream_name].kind, - node_name=edge.upstream_name, - ) - for edge in python_graph.dependency_edges - if edge.downstream_name == node.name - ) + tuple(_sql_ref_node_id(sql_dep) for sql_dep in node.sql_deps) - return DagCheck( - id=_python_node_id(kind=node.kind, node_name=node.name), - kind=NodeKind.PYTHON_CHECK.value, - name=node.name, - checked_asset_ids=checked_asset_ids, - path=str(node.relative_path), - description=node.description, - severity=(AuditSeverity(node.check.severity.value) if node.check is not None else None), - tags=node.tags, - group=node.group, - meta=node.meta or {}, - ) - - -def _dag_target(target: CompiledRelationLocation) -> DagTarget: - return DagTarget( - database=target.database, - schema=target.schema, - name=target.name, - qualified_name=target.qualified_name, - logical_database=target.logical_database, - logical_schema=target.logical_schema, - ) - - -def _target_asset_key(target: CompiledRelationLocation) -> tuple[str, ...]: - return tuple( - part - for part in ( - target.logical_database or target.database, - target.logical_schema or target.schema, - target.name, - ) - if part - ) - - -def _source_asset_key(entry: SourceEntry) -> tuple[str, ...]: - parts: tuple[str | None, ...] = (entry.database, entry.schema, entry.table or entry.name) - return tuple(part for part in parts if part) or (entry.name,) - - -def _node_id(key: CompiledObjectKey) -> str: - return f"{key.resource_type}:{key.name}" - - -def _loader_node_id(loader_name: str) -> str: - return f"loader:{loader_name}" - - -def _python_node_id(*, kind: PythonNodeKind, node_name: str) -> str: - if kind == PythonNodeKind.LOADER: - return _loader_node_id(node_name) - return f"{kind.value}:{node_name}" - - -def _sql_ref_node_id(ref: SqlResourceRef) -> str: - return _node_id(sql_ref_key(ref)) - - -def _source_by_loader(graph: ProjectGraph) -> dict[str, SourceEntry]: - return { - source.source_entry.loader: source.source_entry - for source in graph.project.sources - if source.source_entry.loader is not None - } - - -def _loader_to_source_entry( - *, project: CompiledProjectView, loader: DiscoveredLoaderFunction -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=project.effective_target_database, - default_schema=project.effective_target_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _schema_column(column: SchemaColumn) -> DagColumn: - return DagColumn( - name=column.name, - type=column.type, - nullable=column.nullable, - description=column.description, - meta=column.meta, - ) - - -def _source_column(column: SourceColumnEntry) -> DagColumn: - return DagColumn( - name=column.name, - type=column.type, - nullable=column.nullable, - description=column.description, - meta=column.meta, - ) - - -def _model_columns(model: CompiledModelView) -> tuple[DagColumn, ...]: - if model.schema_entry is not None and model.schema_entry.columns: - return tuple(_schema_column(column) for column in model.schema_entry.columns) - if model.inferred_columns is not None: - return tuple( - DagColumn(name=column.name, type=column.type) for column in model.inferred_columns - ) - return () - - -def _model_tags(model: CompiledModelView) -> tuple[str, ...]: - schema_tags: tuple[str, ...] = () - if model.schema_entry is not None: - schema_tags = model.schema_entry.tags - config_tags: object = model.config.values.get("tags", ()) - if isinstance(config_tags, (list, tuple)): - return tuple(dict.fromkeys((*schema_tags, *(str(tag) for tag in config_tags)))) - return schema_tags - - -def _python_column_lineage( - lineage: dict[str, tuple[ColumnLineageRef, ...]] | None, -) -> dict[str, tuple[DagColumnLineageRef, ...]]: - if lineage is None: - return {} - result: dict[str, tuple[DagColumnLineageRef, ...]] = {} - for column, refs in lineage.items(): - converted_refs: list[DagColumnLineageRef] = [] - for ref in refs: - converted_refs.append(DagColumnLineageRef(node=ref.node, column=ref.column)) - result[column] = tuple(converted_refs) - return result - - -def _qualified_name(*, database: str | None, schema: str | None, name: str) -> str: - return ".".join(part for part in (database, schema, name) if part) - - -def _drop_none(value: object) -> object: - if isinstance(value, StrEnum): - return value.value - if isinstance(value, dict): - result: dict[object, object] = {} - for key, item in value.items(): - if item is None: - continue - cleaned: object = _drop_none(item) - if _is_empty_optional_field(key=key, value=cleaned): - continue - result[key] = cleaned - return result - if isinstance(value, list): - return [_drop_none(item) for item in value] - if isinstance(value, tuple): - return [_drop_none(item) for item in value] - return value - - -def _is_empty_optional_field(*, key: object, value: object) -> bool: - if key in REQUIRED_DAG_COLLECTION_KEYS: - return False - return value in ({}, []) diff --git a/src/sqlbuild/compiler/dag/constants.py b/src/sqlbuild/compiler/dag/constants.py deleted file mode 100644 index 6e85088d9d..0000000000 --- a/src/sqlbuild/compiler/dag/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Static DAG artifact constants.""" - -REQUIRED_DAG_COLLECTION_KEYS: frozenset[str] = frozenset({"nodes", "edges", "checks"}) diff --git a/src/sqlbuild/compiler/dag/main/build.py b/src/sqlbuild/compiler/dag/main/build.py deleted file mode 100644 index 38e1f8f0b9..0000000000 --- a/src/sqlbuild/compiler/dag/main/build.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Build the static SQLBuild DAG JSON artifact.""" - -from __future__ import annotations - -from sqlbuild.compiler.dag._helpers.artifact import build_dag_artifact, format_dag_json -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - -def build_dag_json( - *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None -) -> str: - """Build and serialize a static SQLBuild DAG artifact.""" - - return format_dag_json( - artifact=build_dag_artifact( - graph=graph, - project_name=project_name, - python_graph=python_graph, - ) - ) diff --git a/src/sqlbuild/compiler/dag/models.py b/src/sqlbuild/compiler/dag/models.py deleted file mode 100644 index 65c74c8223..0000000000 --- a/src/sqlbuild/compiler/dag/models.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Static DAG artifact models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from sqlbuild.compiler.auditing.types import AuditSeverity - - -@dataclass(frozen=True) -class DagTarget: - """Warehouse identity for one SQLBuild DAG node.""" - - database: str | None = None - schema: str | None = None - name: str | None = None - qualified_name: str | None = None - logical_database: str | None = None - logical_schema: str | None = None - - -@dataclass(frozen=True) -class DagColumn: - """Column metadata attached to one DAG node.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DagColumnLineageRef: - """Declared upstream column reference for one DAG node column.""" - - node: str - column: str - - -@dataclass(frozen=True) -class DagFunctionArgument: - """Function argument metadata attached to one function node.""" - - name: str - type: str - - -@dataclass(frozen=True) -class DagNode: - """One asset-like node in the static SQLBuild DAG.""" - - id: str - kind: str - name: str - asset_key: tuple[str, ...] - target: DagTarget | None = None - path: str | None = None - description: str | None = None - sql: str | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[DagColumn, ...] = field(default_factory=tuple) - column_lineage: dict[str, tuple[DagColumnLineageRef, ...]] = field(default_factory=dict) - materialization_type: str | None = None - language: str | None = None - return_kind: str | None = None - returns: str | None = None - loader: str | None = None - arguments: tuple[DagFunctionArgument, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DagEdge: - """One static asset dependency edge.""" - - from_id: str - to_id: str - - -@dataclass(frozen=True) -class DagCheck: - """One check definition attached to one or more DAG assets.""" - - id: str - kind: str - name: str - checked_asset_ids: tuple[str, ...] - path: str | None = None - description: str | None = None - severity: AuditSeverity | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - meta: dict[str, object] = field(default_factory=dict) - attachment_kind: str | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - mode: str | None = None - assertion_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - graph_asset_ids: tuple[str, ...] = field(default_factory=tuple) - fixture_refs: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DagArtifact: - """Complete static SQLBuild DAG artifact.""" - - version: int - project_name: str - nodes: tuple[DagNode, ...] - edges: tuple[DagEdge, ...] - checks: tuple[DagCheck, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index 249dd2d798..c98268963a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -41,7 +41,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor ]) Path(sys.argv[2]).write_text(json.dumps({ "aggregate_models": "sqlbuild.cli.commands.models.runtime" in sys.modules, - "dag": "sqlbuild.compiler.dag.main.build" in sys.modules, "manifest": "sqlbuild.compiler.manifest.main.build" in sys.modules, }), encoding="utf-8") raise SystemExit(exit_code) @@ -59,7 +58,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor assert summary["models"] == test_case.expected_count assert json.loads(import_status_path.read_text(encoding="utf-8")) == { "aggregate_models": False, - "dag": False, "manifest": False, } diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 5226a56150..b8815d79de 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -8,6 +8,7 @@ import pytest +from sqlbuild._native import build_dag_json from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -15,7 +16,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.dag.main.build import build_dag_json from sqlbuild.compiler.dag.types import NodeKind from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs From e42d516aa719ceef4f4e23b11d1622d36fd7c8d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:47:50 +0100 Subject: [PATCH 02/26] refactor: build compile inputs in the native compile driver --- .../_helpers/compile_driver/inputs.rs | 450 ++++++++++++++++++ .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/project.rs | 2 +- .../_helpers/compile_driver/python_calls.rs | 11 +- .../_helpers/compile_driver/registration.rs | 5 +- python-boundary.toml | 2 +- scripts/fensu_policy/constants.py | 1 - .../rules/canonical_operations.py | 2 +- src/sqlbuild/_native.pyi | 14 + .../compile/main/_build_compile_inputs.py | 252 ---------- .../scopes/main/load_or_build_scope_index.py | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 8 +- .../compiler/analysis_session/helpers.py | 2 +- .../src/sqlbuild/compiler/compile/helpers.py | 2 +- .../compiler/compiled_project/helpers.py | 2 +- .../test_native_parameter_boundaries.py | 2 +- .../compiler/explicit_references/helpers.py | 2 +- .../sqlbuild/compiler/model_config/helpers.py | 2 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 2 +- .../compiler/project_assembly/helpers.py | 2 +- .../sqlbuild/integrations/dbt/test_graph.py | 2 +- .../integrations/dbt/test_manifest.py | 2 +- .../rules/test_canonical_operations.py | 2 +- .../compile/_helpers/config/helpers.py | 2 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../compile/_helpers/test_assembly.py | 2 +- .../compile/_helpers/test_audit_factories.py | 2 +- .../_helpers/test_declaration_scopes.py | 2 +- .../compile/_helpers/test_declarations.py | 2 +- .../compile/_helpers/test_model_schemas.py | 2 +- .../compiler/compile/test_cursor_start.py | 2 +- .../sqlbuild/compiler/compile/test_main.py | 2 +- .../compile/test_resolved_connection.py | 2 +- .../sqlbuild/compiler/dag/main/test_build.py | 3 +- 34 files changed, 508 insertions(+), 287 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs delete mode 100644 src/sqlbuild/compiler/compile/main/_build_compile_inputs.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs new file mode 100644 index 0000000000..fc76e5b89e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -0,0 +1,450 @@ +//! The first attached compile input snapshot: target, vars, macros, scopes, models and resources. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; + +const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; +const ATTACHMENT_MODULE: &str = "sqlbuild.compiler.compile._helpers.attachment.core"; +const DECLARATION_SCOPE_MODULE: &str = + "sqlbuild.compiler.compile._helpers.attachment.declaration_scope"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const ATTACH_CHECKS_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_checks"; +const ATTACH_RESOURCES_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_resources"; +const INPUTS_VIEW_MODULE: &str = "sqlbuild.compiler.compiled_project.main._compile_inputs_view"; +const INPUTS_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.inputs_with"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const NATIVE_MODULE: &str = "sqlbuild._native"; + +const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; +const OPTIONAL: [&str; 8] = [ + "selected_target", + "cli_vars", + "run_id", + "no_sql_validation", + "defer_model_sql_validation", + "external_sql_reference_resolver", + "resolved_connection", + "no_cache", +]; + +/// The effective target, vars, settings and macro context every later stage shares. +struct TargetContext<'py> { + target_name: Bound<'py, PyAny>, + target: Bound<'py, PyAny>, + compile_cache_dir: Bound<'py, PyAny>, + vars: Bound<'py, PyAny>, + settings: Bound<'py, PyAny>, + macro_context: Bound<'py, PyAny>, +} + +/// The attached models with the native project that owns them and their declarations. +struct ModelBuild<'py> { + inputs: Bound<'py, PyAny>, + native_project: Bound<'py, PyAny>, + enums: Bound<'py, PyAny>, + constants: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, +} + +/// Seeds, functions and sources, and `(tests, scenarios, audits, diagnostics)` as attached. +struct AttachedResources<'py> { + seeds: Bound<'py, PyAny>, + functions: Bound<'py, PyAny>, + sources: Bound<'py, PyAny>, + checks: Bound<'py, PyAny>, +} + +/// Attach discovered metadata into the first compile input snapshot. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_compile_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_compile_inputs", options, (&REQUIRED, &OPTIONAL))?; + let collector: Bound<'py, PyAny> = + imported(py, COLLECTOR_MODULE, "collect_compile_diagnostics")?.call0()?; + let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + within(&collector, |violations| { + Ok((compile_inputs(py, &arguments)?, violations.clone())) + })?; + let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + if !collected.is_truthy()? { + return Ok(built); + } + let diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + built + .getattr("diagnostics")? + .try_iter()? + .chain(collected.try_iter()?) + .collect::>>()?, + )?; + call_with( + &imported(py, INPUTS_WITH_MODULE, "inputs_with")?, + &[("inputs", built), ("diagnostics", diagnostics.into_any())], + ) +} + +/// The undecorated `build_compile_inputs`. +fn compile_inputs<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { + let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; + let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; + let run_id: Bound<'py, PyAny> = call_with( + &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, + &[("selected_run_id", arguments.value("run_id")?)], + )?; + let loaded_macros: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "load_project_macros")? + .call1((discovered.getattr("macro_files")?,))?; + let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; + let declaration_scope: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATION_SCOPE_MODULE, "build_declaration_scope")?, + &[ + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros), + ("sql_lexical_syntax", syntax.clone()), + ("compile_cache_dir", target.compile_cache_dir.clone()), + ], + )?; + let value_renderer: Bound<'py, PyAny> = adapter_context.getattr("value_renderer")?; + let model_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "ModelInputBuildContext")?, + &[ + ("effective_vars", target.vars.clone()), + ("effective_settings", target.settings.clone()), + ("target_config", target.target.clone()), + ("effective_target_name", target.target_name.clone()), + ("run_id", run_id.clone()), + ("macro_context", target.macro_context.clone()), + ("loaded_macros", declaration_scope.getattr("loaded_macros")?), + ("value_renderer", value_renderer.clone()), + ( + "collection_rendering", + adapter_context.getattr("collection_rendering")?, + ), + ("sql_lexical_syntax", syntax.clone()), + ( + "declaration_resolver", + declaration_scope.getattr("resolver")?, + ), + ], + )?; + let models: ModelBuild<'py> = model_build(arguments, &discovered, &model_context)?; + let attached: AttachedResources<'py> = attached_resources( + arguments, + &adapter_context, + (&discovered, &declaration_scope), + &models, + )?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("discovered_inputs", discovered.clone()), + ("sql_lexical_syntax", syntax), + ("run_id", run_id), + ("effective_target_name", target.target_name.clone()), + ("effective_target", target.target.clone()), + ("compile_cache_dir", target.compile_cache_dir.clone()), + ( + "effective_connection", + effective_connection(arguments, &discovered, &target)?, + ), + ("effective_settings", target.settings.clone()), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ("effective_vars", target.vars.clone()), + ("macro_context", target.macro_context.clone()), + ("loaded_macros", declaration_scope.getattr("loaded_macros")?), + ("public_enums", models.enums.clone()), + ("public_constants", models.constants.clone()), + ("model_inputs", models.inputs.clone()), + ("seed_inputs", attached.seeds), + ("source_inputs", attached.sources), + ("sql_function_inputs", attached.functions), + ("test_inputs", attached.checks.get_item(0)?), + ("scenario_inputs", attached.checks.get_item(1)?), + ("audit_inputs", attached.checks.get_item(2)?), + ("diagnostics", attached.checks.get_item(3)?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ("scope_index", declaration_scope.getattr("index")?), + ("declaration_scope", declaration_scope), + ], + )?; + call_with( + &imported(py, INPUTS_VIEW_MODULE, "compile_inputs_view")?, + &[ + ("fields", fields.into_any()), + ("native_project", models.native_project), + ], + ) +} + +/// The selected target with the macro call store, and the effective vars, settings and macro context. +fn target_context<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + adapter_context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let selected: Bound<'py, PyAny> = call_with( + &imported( + py, + CALL_STORE_MODULE, + "target_context_with_macro_call_store", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", arguments.value("selected_target")?), + ("no_cache", arguments.value_or_false("no_cache")?), + ], + )?; + let target_name: Bound<'py, PyAny> = selected.get_item(0)?; + let target: Bound<'py, PyAny> = selected.get_item(1)?; + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { + PyDict::new(py).into_any() + } else { + cli_vars + }; + let vars: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_effective_vars")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("target_config", target.clone()), + ("cli_vars", cli_vars), + ], + )?; + let settings: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_effective_settings")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ], + )?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config), + ], + )?; + let macro_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ("adapter_name", adapter_name), + ("sql_analysis_enabled", settings.getattr("sql_analysis")?), + ("target_name", target_name.clone()), + ("vars", vars.clone()), + ( + "_value_renderer", + adapter_context.getattr("value_renderer")?, + ), + ( + "_collection_rendering", + adapter_context.getattr("collection_rendering")?, + ), + ( + "_enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + Ok(TargetContext { + target_name, + target, + compile_cache_dir: selected.get_item(2)?, + vars, + settings, + macro_context, + }) +} + +/// `_build_models_with_declarations`: attach every model with the public declaration indexes. +fn model_build<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let native: Bound<'py, PyModule> = py.import(NATIVE_MODULE)?; + let declarations: Bound<'py, PyAny> = native + .getattr("public_declaration_indexes")? + .call1((discovered,))?; + let enums: Bound<'py, PyAny> = declarations.get_item(0)?; + let constants: Bound<'py, PyAny> = declarations.get_item(1)?; + let schemas: Bound<'py, PyAny> = native + .getattr("public_model_schema_index")? + .call1((discovered,))?; + let native_project: Bound<'py, PyAny> = native.getattr("NativeCompiledProject")?.call0()?; + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let model_context: Bound<'py, PyAny> = replace.call( + (context,), + Some(&keywords( + py, + &[ + ("public_enums", enums.clone()), + ("public_constants", constants.clone()), + ("public_model_schemas", schemas), + ], + )?), + )?; + let inputs: Bound<'py, PyAny> = call_with( + &imported(py, ATTACHMENT_MODULE, "build_model_inputs")?, + &[ + ("discovered_inputs", discovered.clone()), + ("context", model_context), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ( + "defer_model_sql_validation", + arguments.value_or_false("defer_model_sql_validation")?, + ), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ("native_project", native_project.clone()), + ], + )?; + let context: Bound<'py, PyAny> = replace.call( + (context,), + Some(&keywords( + py, + &[ + ("public_enums", enums.clone()), + ("public_constants", constants.clone()), + ], + )?), + )?; + Ok(ModelBuild { + inputs, + native_project, + enums, + constants, + context, + }) +} + +/// The seeds, functions, sources and checks attached to the models. +fn attached_resources<'py>( + arguments: &KeywordArguments<'py>, + adapter_context: &Bound<'py, PyAny>, + (discovered, declaration_scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + models: &ModelBuild<'py>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let context: &Bound<'py, PyAny> = &models.context; + let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let resources: Bound<'py, PyAny> = call_with( + &imported(py, ATTACH_RESOURCES_MODULE, "attach_resources")?, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings.clone(), + context.getattr("target_config")?, + context.getattr("macro_context")?.getattr("adapter_name")?, + ], + )? + .into_any(), + ), + ( + "no_sql_validation", + arguments.value_or_false("no_sql_validation")?, + ), + ( + "python_functions_inherit_default_namespace", + adapter_context.getattr("python_functions_inherit_default_namespace")?, + ), + ], + )?; + let sources: Bound<'py, PyAny> = resources.get_item(2)?; + let checks: Bound<'py, PyAny> = call_with( + &imported(py, ATTACH_CHECKS_MODULE, "attach_checks")?, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings, + adapter_context.getattr("value_renderer")?, + context.getattr("sql_lexical_syntax")?, + ], + )? + .into_any(), + ), + ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ("source_inputs", sources.clone()), + ], + )?; + Ok(AttachedResources { + seeds: resources.get_item(0)?, + functions: resources.get_item(1)?, + sources, + checks, + }) +} + +/// The resolved connection when given, otherwise the configured one for the target. +fn effective_connection<'py>( + arguments: &KeywordArguments<'py>, + discovered: &Bound<'py, PyAny>, + target: &TargetContext<'py>, +) -> PyResult> { + let resolved: Bound<'py, PyAny> = arguments.value("resolved_connection")?; + if !resolved.is_none() { + return Ok(resolved); + } + call_with( + &imported( + discovered.py(), + ATTACHMENT_MODULE, + "build_effective_connection", + )?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_config", target.target.clone()), + ("effective_vars", target.vars.clone()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_compile_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 2aa258368d..348234e24d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod analysis; pub(crate) mod command; +pub(crate) mod inputs; pub(crate) mod phases; pub(crate) mod project; pub(crate) mod python_calls; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 499b76e4b3..cb6aaa799a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; -const BUILD_INPUTS_MODULE: &str = "sqlbuild.compiler.compile.main._build_compile_inputs"; +const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild.compiler.compile.main._assemble_project"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs index 84ca7fe197..22c3e7f7b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; /// `module.name`, imported. pub(crate) fn imported<'py>( @@ -123,6 +123,15 @@ impl<'py> KeywordArguments<'py> { self.value(name)?.is_truthy() } + /// The argument `name`, or `False` when it was not passed. + pub(crate) fn value_or_false(&self, name: &str) -> PyResult> { + let py: Python<'py> = self.values.py(); + Ok(self + .values + .get_item(name)? + .unwrap_or_else(|| PyBool::new(py, false).to_owned().into_any())) + } + /// Whether `name` was passed. pub(crate) fn passed(&self, name: &str) -> PyResult { self.values.contains(name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index fed37dac7a..c5fd1c60d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_driver::{analysis, command, project}; +use crate::bindings::_helpers::compile_driver::{analysis, command, inputs, project}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; command::register(module)?; - project::register(module) + project::register(module)?; + inputs::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index e721d87088..3e2161247f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 136437 +remaining_lines = 136185 [[stays]] path = "executor" diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index b3fe3bae74..c0eddf8ab9 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -19,7 +19,6 @@ ) ALLOWED_MACRO_LOAD_PATHS: frozenset[str] = frozenset( { - "src/sqlbuild/compiler/compile/main/_build_compile_inputs.py", "src/sqlbuild/compiler/compile/main/sql_expansion_context.py", "src/sqlbuild/compiler/compile/_helpers/render/macros.py", } diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 75f68074e5..69bc123b1f 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -170,7 +170,7 @@ def source_freshness_sql_ownership(*, module: object, ctx: RuleContext) -> list[ code="XSB062", family=Family.CUSTOM, slug="single-macro-load-site", - message="project macros must be loaded once in build_compile_inputs", + message="project macros must be loaded once by the native compile driver", remediation="Pass loaded_macros down instead of calling load_project_macros again.", ) def single_macro_load_site(*, module: object, ctx: RuleContext) -> list[Fault]: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ba24069e0a..41769857bc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13,6 +13,7 @@ from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, CompileAnalysisSelection, CompileAuditInput, CompiledAudit, @@ -971,6 +972,19 @@ def build_project_graph( on_progress: Callable[[str], None] | None = None, no_cache: bool = False, ) -> ProjectGraph: ... +def build_compile_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter_context: CompileAdapterContext, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, + run_id: str | None = None, + no_sql_validation: bool = False, + defer_model_sql_validation: bool = False, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + resolved_connection: dict[str, object] | None = None, + no_cache: bool = False, +) -> CompileInputsView: ... def run_compile_command( request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None ) -> int: ... diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py deleted file mode 100644 index d5e691c979..0000000000 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Build the first attached compile input snapshot from discovered inputs.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment.core import ( - build_effective_connection, - build_effective_settings, - build_effective_vars, - build_model_inputs, -) -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - with_collected_compile_diagnostics, -) -from sqlbuild.compiler.compile._helpers.macro_bridge.call_store import ( - target_context_with_macro_call_store, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id -from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, - DeclarationResolutionContext, - DeclarationScopeBuild, - LoadedMacro, - MacroContext, - ModelInputBuildContext, - ModelInputScopeBuild, -) -from sqlbuild.compiler.compile.types import ( - AuditInput, - CompileInputsView, - FunctionInput, - ModelInput, - SeedInput, - SourceInput, -) -from sqlbuild.compiler.compiled_project.main._attach_checks import attach_checks -from sqlbuild.compiler.compiled_project.main._attach_resources import attach_resources -from sqlbuild.compiler.compiled_project.main._compile_inputs_view import compile_inputs_view -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - EnumDeclaration, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig - - -@with_collected_compile_diagnostics -def build_compile_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter_context: CompileAdapterContext, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, - run_id: str | None = None, - no_sql_validation: bool = False, - defer_model_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - resolved_connection: dict[str, object] | None = None, - no_cache: bool = False, -) -> CompileInputsView: - """Attach discovered metadata into the first compile input snapshot.""" - - effective_target_name: str | None - effective_target: TargetConfig | None - compile_cache_dir: Path | None - effective_target_name, effective_target, compile_cache_dir = ( - target_context_with_macro_call_store( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - no_cache=no_cache, - ) - ) - - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=effective_target, - cli_vars={} if cli_vars is None else cli_vars, - ) - effective_settings: SettingsConfig = build_effective_settings( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - macro_context: MacroContext = MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - sql_analysis_enabled=effective_settings.sql_analysis, - target_name=effective_target_name, - vars=effective_vars, - _value_renderer=adapter_context.value_renderer, - _collection_rendering=adapter_context.collection_rendering, - _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), - ) - resolved_run_id: str = resolve_run_id(selected_run_id=run_id) - loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) - declaration_scope: DeclarationScopeBuild = build_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - model_context: ModelInputBuildContext = ModelInputBuildContext( - effective_vars=effective_vars, - effective_settings=effective_settings, - target_config=effective_target, - effective_target_name=effective_target_name, - run_id=resolved_run_id, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - value_renderer=adapter_context.value_renderer, - collection_rendering=adapter_context.collection_rendering, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - declaration_resolver=declaration_scope.resolver, - ) - model_build: ModelInputScopeBuild = _build_models_with_declarations( - discovered_inputs=discovered_inputs, - context=model_context, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - model_context = model_build.context - seed_inputs: tuple[SeedInput, ...] - sql_function_inputs: tuple[FunctionInput, ...] - source_inputs: tuple[SourceInput, ...] - seed_inputs, sql_function_inputs, source_inputs = attach_resources( - discovered_inputs=discovered_inputs, - native_project=model_build.native_project, - settings=( - model_context.effective_settings, - model_context.target_config, - model_context.macro_context.adapter_name, - ), - no_sql_validation=no_sql_validation, - python_functions_inherit_default_namespace=( - adapter_context.python_functions_inherit_default_namespace - ), - ) - test_inputs: tuple[CompileSqlTestInput, ...] - scenario_inputs: tuple[CompileSqlScenarioInput, ...] - audit_inputs: tuple[AuditInput, ...] - diagnostics: tuple[CompilerDiagnostic, ...] - test_inputs, scenario_inputs, audit_inputs, diagnostics = attach_checks( - discovered_inputs=discovered_inputs, - native_project=model_build.native_project, - settings=( - model_context.effective_settings, - adapter_context.value_renderer, - model_context.sql_lexical_syntax, - ), - scan_cache=declaration_scope.sql_test_scans, - source_inputs=source_inputs, - ) - fields: dict[str, Any] = { - "project_config": discovered_inputs.project_config, - "local_config": discovered_inputs.local_config, - "discovered_inputs": discovered_inputs, - "sql_lexical_syntax": adapter_context.sql_lexical_syntax, - "run_id": resolved_run_id, - "effective_target_name": effective_target_name, - "effective_target": effective_target, - "compile_cache_dir": compile_cache_dir, - "effective_connection": ( - resolved_connection - if resolved_connection is not None - else build_effective_connection( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=effective_target, - effective_vars=effective_vars, - ) - ), - "effective_settings": effective_settings, - "no_sql_validation": no_sql_validation, - "effective_vars": effective_vars, - "macro_context": macro_context, - "loaded_macros": declaration_scope.loaded_macros, - "public_enums": model_build.declarations.enums, - "public_constants": model_build.declarations.constants, - "model_inputs": model_build.inputs, - "seed_inputs": seed_inputs, - "source_inputs": source_inputs, - "sql_function_inputs": sql_function_inputs, - "test_inputs": test_inputs, - "scenario_inputs": scenario_inputs, - "audit_inputs": audit_inputs, - "diagnostics": diagnostics, - "external_sql_reference_resolver": external_sql_reference_resolver, - "scope_index": declaration_scope.index, - "declaration_scope": declaration_scope, - } - return compile_inputs_view(fields=fields, native_project=model_build.native_project) - - -def _build_models_with_declarations( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool, - defer_model_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> ModelInputScopeBuild: - public_enums: dict[str, EnumDeclaration] - public_constants: dict[str, ConstantDeclaration] - public_enums, public_constants = _native.public_declaration_indexes(discovered_inputs) - public_model_schemas: dict[str, ModelSchemaDeclaration] = _native.public_model_schema_index( - discovered_inputs - ) - declarations: DeclarationResolutionContext = DeclarationResolutionContext( - enums=public_enums, - constants=public_constants, - ) - native_project: _native.NativeCompiledProject = _native.NativeCompiledProject() - model_inputs: tuple[ModelInput, ...] = build_model_inputs( - discovered_inputs=discovered_inputs, - context=replace( - context, - public_enums=public_enums, - public_constants=public_constants, - public_model_schemas=public_model_schemas, - ), - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - native_project=native_project, - ) - return ModelInputScopeBuild( - inputs=model_inputs, - native_project=native_project, - declarations=declarations, - context=replace( - context, - public_enums=declarations.enums, - public_constants=declarations.constants, - ), - ) diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index 587de54257..2228310de3 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -5,9 +5,9 @@ from functools import partial from pathlib import Path +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 3edd452745..2bfb3c12c1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -13,7 +13,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module -import sqlbuild.compiler.compile.main._build_compile_inputs as compile_inputs_module +import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rules as rules_module @@ -810,7 +810,7 @@ def counting_build( compile_cache_dir=compile_cache_dir, ) - monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -877,9 +877,9 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo macro_loads.append(loaded) return loaded - monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(compile_inputs_module, "load_project_macros", recording_load) + monkeypatch.setattr(macros_module, "load_project_macros", recording_load) monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 98dc669867..b5df03e7c4 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,6 +16,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main @@ -52,7 +53,6 @@ ) from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, CompactLineageFacts, diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 8e7d795f39..13908ab46e 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -14,10 +14,10 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, LoadedMacro, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index de07a054b3..4d464da3a7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -13,6 +13,7 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds @@ -20,7 +21,6 @@ from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileModelConfig, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index fb7b8a8336..383b963d48 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -6,11 +6,11 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 4765de1b71..b2f58fbc6a 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -7,9 +7,9 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import build_compile_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompilerDiagnostic, diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 8800958555..b65f9562fb 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -12,10 +12,10 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 0c9ead54d6..0849576df7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,11 +8,11 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledLineageColumnFact, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 38cdc4e4bc..e7f6608e56 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,11 +13,11 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly import project as project_module from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 1d20d66dff..fca78a5ce8 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,8 +5,8 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 2d80ea5cec..7aa3d7a5b4 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index c007623560..4a091b1aab 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -291,7 +291,7 @@ def test_given_freshness_code_when_checking_sql_ownership_then_matches_contract( ), CustomRuleTestCase( description="compile input macro load passes", - path="src/sqlbuild/compiler/compile/main/_build_compile_inputs.py", + path="src/sqlbuild/compiler/compile/main/sql_expansion_context.py", source="loaded = load_project_macros(())\n", expected_fault_count=0, ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 7af83aa85b..2369707552 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -3,12 +3,12 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile._helpers.attachment.model_config import ( build_native_model_config, native_model_config_session, native_validation_error, ) -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index a094251b3a..2d1ebf2fb0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -4,6 +4,7 @@ from types import MappingProxyType from typing import cast +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -23,7 +24,6 @@ load_project_macros, ) from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledDirectLogicSqlTestPayload, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 41e1f8d240..f6e1775f28 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index b6f7ddbcba..df2f5f8ca9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 40097e240e..7e8c11d358 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index ce03a1beb0..08a1278e14 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index e4eec95179..741e2e9f9d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import build_compile_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index c33c1c63cb..281ba89744 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 0e6cc220e6..c422832da6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index 1117887f2f..304bff465b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild._native import build_compile_inputs from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index b8815d79de..b06ee11b11 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -8,9 +8,8 @@ import pytest -from sqlbuild._native import build_dag_json +from sqlbuild._native import build_compile_inputs, build_dag_json from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) From c8fb2885002564e4e507b816150f514309298b3d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:38:37 +0100 Subject: [PATCH 03/26] refactor: read literal context SQL from Python functions natively Parse the dedented function source with the ruff parser under CPython's tree shape and ast.walk order, and delete the Python ast scan. Python keeps inspect.getsourcelines and textwrap.dedent. --- .../src/cpython_tree/_helpers/children.rs | 388 ++++++++++++++++++ .../src/cpython_tree/_helpers/joined.rs | 64 +++ .../src/cpython_tree/_helpers/lines.rs | 10 + .../src/cpython_tree/_helpers/mod.rs | 3 + .../src/cpython_tree/constants.rs | 4 + .../src/cpython_tree/main/joined_values.rs | 29 ++ .../src/cpython_tree/main/mod.rs | 4 + .../cpython_tree/main/parse_cpython_module.rs | 28 ++ .../src/cpython_tree/main/source_position.rs | 17 + .../cpython_tree/main/walk_breadth_first.rs | 17 + .../sqlbuild-pysource/src/cpython_tree/mod.rs | 7 + .../src/cpython_tree/models.rs | 52 +++ .../src/cpython_tree/tests.rs | 4 + .../tests/test_joined_constants.rs | 70 ++++ .../src/cpython_tree/tests/test_types.rs | 5 + crates/sqlbuild-pysource/src/lib.rs | 2 + .../src/literal_sql/_helpers/arguments.rs | 67 +++ .../src/literal_sql/_helpers/mod.rs | 1 + .../src/literal_sql/constants.rs | 10 + .../src/literal_sql/main/literal_sql_calls.rs | 61 +++ .../src/literal_sql/main/mod.rs | 1 + .../sqlbuild-pysource/src/literal_sql/mod.rs | 7 + .../src/literal_sql/models.rs | 19 + .../src/literal_sql/tests.rs | 4 + .../tests/test_literal_sql_calls.rs | 67 +++ .../src/literal_sql/tests/test_types.rs | 5 + .../_helpers/analysis/explicit_references.rs | 23 ++ src/sqlbuild/_native.pyi | 3 + .../_helpers/explicit_references/checks.py | 74 +--- 29 files changed, 981 insertions(+), 65 deletions(-) create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/constants.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/mod.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/models.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/tests.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs create mode 100644 crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/constants.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/main/mod.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/mod.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/models.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/tests.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs create mode 100644 crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs new file mode 100644 index 0000000000..e04e5cfbf0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/children.rs @@ -0,0 +1,388 @@ +//! The children CPython's `ast.iter_child_nodes` yields, in `_fields` order. + +use ruff_python_ast::{ + ElifElseClause, ExceptHandler, Expr, Parameters, Pattern, Stmt, Suite, TypeParam, TypeParams, +}; + +use crate::cpython_tree::_helpers::joined::{element_pieces, merged_values}; +use crate::cpython_tree::main::joined_values::joined_values; +use crate::cpython_tree::models::CpythonNode; + +/// The child nodes of `node` that can hold other nodes; operator and context leaves are omitted. +pub(crate) fn children<'a>(node: &CpythonNode<'a>) -> Vec> { + match node { + CpythonNode::Stmt(statement) => statement_children(statement), + CpythonNode::Elif(clauses) => elif_children(clauses), + CpythonNode::Expr(expression) => expression_children(expression), + CpythonNode::Arguments(parameters) => argument_children(parameters), + CpythonNode::Arg(parameter) => optional(parameter.annotation.as_deref()), + CpythonNode::Keyword(keyword) => vec![CpythonNode::Expr(&keyword.value)], + CpythonNode::WithItem(item) => [ + vec![CpythonNode::Expr(&item.context_expr)], + optional(item.optional_vars.as_deref()), + ] + .concat(), + CpythonNode::MatchCase(case) => [ + vec![CpythonNode::Pattern(&case.pattern)], + optional(case.guard.as_deref()), + statements(&case.body), + ] + .concat(), + CpythonNode::Pattern(pattern) => pattern_children(pattern), + CpythonNode::Comprehension(comprehension) => [ + vec![ + CpythonNode::Expr(&comprehension.target), + CpythonNode::Expr(&comprehension.iter), + ], + expressions(&comprehension.ifs), + ] + .concat(), + CpythonNode::ExceptHandler(ExceptHandler::ExceptHandler(handler)) => [ + optional(handler.type_.as_deref()), + statements(&handler.body), + ] + .concat(), + CpythonNode::TypeParam(parameter) => type_param_children(parameter), + CpythonNode::Text(_) => Vec::new(), + CpythonNode::Interpolation(interpolation) => { + let spec: Vec> = interpolation + .format_spec + .iter() + .map(|spec| CpythonNode::FormatSpec(spec)) + .collect(); + [vec![CpythonNode::Expr(&interpolation.expression)], spec].concat() + } + CpythonNode::FormatSpec(spec) => merged_values(element_pieces(&spec.elements)), + } +} + +fn optional(expression: Option<&Expr>) -> Vec> { + expression.map(CpythonNode::Expr).into_iter().collect() +} + +fn expressions(expressions: &[Expr]) -> Vec> { + expressions.iter().map(CpythonNode::Expr).collect() +} + +fn statements(body: &Suite) -> Vec> { + body.iter().map(CpythonNode::Stmt).collect() +} + +fn decorators(decorator_list: &[ruff_python_ast::Decorator]) -> Vec> { + decorator_list + .iter() + .map(|decorator| CpythonNode::Expr(&decorator.expression)) + .collect() +} + +fn type_params(parameters: Option<&TypeParams>) -> Vec> { + parameters + .map(|parameters| parameters.iter().map(CpythonNode::TypeParam).collect()) + .unwrap_or_default() +} + +fn statement_children(statement: &Stmt) -> Vec> { + match statement { + Stmt::FunctionDef(function) => [ + vec![CpythonNode::Arguments(&function.parameters)], + statements(&function.body), + decorators(&function.decorator_list), + optional(function.returns.as_deref()), + type_params(function.type_params.as_deref()), + ] + .concat(), + Stmt::ClassDef(class) => [ + expressions(class.bases()), + class.keywords().iter().map(CpythonNode::Keyword).collect(), + statements(&class.body), + decorators(&class.decorator_list), + type_params(class.type_params.as_deref()), + ] + .concat(), + Stmt::Return(statement) => optional(statement.value.as_deref()), + Stmt::Delete(statement) => expressions(&statement.targets), + Stmt::TypeAlias(statement) => [ + vec![CpythonNode::Expr(&statement.name)], + type_params(statement.type_params.as_deref()), + vec![CpythonNode::Expr(&statement.value)], + ] + .concat(), + Stmt::Assign(statement) => [ + expressions(&statement.targets), + vec![CpythonNode::Expr(&statement.value)], + ] + .concat(), + Stmt::AugAssign(statement) => vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.value), + ], + Stmt::AnnAssign(statement) => [ + vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.annotation), + ], + optional(statement.value.as_deref()), + ] + .concat(), + Stmt::For(statement) => [ + vec![ + CpythonNode::Expr(&statement.target), + CpythonNode::Expr(&statement.iter), + ], + statements(&statement.body), + statements(&statement.orelse), + ] + .concat(), + Stmt::While(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + statements(&statement.body), + statements(&statement.orelse), + ] + .concat(), + Stmt::If(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + statements(&statement.body), + else_children(&statement.elif_else_clauses), + ] + .concat(), + Stmt::With(statement) => [ + statement.items.iter().map(CpythonNode::WithItem).collect(), + statements(&statement.body), + ] + .concat(), + Stmt::Match(statement) => [ + vec![CpythonNode::Expr(&statement.subject)], + statement.cases.iter().map(CpythonNode::MatchCase).collect(), + ] + .concat(), + Stmt::Raise(statement) => [ + optional(statement.exc.as_deref()), + optional(statement.cause.as_deref()), + ] + .concat(), + Stmt::Try(statement) => [ + statements(&statement.body), + statement + .handlers + .iter() + .map(CpythonNode::ExceptHandler) + .collect(), + statements(&statement.orelse), + statements(&statement.finalbody), + ] + .concat(), + Stmt::Assert(statement) => [ + vec![CpythonNode::Expr(&statement.test)], + optional(statement.msg.as_deref()), + ] + .concat(), + Stmt::Expr(statement) => vec![CpythonNode::Expr(&statement.value)], + Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// The `orelse` of an `If`: a nested `If` for an `elif`, or the `else` body. +fn else_children(clauses: &[ElifElseClause]) -> Vec> { + match clauses.first() { + Some(clause) if clause.test.is_some() => vec![CpythonNode::Elif(clauses)], + Some(clause) => statements(&clause.body), + None => Vec::new(), + } +} + +fn elif_children(clauses: &[ElifElseClause]) -> Vec> { + clauses + .split_first() + .map(|(clause, rest)| { + [ + optional(clause.test.as_ref()), + statements(&clause.body), + else_children(rest), + ] + .concat() + }) + .unwrap_or_default() +} + +fn expression_children(expression: &Expr) -> Vec> { + match expression { + Expr::BoolOp(node) => expressions(&node.values), + Expr::Named(node) => vec![ + CpythonNode::Expr(&node.target), + CpythonNode::Expr(&node.value), + ], + Expr::BinOp(node) => vec![ + CpythonNode::Expr(&node.left), + CpythonNode::Expr(&node.right), + ], + Expr::UnaryOp(node) => vec![CpythonNode::Expr(&node.operand)], + Expr::Lambda(node) => [ + node.parameters + .iter() + .map(|parameters| CpythonNode::Arguments(parameters)) + .collect(), + vec![CpythonNode::Expr(&node.body)], + ] + .concat(), + Expr::If(node) => vec![ + CpythonNode::Expr(&node.test), + CpythonNode::Expr(&node.body), + CpythonNode::Expr(&node.orelse), + ], + Expr::Dict(node) => node + .iter_keys() + .flatten() + .chain(node.iter_values()) + .map(CpythonNode::Expr) + .collect(), + Expr::Set(node) => expressions(&node.elts), + Expr::ListComp(node) => comprehension_children(&node.elt, &node.generators), + Expr::SetComp(node) => comprehension_children(&node.elt, &node.generators), + Expr::Generator(node) => comprehension_children(&node.elt, &node.generators), + Expr::DictComp(node) => [ + optional(node.key.as_deref()), + comprehension_children(&node.value, &node.generators), + ] + .concat(), + Expr::Await(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Yield(node) => optional(node.value.as_deref()), + Expr::YieldFrom(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Compare(node) => [ + vec![CpythonNode::Expr(&node.left)], + expressions(&node.comparators), + ] + .concat(), + Expr::Call(node) => [ + vec![CpythonNode::Expr(&node.func)], + expressions(&node.arguments.args), + node.arguments + .keywords + .iter() + .map(CpythonNode::Keyword) + .collect(), + ] + .concat(), + Expr::FString(_) | Expr::TString(_) => joined_values(expression), + Expr::Attribute(node) => vec![CpythonNode::Expr(&node.value)], + Expr::Subscript(node) => vec![ + CpythonNode::Expr(&node.value), + CpythonNode::Expr(&node.slice), + ], + Expr::Starred(node) => vec![CpythonNode::Expr(&node.value)], + Expr::List(node) => expressions(&node.elts), + Expr::Tuple(node) => expressions(&node.elts), + Expr::Slice(node) => [ + optional(node.lower.as_deref()), + optional(node.upper.as_deref()), + optional(node.step.as_deref()), + ] + .concat(), + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::Name(_) + | Expr::IpyEscapeCommand(_) => Vec::new(), + } +} + +fn comprehension_children<'a>( + element: &'a Expr, + generators: &'a [ruff_python_ast::Comprehension], +) -> Vec> { + [ + vec![CpythonNode::Expr(element)], + generators.iter().map(CpythonNode::Comprehension).collect(), + ] + .concat() +} + +/// `arguments` fields: posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults. +fn argument_children(parameters: &Parameters) -> Vec> { + let positional = || parameters.posonlyargs.iter().chain(parameters.args.iter()); + [ + positional() + .map(|parameter| CpythonNode::Arg(¶meter.parameter)) + .collect(), + parameters + .vararg + .iter() + .map(|parameter| CpythonNode::Arg(parameter)) + .collect(), + parameters + .kwonlyargs + .iter() + .map(|parameter| CpythonNode::Arg(¶meter.parameter)) + .collect(), + parameters + .kwonlyargs + .iter() + .filter_map(|parameter| parameter.default.as_deref()) + .map(CpythonNode::Expr) + .collect(), + parameters + .kwarg + .iter() + .map(|parameter| CpythonNode::Arg(parameter)) + .collect(), + positional() + .filter_map(|parameter| parameter.default.as_deref()) + .map(CpythonNode::Expr) + .collect::>>(), + ] + .concat() +} + +fn pattern_children(pattern: &Pattern) -> Vec> { + match pattern { + Pattern::MatchValue(node) => vec![CpythonNode::Expr(&node.value)], + Pattern::MatchSequence(node) => node.patterns.iter().map(CpythonNode::Pattern).collect(), + Pattern::MatchMapping(node) => [ + expressions(&node.keys), + node.patterns.iter().map(CpythonNode::Pattern).collect(), + ] + .concat(), + Pattern::MatchClass(node) => [ + vec![CpythonNode::Expr(&node.cls)], + node.arguments + .patterns + .iter() + .map(CpythonNode::Pattern) + .collect(), + node.arguments + .keywords + .iter() + .map(|keyword| CpythonNode::Pattern(&keyword.pattern)) + .collect(), + ] + .concat(), + Pattern::MatchAs(node) => node + .pattern + .iter() + .map(|inner| CpythonNode::Pattern(inner)) + .collect(), + Pattern::MatchOr(node) => node.patterns.iter().map(CpythonNode::Pattern).collect(), + Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => Vec::new(), + } +} + +fn type_param_children(parameter: &TypeParam) -> Vec> { + match parameter { + TypeParam::TypeVar(node) => [ + optional(node.bound.as_deref()), + optional(node.default.as_deref()), + ] + .concat(), + TypeParam::ParamSpec(node) => optional(node.default.as_deref()), + TypeParam::TypeVarTuple(node) => optional(node.default.as_deref()), + } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs new file mode 100644 index 0000000000..4ae730df7f --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/joined.rs @@ -0,0 +1,64 @@ +//! CPython's merging of adjacent string pieces in f-strings, t-strings and format specs. + +use ruff_python_ast::{InterpolatedStringElement, InterpolatedStringElements}; +use ruff_text_size::{TextLen, TextSize}; + +use crate::cpython_tree::models::{CpythonNode, TextConstant}; + +/// One piece CPython concatenates: literal text at an offset, or an interpolation. +pub(crate) enum JoinedPiece<'a> { + Text(&'a str, TextSize), + Interpolation(&'a ruff_python_ast::InterpolatedElement), +} + +/// The pieces of f-string elements; a self-documenting `{x=}` contributes its text first. +pub(crate) fn element_pieces(elements: &InterpolatedStringElements) -> Vec> { + elements.iter().flat_map(pieces).collect() +} + +fn pieces(element: &InterpolatedStringElement) -> Vec> { + match element { + InterpolatedStringElement::Literal(literal) => { + vec![JoinedPiece::Text(&literal.value, literal.range.start())] + } + InterpolatedStringElement::Interpolation(interpolation) => { + let after_brace: TextSize = interpolation.range.start() + '{'.text_len(); + let debug: Option> = interpolation + .debug_text + .as_ref() + .map(|debug_text| JoinedPiece::Text(debug_text.as_str(), after_brace)); + debug + .into_iter() + .chain([JoinedPiece::Interpolation(interpolation)]) + .collect() + } + } +} + +/// Merge runs of text into constants located at the run's first piece, dropping empty runs. +pub(crate) fn merged_values(pieces: Vec>) -> Vec> { + let mut values: Vec> = Vec::new(); + let mut run: Option = None; + for piece in pieces { + match piece { + JoinedPiece::Text(text, start) => { + let constant: &mut TextConstant = run.get_or_insert_with(|| TextConstant { + value: String::new(), + start, + }); + constant.value.push_str(text); + } + JoinedPiece::Interpolation(interpolation) => { + values.extend(non_empty(run.take())); + values.push(CpythonNode::Interpolation(interpolation)); + } + } + } + values.extend(non_empty(run)); + values +} + +fn non_empty<'a>(run: Option) -> Option> { + run.filter(|constant| !constant.value.is_empty()) + .map(CpythonNode::Text) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs new file mode 100644 index 0000000000..520323cac8 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/lines.rs @@ -0,0 +1,10 @@ +//! The line starts of a source whose newlines are already translated. + +use crate::cpython_tree::models::SourceLines; + +/// Where each `\n`-terminated line of `source` starts. +pub(crate) fn source_lines(source: &str) -> SourceLines { + let mut starts: Vec = vec![0]; + starts.extend(source.match_indices('\n').map(|(index, _)| index + 1)); + SourceLines { starts } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs new file mode 100644 index 0000000000..b6ef371a6a --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod children; +pub(crate) mod joined; +pub(crate) mod lines; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/constants.rs b/crates/sqlbuild-pysource/src/cpython_tree/constants.rs new file mode 100644 index 0000000000..8d13b92cfd --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/constants.rs @@ -0,0 +1,4 @@ +//! Constants of CPython's parse of a source text. + +/// The byte order mark CPython rejects as a non-printable character at the start of a `str` source. +pub(crate) const BYTE_ORDER_MARK: char = '\u{feff}'; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs new file mode 100644 index 0000000000..2c892bbf20 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/joined_values.rs @@ -0,0 +1,29 @@ +//! The `values` CPython gives an f-string's `JoinedStr` or a t-string's `TemplateStr`. + +use ruff_python_ast::{Expr, FStringPart}; + +use crate::cpython_tree::_helpers::joined::{JoinedPiece, element_pieces, merged_values}; +use crate::cpython_tree::models::CpythonNode; + +/// The merged constants and interpolations of an f-string or t-string; empty for other nodes. +pub(crate) fn joined_values(expression: &Expr) -> Vec> { + let pieces: Vec> = match expression { + Expr::FString(fstring) => fstring + .value + .iter() + .flat_map(|part| match part { + FStringPart::Literal(literal) => { + vec![JoinedPiece::Text(&literal.value, literal.range.start())] + } + FStringPart::FString(part) => element_pieces(&part.elements), + }) + .collect(), + Expr::TString(tstring) => tstring + .value + .iter() + .flat_map(|part| element_pieces(&part.elements)) + .collect(), + _ => Vec::new(), + }; + merged_values(pieces) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs new file mode 100644 index 0000000000..67ecd869e5 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod joined_values; +pub(crate) mod parse_cpython_module; +pub(crate) mod source_position; +pub(crate) mod walk_breadth_first; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs new file mode 100644 index 0000000000..935723d9b7 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/parse_cpython_module.rs @@ -0,0 +1,28 @@ +//! Parse a module the way CPython's `ast.parse` accepts a `str` source. + +use ruff_python_ast::PythonVersion; +use ruff_python_parser::{Mode, ParseOptions, Parsed}; + +use crate::cpython_tree::_helpers::lines::source_lines; +use crate::cpython_tree::constants::BYTE_ORDER_MARK; +use crate::cpython_tree::models::ParsedSource; + +/// `source` parsed for the `(major, minor)` Python with CPython's newline translation, if valid. +pub(crate) fn parse_cpython_module(source: &str, python_version: (u8, u8)) -> Option { + if source.starts_with(BYTE_ORDER_MARK) { + return None; + } + let translated: String = source.replace("\r\n", "\n").replace('\r', "\n"); + let (major, minor) = python_version; + let options: ParseOptions = + ParseOptions::from(Mode::Module).with_target_version(PythonVersion { major, minor }); + let parsed: Parsed<_> = ruff_python_parser::parse_unchecked(&translated, options); + if !parsed.has_no_syntax_errors() { + return None; + } + let module = parsed.try_into_module()?.into_syntax(); + Some(ParsedSource { + module, + lines: source_lines(&translated), + }) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs new file mode 100644 index 0000000000..d4e918ba05 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/source_position.rs @@ -0,0 +1,17 @@ +//! CPython's `lineno` and `col_offset` for a byte offset. + +use ruff_text_size::TextSize; + +use crate::cpython_tree::models::SourceLines; + +/// The 1-based line and the UTF-8 byte column of `offset`, as CPython reports a node start. +pub(crate) fn source_position(lines: &SourceLines, offset: TextSize) -> (usize, usize) { + let offset: usize = offset.to_usize(); + let line: usize = lines.starts.partition_point(|start| *start <= offset); + let start: usize = lines + .starts + .get(line.saturating_sub(1)) + .copied() + .unwrap_or_default(); + (line, offset - start) +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs b/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs new file mode 100644 index 0000000000..4307efbf3c --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/main/walk_breadth_first.rs @@ -0,0 +1,17 @@ +//! CPython's `ast.walk` order over mirrored nodes. + +use std::collections::VecDeque; + +use crate::cpython_tree::_helpers::children::children; +use crate::cpython_tree::models::CpythonNode; + +/// `roots` and every node below them, breadth first with children in CPython's `_fields` order. +pub(crate) fn walk_breadth_first<'a>(roots: Vec>) -> Vec> { + let mut queue: VecDeque> = VecDeque::from(roots); + let mut visited: Vec> = Vec::new(); + while let Some(node) = queue.pop_front() { + queue.extend(children(&node)); + visited.push(node); + } + visited +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/mod.rs b/crates/sqlbuild-pysource/src/cpython_tree/mod.rs new file mode 100644 index 0000000000..2ffdf440f3 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/models.rs b/crates/sqlbuild-pysource/src/cpython_tree/models.rs new file mode 100644 index 0000000000..bcbe85e2fd --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/models.rs @@ -0,0 +1,52 @@ +//! CPython `ast` nodes mirrored over the ruff tree, and the line starts of a source. + +use ruff_python_ast::{ + Comprehension, ElifElseClause, ExceptHandler, Expr, InterpolatedElement, + InterpolatedStringFormatSpec, Keyword, MatchCase, ModModule, Parameter, Parameters, Pattern, + Stmt, TypeParam, WithItem, +}; +use ruff_text_size::TextSize; + +/// One node of the tree CPython's `ast.parse` builds, borrowed from the ruff node it mirrors. +#[derive(Clone, Debug)] +pub(crate) enum CpythonNode<'a> { + Stmt(&'a Stmt), + /// The `If` an `elif` clause becomes; the clauses after it form its `orelse`. + Elif(&'a [ElifElseClause]), + Expr(&'a Expr), + Arguments(&'a Parameters), + Arg(&'a Parameter), + Keyword(&'a Keyword), + WithItem(&'a WithItem), + MatchCase(&'a MatchCase), + Pattern(&'a Pattern), + Comprehension(&'a Comprehension), + ExceptHandler(&'a ExceptHandler), + TypeParam(&'a TypeParam), + /// A string `Constant` of an f-string, t-string or format spec, merged as CPython merges it. + Text(TextConstant), + /// A `FormattedValue` or `Interpolation`. + Interpolation(&'a InterpolatedElement), + /// The `JoinedStr` of a format spec. + FormatSpec(&'a InterpolatedStringFormatSpec), +} + +/// A merged string constant and the offset CPython locates it at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TextConstant { + pub(crate) value: String, + pub(crate) start: TextSize, +} + +/// A module CPython accepts, parsed from its source with newlines translated as CPython does. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ParsedSource { + pub(crate) module: ModModule, + pub(crate) lines: SourceLines, +} + +/// The byte offsets where the lines of one newline-translated source start. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SourceLines { + pub(crate) starts: Vec, +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests.rs new file mode 100644 index 0000000000..4df003773b --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_joined_constants.rs"] +mod joined_constants; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs new file mode 100644 index 0000000000..3b6cc1f3c0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_joined_constants.rs @@ -0,0 +1,70 @@ +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::cpython_tree::tests::test_types::JoinedConstantTestCase; + +/// Expectations recorded from CPython 3.12, 3.13 and 3.14, which agree. +#[test] +fn given_interpolated_strings_when_walking_then_constants_merge_as_in_cpython() { + let test_cases = [ + JoinedConstantTestCase { + description: "debug text merges with the literal before it", + source: "f\"a{q=}b\"", + expected_constants: &[("aq=", 1, 2), ("b", 1, 7)], + }, + JoinedConstantTestCase { + description: "debug text keeps its spaces and starts after the brace", + source: "f\"{ q = }\"", + expected_constants: &[(" q = ", 1, 3)], + }, + JoinedConstantTestCase { + description: "implicit concatenation merges across parts", + source: "f\"a\" f\"b{x}\" \"c\" f\"d\"", + expected_constants: &[("ab", 1, 2), ("cd", 1, 13)], + }, + JoinedConstantTestCase { + description: "an empty plain string starts a run and an empty f-string does not", + source: "x = [\"\" f\"a{x}\", f\"{x}\" f\"\" \"b\"]", + expected_constants: &[("a", 1, 5), ("b", 1, 28)], + }, + JoinedConstantTestCase { + description: "format specs are walked one level deeper", + source: "f\"{x:>{y}d}\" \"\\u00e9\"", + expected_constants: &[("é", 1, 13), (">", 1, 5), ("d", 1, 9)], + }, + JoinedConstantTestCase { + description: "lines end at carriage returns", + source: "x = 1\r\ny = f\"\"\"a\r{x}\"\"\"", + expected_constants: &[("a\n", 2, 8)], + }, + ]; + for test_case in test_cases { + let parsed: Option = parse_cpython_module(test_case.source, (3, 12)); + let statements: Vec> = parsed + .iter() + .flat_map(|parsed| parsed.module.body.iter().map(CpythonNode::Stmt)) + .collect(); + let constants: Vec<(String, usize, usize)> = walk_breadth_first(statements) + .into_iter() + .filter_map(|node| { + let CpythonNode::Text(constant) = node else { + return None; + }; + let (line, column) = parsed.as_ref().map_or((0, 0), |parsed| { + source_position(&parsed.lines, constant.start) + }); + Some((constant.value, line, column)) + }) + .collect(); + let found: Vec<(&str, usize, usize)> = constants + .iter() + .map(|(value, line, column)| (value.as_str(), *line, *column)) + .collect(); + assert_eq!( + found, test_case.expected_constants, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs new file mode 100644 index 0000000000..9186ebc495 --- /dev/null +++ b/crates/sqlbuild-pysource/src/cpython_tree/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct JoinedConstantTestCase { + pub(super) description: &'static str, + pub(super) source: &'static str, + pub(super) expected_constants: &'static [(&'static str, usize, usize)], +} diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index da8c2d78f5..37d486260c 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -2,5 +2,7 @@ #![forbid(unsafe_code)] +pub(crate) mod cpython_tree; +pub mod literal_sql; pub mod macro_modules; pub mod module_parse; diff --git a/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs b/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs new file mode 100644 index 0000000000..55b31aaa2a --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/_helpers/arguments.rs @@ -0,0 +1,67 @@ +//! The context parameter, the SQL argument of a call and its literal text. + +use ruff_python_ast::{Expr, ExprCall, Parameters}; + +use crate::cpython_tree::main::joined_values::joined_values; +use crate::cpython_tree::models::CpythonNode; +use crate::literal_sql::constants::{CONTEXT_PARAMETER_NAMES, SQL_PARAMETER_NAME}; + +/// The first well-known context parameter name, else the first parameter, ignoring `*args` and `**kwargs`. +pub(crate) fn context_name(parameters: &Parameters) -> Option<&str> { + let names: Vec<&str> = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .map(|parameter| parameter.parameter.name.as_str()) + .collect(); + names + .iter() + .find(|name| CONTEXT_PARAMETER_NAMES.contains(name)) + .or_else(|| names.first()) + .copied() +} + +/// The first positional argument, else the `sql=` keyword, also inside a `**{...}` literal. +pub(crate) fn sql_argument(call: &ExprCall) -> Option<&Expr> { + if let Some(first) = call.arguments.args.first() { + return Some(first); + } + for keyword in &call.arguments.keywords { + match &keyword.arg { + Some(name) if name.as_str() == SQL_PARAMETER_NAME => return Some(&keyword.value), + Some(_) => {} + None => { + let Expr::Dict(dict) = &keyword.value else { + continue; + }; + for item in &dict.items { + if let Some(Expr::StringLiteral(key)) = &item.key + && key.value.to_str() == SQL_PARAMETER_NAME + { + return Some(&item.value); + } + } + } + } + } + None +} + +/// A string literal's text, or an f-string's with numbered placeholders for its values. +pub(crate) fn literal_text(expression: &Expr, placeholder: &str) -> Option { + match expression { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + Expr::FString(_) => Some( + joined_values(expression) + .iter() + .enumerate() + .map(|(index, value)| match value { + CpythonNode::Text(constant) => constant.value.clone(), + _ => format!("{placeholder}{index}"), + }) + .collect(), + ), + _ => None, + } +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs new file mode 100644 index 0000000000..0bcb87fca2 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod arguments; diff --git a/crates/sqlbuild-pysource/src/literal_sql/constants.rs b/crates/sqlbuild-pysource/src/literal_sql/constants.rs new file mode 100644 index 0000000000..f1177cfe21 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/constants.rs @@ -0,0 +1,10 @@ +//! The context methods and parameters that carry SQL text. + +/// Context methods whose first argument is SQL. +pub(crate) const SQL_METHOD_NAMES: [&str; 2] = ["query", "execute_sql"]; + +/// The keyword name of the SQL argument. +pub(crate) const SQL_PARAMETER_NAME: &str = "sql"; + +/// Parameter names preferred as the context over the first parameter. +pub(crate) const CONTEXT_PARAMETER_NAMES: [&str; 4] = ["ctx", "context", "_ctx", "hook_context"]; diff --git a/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs b/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs new file mode 100644 index 0000000000..7d4a8a9019 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/main/literal_sql_calls.rs @@ -0,0 +1,61 @@ +//! Literal SQL passed to the context's SQL methods inside one function. + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::literal_sql::_helpers::arguments::{context_name, literal_text, sql_argument}; +use crate::literal_sql::constants::SQL_METHOD_NAMES; +use crate::literal_sql::models::{FunctionSource, LiteralSql}; + +/// The context's `query`/`execute_sql` calls with literal SQL, in `ast.walk` order; none when unparsable. +#[must_use] +pub fn literal_sql_calls(source: &FunctionSource<'_>) -> Vec { + let Some(parsed): Option = + parse_cpython_module(source.text, source.python_version) + else { + return Vec::new(); + }; + let Some(Stmt::FunctionDef(definition)) = parsed.module.body.first() else { + return Vec::new(); + }; + let Some(context) = context_name(&definition.parameters) else { + return Vec::new(); + }; + let mut calls: Vec = Vec::new(); + for node in walk_breadth_first( + parsed + .module + .body + .iter() + .take(1) + .map(CpythonNode::Stmt) + .collect(), + ) { + let CpythonNode::Expr(Expr::Call(call)) = node else { + continue; + }; + let Expr::Attribute(attribute) = call.func.as_ref() else { + continue; + }; + let is_context_method: bool = SQL_METHOD_NAMES.contains(&attribute.attr.as_str()) + && matches!(attribute.value.as_ref(), Expr::Name(name) if name.id.as_str() == context); + if !is_context_method { + continue; + } + if let Some(sql) = + sql_argument(call).and_then(|argument| literal_text(argument, source.placeholder)) + { + let (line, _) = source_position(&parsed.lines, call.range().start()); + calls.push(LiteralSql { + method: attribute.attr.to_string(), + sql, + line: source.start_line + line - 1, + }); + } + } + calls +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs new file mode 100644 index 0000000000..a1db6e1556 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/main/mod.rs @@ -0,0 +1 @@ +pub mod literal_sql_calls; diff --git a/crates/sqlbuild-pysource/src/literal_sql/mod.rs b/crates/sqlbuild-pysource/src/literal_sql/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/literal_sql/models.rs b/crates/sqlbuild-pysource/src/literal_sql/models.rs new file mode 100644 index 0000000000..e9db38c94c --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/models.rs @@ -0,0 +1,19 @@ +//! A function source to read literal SQL from, and the literal SQL calls found in it. + +/// A dedented function source, its first line in the file, and the interpreter it runs on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FunctionSource<'a> { + pub text: &'a str, + pub start_line: usize, + /// The prefix of the numbered stand-ins for interpolated f-string values. + pub placeholder: &'a str, + pub python_version: (u8, u8), +} + +/// One context SQL call with literal SQL: the method, the SQL text and its line in the file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiteralSql { + pub method: String, + pub sql: String, + pub line: usize, +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests.rs b/crates/sqlbuild-pysource/src/literal_sql/tests.rs new file mode 100644 index 0000000000..18c3b81c65 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_literal_sql_calls.rs"] +mod literal_sql_calls; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs b/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs new file mode 100644 index 0000000000..b7f13f05e1 --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests/test_literal_sql_calls.rs @@ -0,0 +1,67 @@ +use crate::literal_sql::main::literal_sql_calls::literal_sql_calls; +use crate::literal_sql::models::{FunctionSource, LiteralSql}; +use crate::literal_sql::tests::test_types::LiteralSqlTestCase; + +const PLACEHOLDER: &str = "__value_"; + +/// Expectations recorded from the CPython 3.12 `ast` implementation this replaces. +#[test] +fn given_function_source_when_reading_literal_sql_then_matches_cpython_walk() { + let test_cases = [ + LiteralSqlTestCase { + description: "positional and keyword SQL in walk order", + source: "def task(ctx):\n ctx.execute_sql(sql=\"delete from orders\")\n if ready:\n ctx.query(\"select * from customers\")\n ctx.query(f\"select * from {table} where id = {key!r:>{width}}\")\n", + expected_calls: &[ + ("execute_sql", "delete from orders", 11), + ("query", "select * from __value_1 where id = __value_3", 14), + ("query", "select * from customers", 13), + ], + }, + LiteralSqlTestCase { + description: "a well-known context name wins over the first parameter", + source: "def hook(self, context):\n self.query(\"select 1\")\n context.query(**{\"sql\": \"select 2\"})\n", + expected_calls: &[("query", "select 2", 12)], + }, + LiteralSqlTestCase { + description: "implicit concatenation, debug text and bytes", + source: "async def check(ctx, /):\n ctx.query(\"select \" \"1\")\n ctx.query(f\"{ value = }\" \"tail\")\n ctx.query(b\"select 3\")\n ctx.query(sql)\n", + expected_calls: &[ + ("query", "select 1", 11), + ("query", " value = __value_1tail", 12), + ], + }, + LiteralSqlTestCase { + description: "nested calls come after their parents", + source: "def task(ctx):\n ctx.query(ctx.query(\"inner\"))\n ctx.query(\"outer\")\n", + expected_calls: &[("query", "outer", 12), ("query", "inner", 11)], + }, + LiteralSqlTestCase { + description: "a source that does not parse", + source: "def task(ctx):\n ctx.query(\"select 1\"\n", + expected_calls: &[], + }, + LiteralSqlTestCase { + description: "a source that is not a function", + source: "owner = lambda ctx: ctx.query(\"select 1\")\n", + expected_calls: &[], + }, + LiteralSqlTestCase { + description: "a function without parameters", + source: "def task():\n ctx.query(\"select 1\")\n", + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let calls: Vec = literal_sql_calls(&FunctionSource { + text: test_case.source, + start_line: 10, + placeholder: PLACEHOLDER, + python_version: (3, 12), + }); + let found: Vec<(&str, &str, usize)> = calls + .iter() + .map(|call| (call.method.as_str(), call.sql.as_str(), call.line)) + .collect(); + assert_eq!(found, test_case.expected_calls, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs b/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs new file mode 100644 index 0000000000..a8bc6b777c --- /dev/null +++ b/crates/sqlbuild-pysource/src/literal_sql/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct LiteralSqlTestCase { + pub(super) description: &'static str, + pub(super) source: &'static str, + pub(super) expected_calls: &'static [(&'static str, &'static str, usize)], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 23790b3674..0d09ab978a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -7,6 +7,8 @@ use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::models::{ReferenceReport, ReferenceRequest}; use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; +use sqlbuild_pysource::literal_sql::models::FunctionSource; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; @@ -44,6 +46,26 @@ fn check_native_explicit_references(py: Python<'_>, request_json: &str) -> PyRes }) } +/// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. +#[pyfunction] +fn python_literal_sql_calls( + py: Python<'_>, + source: &str, + start_line: usize, + placeholder: &str, +) -> Vec<(String, String, usize)> { + let version = py.version_info(); + literal_sql_calls(&FunctionSource { + text: source, + start_line, + placeholder, + python_version: (version.major, version.minor), + }) + .into_iter() + .map(|call| (call.method, call.sql, call.line)) + .collect() +} + fn report_value(report: &ReferenceReport) -> PyResult { serde_json::to_value(report).map_err(|error| { compiler_error(native_failure( @@ -55,5 +77,6 @@ fn report_value(report: &ReferenceReport) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(check_native_explicit_references, module)?)?; + module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 41769857bc..521b616a2f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1715,6 +1715,9 @@ def assemble_project_resource_facts( tuple[str, int, str, str, str | None, int | None] | None, ]: ... def check_native_explicit_references(request_json: str, /) -> str: ... +def python_literal_sql_calls( + source: str, start_line: int, placeholder: str +) -> list[tuple[str, str, int]]: ... def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index 9e06ffcca5..8c65c7639c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -2,10 +2,9 @@ from __future__ import annotations -import ast import inspect import textwrap -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from itertools import chain from pathlib import Path @@ -47,9 +46,6 @@ _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") _RAISE_KEY: str = "raise" _REPORT_KEY: str = "report" -_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"}) -_SQL_PARAMETER_NAME: str = "sql" -_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"}) _VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" @@ -82,7 +78,7 @@ def explicit_reference_report( _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () ) literals: tuple[tuple[_LiteralSql, ...], ...] = tuple( - tuple(_literal_sql_calls(owner.function)) for owner in owners + _literal_sql_calls(owner.function) for owner in owners ) paths: list[Path] = sorted({owner.relative_path for owner in owners}) path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} @@ -373,68 +369,16 @@ def _loader_owners( return tuple(owners) -def _literal_sql_calls(function: Callable[..., object]) -> Iterator[_LiteralSql]: +def _literal_sql_calls(function: Callable[..., object]) -> tuple[_LiteralSql, ...]: try: lines: list[str] start_line: int lines, start_line = inspect.getsourcelines(function) - tree: ast.Module = ast.parse(textwrap.dedent("".join(lines))) - except (OSError, TypeError, SyntaxError): - return - definition: ast.stmt | None = tree.body[0] if tree.body else None - if not isinstance(definition, ast.FunctionDef | ast.AsyncFunctionDef): - return - parameters: list[str] = [ - argument.arg - for argument in ( - *definition.args.posonlyargs, - *definition.args.args, - *definition.args.kwonlyargs, + except (OSError, TypeError): + return () + return tuple( + _LiteralSql(method=method, sql=sql, line=line) + for method, sql, line in _native.python_literal_sql_calls( + textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER ) - ] - context_name: str | None = next( - (name for name in parameters if name in _CONTEXT_PARAMETER_NAMES), - parameters[0] if parameters else None, ) - if context_name is None: - return - for node in ast.walk(definition): - if not ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr in _SQL_METHOD_NAMES - and isinstance(node.func.value, ast.Name) - and node.func.value.id == context_name - ): - continue - argument: ast.expr | None = _sql_argument(node) - sql: str | None = _literal_text(argument) if argument is not None else None - if sql is not None: - yield _LiteralSql(method=node.func.attr, sql=sql, line=start_line + node.lineno - 1) - - -def _sql_argument(call: ast.Call) -> ast.expr | None: - if call.args: - return call.args[0] - for keyword in call.keywords: - if keyword.arg == _SQL_PARAMETER_NAME: - return keyword.value - if keyword.arg is None and isinstance(keyword.value, ast.Dict): - for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): - if isinstance(key, ast.Constant) and key.value == _SQL_PARAMETER_NAME: - return value - return None - - -def _literal_text(node: ast.expr) -> str | None: - if isinstance(node, ast.Constant) and isinstance(node.value, str): - return node.value - if not isinstance(node, ast.JoinedStr): - return None - parts: list[str] = [] - for index, value in enumerate(node.values): - if isinstance(value, ast.Constant) and isinstance(value.value, str): - parts.append(value.value) - else: - parts.append(f"{_VALUE_PLACEHOLDER}{index}") - return "".join(parts) From 9fcc2ffa32358afb14c79bab42d2e97aa71a7fbe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:38:50 +0100 Subject: [PATCH 04/26] refactor: locate Python UDF decorators natively Resource diagnostics find the @udf decorator's line and UTF-8 byte column in Rust instead of calling back into a Python ast locator, which is deleted. --- .../src/definition_locations/constants.rs | 4 + .../main/decorator_location.rs | 45 +++++++++++ .../src/definition_locations/main/mod.rs | 1 + .../src/definition_locations/mod.rs | 5 ++ .../src/definition_locations/tests.rs | 4 + .../tests/test_decorator_location.rs | 77 +++++++++++++++++++ .../definition_locations/tests/test_types.rs | 6 ++ crates/sqlbuild-pysource/src/lib.rs | 1 + .../compiled_project/resource_diagnostics.rs | 20 ++--- .../_helpers/assurance_attachment.py | 2 - .../_helpers/resource_locators.py | 29 ------- 11 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 crates/sqlbuild-pysource/src/definition_locations/constants.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/main/mod.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/mod.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/tests.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs create mode 100644 crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py diff --git a/crates/sqlbuild-pysource/src/definition_locations/constants.rs b/crates/sqlbuild-pysource/src/definition_locations/constants.rs new file mode 100644 index 0000000000..6b93361f5d --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/constants.rs @@ -0,0 +1,4 @@ +//! The location reported when no decorator is found. + +/// The line and column offset of a missing decorator. +pub(crate) const MISSING_DECORATOR_LOCATION: (usize, usize) = (1, 1); diff --git a/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs b/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs new file mode 100644 index 0000000000..384afa2f18 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/main/decorator_location.rs @@ -0,0 +1,45 @@ +//! Where a named decorator of a function sits in a module source. + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::definition_locations::constants::MISSING_DECORATOR_LOCATION; + +/// The `lineno` and byte `col_offset` of the first `@decorator` in `ast.walk` order, else `(1, 1)`. +#[must_use] +pub fn decorator_location( + contents: &str, + decorator: &str, + python_version: (u8, u8), +) -> (usize, usize) { + let Some(parsed): Option = parse_cpython_module(contents, python_version) else { + return MISSING_DECORATOR_LOCATION; + }; + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let CpythonNode::Stmt(Stmt::FunctionDef(function)) = node else { + continue; + }; + for expression in function.decorator_list.iter().map(|item| &item.expression) { + if decorator_name(expression) == Some(decorator) { + return source_position(&parsed.lines, expression.range().start()); + } + } + } + MISSING_DECORATOR_LOCATION +} + +fn decorator_name(expression: &Expr) -> Option<&str> { + let target: &Expr = match expression { + Expr::Call(call) => &call.func, + _ => expression, + }; + match target { + Expr::Name(name) => Some(name.id.as_str()), + Expr::Attribute(attribute) => Some(attribute.attr.as_str()), + _ => None, + } +} diff --git a/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs b/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs new file mode 100644 index 0000000000..4cb8145dd4 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/main/mod.rs @@ -0,0 +1 @@ +pub mod decorator_location; diff --git a/crates/sqlbuild-pysource/src/definition_locations/mod.rs b/crates/sqlbuild-pysource/src/definition_locations/mod.rs new file mode 100644 index 0000000000..9b926d1cd2 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests.rs b/crates/sqlbuild-pysource/src/definition_locations/tests.rs new file mode 100644 index 0000000000..6888c15fdf --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_decorator_location.rs"] +mod decorator_location; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs new file mode 100644 index 0000000000..74c26a1947 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorator_location.rs @@ -0,0 +1,77 @@ +use crate::definition_locations::main::decorator_location::decorator_location; +use crate::definition_locations::tests::test_types::DecoratorLocationTestCase; + +/// Expectations recorded from the CPython `ast` implementation this replaces. +#[test] +fn given_module_when_locating_udf_decorator_then_matches_cpython_position() { + let test_cases = [ + DecoratorLocationTestCase { + description: "a call decorator", + contents: "import sqlbuild\n\n\n@sqlbuild.udf(returns=\"INTEGER\")\ndef add_one(value):\n return value + 1\n", + python_version: (3, 12), + expected_location: (4, 1), + }, + DecoratorLocationTestCase { + description: "a bare decorator in a class, found breadth first", + contents: "class Box:\n @udf\n def method(self):\n pass\n\n@ udf\ndef top():\n pass\n", + python_version: (3, 12), + expected_location: (6, 2), + }, + DecoratorLocationTestCase { + description: "columns count UTF-8 bytes", + contents: "x = \"é\"; y = 1\nif x:\n\t@é.udf()\n\tdef f():\n\t\tpass\n", + python_version: (3, 12), + expected_location: (3, 2), + }, + DecoratorLocationTestCase { + description: "carriage returns end lines", + contents: "x = 1\r@udf\rdef f():\r pass\r", + python_version: (3, 12), + expected_location: (2, 1), + }, + DecoratorLocationTestCase { + description: "a parenthesized decorator starts at its parenthesis", + contents: "@(udf)(a=1)\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "a missing decorator", + contents: "@other\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "a syntax error", + contents: "x = 1\n@udf\ndef f(:\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "syntax newer than the interpreter is a syntax error", + contents: "x = t\"tpl\"\n@udf\ndef f():\n pass\n", + python_version: (3, 13), + expected_location: (1, 1), + }, + DecoratorLocationTestCase { + description: "syntax the interpreter supports", + contents: "x = t\"tpl\"\n@udf\ndef f():\n pass\n", + python_version: (3, 14), + expected_location: (2, 1), + }, + DecoratorLocationTestCase { + description: "a leading byte order mark is not printable source", + contents: "\u{feff}@udf\ndef f():\n pass\n", + python_version: (3, 12), + expected_location: (1, 1), + }, + ]; + for test_case in test_cases { + assert_eq!( + decorator_location(test_case.contents, "udf", test_case.python_version), + test_case.expected_location, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs b/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs new file mode 100644 index 0000000000..e8b0557873 --- /dev/null +++ b/crates/sqlbuild-pysource/src/definition_locations/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct DecoratorLocationTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) python_version: (u8, u8), + pub(super) expected_location: (usize, usize), +} diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index 37d486260c..380fbd1ac7 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub(crate) mod cpython_tree; +pub mod definition_locations; pub mod literal_sql; pub mod macro_modules; pub mod module_parse; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index d500e6fba9..c7f9412a8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; +use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; @@ -57,8 +58,6 @@ pub(crate) struct DiagnosticSettings { loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. python_resources: Vec, - /// The Python locators module: YAML entry lines, MODEL header spans and decorators. - locators: Py, } /// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. @@ -432,16 +431,13 @@ fn function_description<'py>( ("function", name, Some(resource_type)), attached.description.as_deref(), || { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("contents", &contents)?; - arguments.set_item("decorator", PYTHON_UDF_DECORATOR)?; - let (line, column): (usize, usize) = diagnostics - .settings - .locators - .bind(py) - .call_method("decorator_location", (), Some(&arguments))? - .extract()?; - diagnostics.location(&path, (line, column), None) + let version = py.version_info(); + let position: (usize, usize) = decorator_location( + &contents, + PYTHON_UDF_DECORATOR, + (version.major, version.minor), + ); + diagnostics.location(&path, position, None) }, &help, ); diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py index d15a8b542a..c652443256 100644 --- a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py +++ b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py @@ -25,7 +25,6 @@ SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.compiled_project._helpers import resource_locators from sqlbuild.compiler.compiled_project._helpers.resource_rows import ( loader_rows, python_resource_rows, @@ -208,7 +207,6 @@ def resource_diagnostics_impl( "python_resources": python_resource_rows( discovered_inputs=discovered_inputs, source_inputs=source_inputs ), - "locators": resource_locators, } ) return (*orphans, *descriptions, *cursors) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py deleted file mode 100644 index 9fdee295d1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py +++ /dev/null @@ -1,29 +0,0 @@ -"""The decorator locator the native resource diagnostics call for authored Python functions.""" - -from __future__ import annotations - -import ast - - -def decorator_location(*, contents: str, decorator: str) -> tuple[int, int]: - """The `@decorator` line and column offset, ignoring docstrings and comments.""" - - try: - module: ast.Module = ast.parse(contents) - except SyntaxError: - return 1, 1 - for node in ast.walk(module): - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - for expression in node.decorator_list: - if _decorator_name(expression) == decorator: - return expression.lineno, expression.col_offset - return 1, 1 - - -def _decorator_name(expression: ast.expr) -> str | None: - target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression - if isinstance(target, ast.Name): - return target.id - if isinstance(target, ast.Attribute): - return target.attr - return None From 15def3ba2468f3c64b39f607f3026448242c3077 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:39:40 +0100 Subject: [PATCH 05/26] refactor: find renamed names in Python strings natively sqb rename and sqb mv read project Python string constants with the ruff parser in CPython's ast.walk order and positions, and match them with Python re IGNORECASE semantics in Rust. The Python ast and regex scan is deleted; Python keeps reading the files. --- crates/sqlbuild-pysource/src/lib.rs | 1 + .../src/string_literals/_helpers/flags.rs | 22 +++ .../src/string_literals/_helpers/mod.rs | 2 + .../src/string_literals/_helpers/python_re.rs | 132 ++++++++++++++++++ .../src/string_literals/constants.rs | 75 ++++++++++ .../src/string_literals/main/mod.rs | 1 + .../main/string_literal_locations.rs | 38 +++++ .../src/string_literals/mod.rs | 7 + .../src/string_literals/models.rs | 8 ++ .../src/string_literals/tests.rs | 4 + .../tests/test_string_literal_locations.rs | 69 +++++++++ .../src/string_literals/tests/test_types.rs | 7 + .../src/bindings/_helpers/refactor/mod.rs | 1 + .../_helpers/refactor/python_strings.rs | 31 ++++ .../_helpers/refactor/registration.rs | 1 + src/sqlbuild/_native.pyi | 3 + .../refactoring/_helpers/python_strings.py | 44 +----- .../compiler/refactoring/constants.py | 4 - .../cli/commands/main/refactor/helpers.py | 10 ++ .../main/refactor/test_rename_columns.py | 9 ++ 20 files changed, 428 insertions(+), 41 deletions(-) create mode 100644 crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/constants.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/main/mod.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/mod.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/models.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/tests.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs create mode 100644 crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index 380fbd1ac7..89a638c28a 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -7,3 +7,4 @@ pub mod definition_locations; pub mod literal_sql; pub mod macro_modules; pub mod module_parse; +pub mod string_literals; diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs new file mode 100644 index 0000000000..0daabc76c6 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/flags.rs @@ -0,0 +1,22 @@ +//! Whether one string constant names a renamed model or column. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::string_literals::_helpers::python_re::{has_word, python_lower, sql_like}; +use crate::string_literals::models::StringQuery; + +/// A name equal to the stripped string, or a whole word of SQL-like text; a context must appear too. +pub(crate) fn flags(value: &str, query: &StringQuery<'_>) -> bool { + let text: Vec = value.chars().collect(); + let is_sql_like: bool = sql_like(&text); + if let Some(context) = query.context + && !(is_sql_like && has_word(&text, context)) + { + return false; + } + let stripped: String = python_lower(python_strip(value)); + query + .names + .iter() + .any(|name| stripped == python_lower(name) || (is_sql_like && has_word(&text, name))) +} diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs new file mode 100644 index 0000000000..0cf7c1e49c --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod flags; +pub(crate) mod python_re; diff --git a/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs b/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs new file mode 100644 index 0000000000..3cd6e3b72e --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/_helpers/python_re.rs @@ -0,0 +1,132 @@ +//! Python `re` with `re.IGNORECASE` and `str.lower` for the refactoring string checks. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; + +use crate::string_literals::constants::{ + EXTRA_CASES, IDENTIFIER_PUNCTUATION, SQL_KEYWORDS, UNCASED_IN_PYTHON, +}; + +/// `re.search(r"\b(select|from|join|insert|update)\b", text, re.I) is not None`. +pub(crate) fn sql_like(text: &[char]) -> bool { + let keywords: Vec> = SQL_KEYWORDS + .iter() + .map(|keyword| keyword.chars().collect()) + .collect(); + (0..=text.len()).any(|start| word_boundary(text, start) && keyword_at(text, start, &keywords)) +} + +fn keyword_at(text: &[char], start: usize, keywords: &[Vec]) -> bool { + keywords.iter().any(|keyword| { + matches_at(text, start, keyword) && word_boundary(text, start + keyword.len()) + }) +} + +/// `re.search(rf"(? bool { + let word: Vec = word.chars().collect(); + if word.len() > text.len() { + return false; + } + (0..=text.len() - word.len()).any(|start| { + let end: usize = start + word.len(); + (start == 0 || !identifier_character(text[start - 1])) + && matches_at(text, start, &word) + && text + .get(end) + .is_none_or(|next| !identifier_character(*next)) + }) +} + +/// Python's `text.lower()`, keeping code points CPython 3.12 does not case-map. +pub(crate) fn python_lower(text: &str) -> String { + let mut lowered: String = String::with_capacity(text.len()); + let mut segment_start: usize = 0; + for (index, character) in text.char_indices() { + if uncased_in_python(character) { + lowered.push_str(&text[segment_start..index].to_lowercase()); + lowered.push(character); + segment_start = index + character.len_utf8(); + } + } + lowered.push_str(&text[segment_start..].to_lowercase()); + lowered +} + +fn matches_at(text: &[char], start: usize, pattern: &[char]) -> bool { + pattern.iter().enumerate().all(|(offset, expected)| { + text.get(start + offset) + .is_some_and(|found| literal_matches(*expected, *found)) + }) +} + +/// Python's `\b` for `str` patterns: a word character on exactly one side. +fn word_boundary(text: &[char], at: usize) -> bool { + let before: bool = at > 0 + && text + .get(at - 1) + .is_some_and(|character| word_character(*character)); + let after: bool = text + .get(at) + .is_some_and(|character| word_character(*character)); + before != after +} + +fn word_character(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// `[A-Za-z0-9_$]` under `re.IGNORECASE`: the folded character is in the folded class. +fn identifier_character(character: char) -> bool { + let folded: char = sre_lower(character); + identifier_base(folded) + || EXTRA_CASES.iter().any(|(code, extras)| { + char::from_u32(*code).is_some_and(identifier_base) + && extras.contains(&u32::from(folded)) + }) +} + +fn identifier_base(character: char) -> bool { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || IDENTIFIER_PUNCTUATION.contains(&character) +} + +/// One pattern literal against one character under `re.IGNORECASE`, as `sre` compiles it. +fn literal_matches(pattern: char, character: char) -> bool { + if !sre_cased(pattern) { + return character == pattern; + } + let lower: char = sre_lower(pattern); + let folded: char = sre_lower(character); + folded == lower || extra_cases(lower).contains(&u32::from(folded)) +} + +fn extra_cases(lower: char) -> &'static [u32] { + EXTRA_CASES + .binary_search_by_key(&u32::from(lower), |(code, _)| *code) + .map_or(&[], |index| EXTRA_CASES[index].1) +} + +fn uncased_in_python(character: char) -> bool { + let code: u32 = u32::from(character); + UNCASED_IN_PYTHON + .iter() + .any(|(first, last)| (*first..=*last).contains(&code)) +} + +/// `_sre.unicode_tolower`: the first code point of the full lowercase mapping. +fn sre_lower(character: char) -> char { + if uncased_in_python(character) { + return character; + } + character.to_lowercase().next().unwrap_or(character) +} + +/// `_sre.unicode_iscased`: the first code point of either case mapping differs. +fn sre_cased(character: char) -> bool { + if uncased_in_python(character) { + return false; + } + sre_lower(character) != character + || character.to_uppercase().next().unwrap_or(character) != character +} diff --git a/crates/sqlbuild-pysource/src/string_literals/constants.rs b/crates/sqlbuild-pysource/src/string_literals/constants.rs new file mode 100644 index 0000000000..260d711f8d --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/constants.rs @@ -0,0 +1,75 @@ +//! Python `re` facts for the refactoring string checks, from CPython 3.12. + +/// The keywords of `\b(select|from|join|insert|update)\b`, which marks a string as SQL-like. +pub(crate) const SQL_KEYWORDS: [&str; 5] = ["select", "from", "join", "insert", "update"]; + +/// `[A-Za-z0-9_$]` before `re.IGNORECASE` folding: identifier characters around a name. +pub(crate) const IDENTIFIER_PUNCTUATION: [char; 2] = ['_', '$']; + +/// Code points Rust case-maps whose case CPython 3.12's Unicode data does not know. +pub(crate) const UNCASED_IN_PYTHON: [(u32, u32); 10] = [ + (0x019B, 0x019B), + (0x0264, 0x0264), + (0x1C89, 0x1C8A), + (0xA7CB, 0xA7CF), + (0xA7D2, 0xA7D5), + (0xA7DA, 0xA7DC), + (0x10D50, 0x10D65), + (0x10D70, 0x10D85), + (0x16EA0, 0x16EB8), + (0x16EBB, 0x16ED3), +]; + +/// `re._casefix._EXTRA_CASES`: lowercase code points that share an uppercase with others. +pub(crate) const EXTRA_CASES: [(u32, &[u32]); 50] = [ + (0x0069, &[0x0131]), + (0x0073, &[0x017F]), + (0x00B5, &[0x03BC]), + (0x0131, &[0x0069]), + (0x017F, &[0x0073]), + (0x0345, &[0x03B9, 0x1FBE]), + (0x0390, &[0x1FD3]), + (0x03B0, &[0x1FE3]), + (0x03B2, &[0x03D0]), + (0x03B5, &[0x03F5]), + (0x03B8, &[0x03D1]), + (0x03B9, &[0x0345, 0x1FBE]), + (0x03BA, &[0x03F0]), + (0x03BC, &[0x00B5]), + (0x03C0, &[0x03D6]), + (0x03C1, &[0x03F1]), + (0x03C2, &[0x03C3]), + (0x03C3, &[0x03C2]), + (0x03C6, &[0x03D5]), + (0x03D0, &[0x03B2]), + (0x03D1, &[0x03B8]), + (0x03D5, &[0x03C6]), + (0x03D6, &[0x03C0]), + (0x03F0, &[0x03BA]), + (0x03F1, &[0x03C1]), + (0x03F5, &[0x03B5]), + (0x0432, &[0x1C80]), + (0x0434, &[0x1C81]), + (0x043E, &[0x1C82]), + (0x0441, &[0x1C83]), + (0x0442, &[0x1C84, 0x1C85]), + (0x044A, &[0x1C86]), + (0x0463, &[0x1C87]), + (0x1C80, &[0x0432]), + (0x1C81, &[0x0434]), + (0x1C82, &[0x043E]), + (0x1C83, &[0x0441]), + (0x1C84, &[0x0442, 0x1C85]), + (0x1C85, &[0x0442, 0x1C84]), + (0x1C86, &[0x044A]), + (0x1C87, &[0x0463]), + (0x1C88, &[0xA64B]), + (0x1E61, &[0x1E9B]), + (0x1E9B, &[0x1E61]), + (0x1FBE, &[0x0345, 0x03B9]), + (0x1FD3, &[0x0390]), + (0x1FE3, &[0x03B0]), + (0xA64B, &[0x1C88]), + (0xFB05, &[0xFB06]), + (0xFB06, &[0xFB05]), +]; diff --git a/crates/sqlbuild-pysource/src/string_literals/main/mod.rs b/crates/sqlbuild-pysource/src/string_literals/main/mod.rs new file mode 100644 index 0000000000..e3ad60f534 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/main/mod.rs @@ -0,0 +1 @@ +pub mod string_literal_locations; diff --git a/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs b/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs new file mode 100644 index 0000000000..49d58000e0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/main/string_literal_locations.rs @@ -0,0 +1,38 @@ +//! String constants of a Python module that name a renamed model or column. + +use ruff_python_ast::Expr; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::string_literals::_helpers::flags::flags; +use crate::string_literals::models::StringQuery; + +/// The 1-based line and column of each flagged `str` constant in `ast.walk` order; none when unparsable. +#[must_use] +pub fn string_literal_locations( + contents: &str, + query: &StringQuery<'_>, + python_version: (u8, u8), +) -> Vec<(usize, usize)> { + let Some(parsed): Option = parse_cpython_module(contents, python_version) else { + return Vec::new(); + }; + let mut locations: Vec<(usize, usize)> = Vec::new(); + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let (value, start): (&str, TextSize) = match &node { + CpythonNode::Expr(Expr::StringLiteral(literal)) => { + (literal.value.to_str(), literal.range().start()) + } + CpythonNode::Text(constant) => (&constant.value, constant.start), + _ => continue, + }; + if flags(value, query) { + let (line, column) = source_position(&parsed.lines, start); + locations.push((line, column + 1)); + } + } + locations +} diff --git a/crates/sqlbuild-pysource/src/string_literals/mod.rs b/crates/sqlbuild-pysource/src/string_literals/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/string_literals/models.rs b/crates/sqlbuild-pysource/src/string_literals/models.rs new file mode 100644 index 0000000000..7ec1f6588d --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/models.rs @@ -0,0 +1,8 @@ +//! The names a refactoring looks for in Python string constants. + +/// Renamed names, and the name a SQL-like string must also mention, if any. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StringQuery<'a> { + pub names: &'a [String], + pub context: Option<&'a str>, +} diff --git a/crates/sqlbuild-pysource/src/string_literals/tests.rs b/crates/sqlbuild-pysource/src/string_literals/tests.rs new file mode 100644 index 0000000000..cd9100cc37 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_string_literal_locations.rs"] +mod string_literal_locations; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs b/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs new file mode 100644 index 0000000000..5dfacadfa0 --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests/test_string_literal_locations.rs @@ -0,0 +1,69 @@ +use crate::string_literals::main::string_literal_locations::string_literal_locations; +use crate::string_literals::models::StringQuery; +use crate::string_literals::tests::test_types::StringLocationTestCase; + +/// Expectations recorded from the CPython 3.12 `ast` and `re` implementation this replaces. +#[test] +fn given_module_when_finding_named_strings_then_matches_cpython_constants() { + let test_cases = [ + StringLocationTestCase { + description: "an exact name, a SQL word and a docstring", + contents: "\"\"\"orders\"\"\"\nquery = \"SELECT * FROM orders o\"\nother = \"orders_archive\"\nname = \" ORDERS \"\n", + names: &["orders"], + context: None, + expected_locations: &[(1, 1), (2, 9), (4, 8)], + }, + StringLocationTestCase { + description: "f-string constants are merged and located at their first piece", + contents: "sql = f\"select {cols} from orders\" \" where\"\nlabel = f\"{orders=}\"\n", + names: &["orders"], + context: None, + expected_locations: &[(1, 22)], + }, + StringLocationTestCase { + description: "a context must appear in SQL-like text", + contents: "a = \"select id from orders\"\nb = \"select id from customers\"\nc = \"id\"\n", + names: &["id"], + context: Some("orders"), + expected_locations: &[(1, 5)], + }, + StringLocationTestCase { + description: "Python ignore-case folds the long s and dotted capital I", + contents: "a = \"ſelect x from t\"\nb = \"SELECT İd FROM t\"\nc = \"select $id from t\"\n", + names: &["id", "x"], + context: None, + expected_locations: &[(1, 5), (2, 5)], + }, + StringLocationTestCase { + description: "an unparsable file", + contents: "x = \"orders\"\ndef f(:\n", + names: &["orders"], + context: None, + expected_locations: &[], + }, + StringLocationTestCase { + description: "match patterns and annotations", + contents: "match x:\n case {\"orders\": 1}:\n pass\ndef f(a: \"orders\"): pass\n", + names: &["orders"], + context: None, + expected_locations: &[(2, 11), (4, 10)], + }, + ]; + for test_case in test_cases { + let names: Vec = test_case + .names + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let query: StringQuery<'_> = StringQuery { + names: &names, + context: test_case.context, + }; + assert_eq!( + string_literal_locations(test_case.contents, &query, (3, 12)), + test_case.expected_locations, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs b/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs new file mode 100644 index 0000000000..248ad43efa --- /dev/null +++ b/crates/sqlbuild-pysource/src/string_literals/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(super) struct StringLocationTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) names: &'static [&'static str], + pub(super) context: Option<&'static str>, + pub(super) expected_locations: &'static [(usize, usize)], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs index 19f416b5bb..432cf39b99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod contract_edits; +pub(crate) mod python_strings; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs new file mode 100644 index 0000000000..9299b29f42 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs @@ -0,0 +1,31 @@ +//! Python string constants that name a renamed model or column, for `sqb rename` and `sqb mv`. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::string_literals::main::string_literal_locations::string_literal_locations; +use sqlbuild_pysource::string_literals::models::StringQuery; + +/// The 1-based `(line, column)` of each flagged string constant; none when `contents` does not parse. +#[pyfunction] +#[pyo3(signature = (contents, names, context))] +fn python_string_locations( + py: Python<'_>, + contents: &str, + names: Vec, + context: Option<&str>, +) -> Vec<(usize, usize)> { + let version = py.version_info(); + string_literal_locations( + contents, + &StringQuery { + names: &names, + context, + }, + (version.major, version.minor), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(python_string_locations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs index fa4ab9dc4a..cc2da0c85a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs @@ -144,5 +144,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(with_model_migration_json, module)?)?; module.add_function(wrap_pyfunction!(commit_refactor_plan_json, module)?)?; crate::bindings::_helpers::refactor::contract_edits::register(module)?; + crate::bindings::_helpers::refactor::python_strings::register(module)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 521b616a2f..63febcc533 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1227,6 +1227,9 @@ def contract_model_edit( def contract_source_edit( row: tuple[str, str, str, list[tuple[str, str | None]], list[tuple[str, str]], bool], / ) -> tuple[str | None, str | None, str | None]: ... +def python_string_locations( + contents: str, names: list[str], context: str | None +) -> list[tuple[int, int]]: ... def resource_identity_suggestion( name: str, private_identity: bool, / ) -> tuple[str, str] | None: ... diff --git a/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py index df96a67817..fccb48461b 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py @@ -1,13 +1,11 @@ -"""Python `ast` scan for strings that name a renamed model or column.""" +"""Project Python files and the strings in them that name a renamed model or column.""" from __future__ import annotations -import ast -import re from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, SQL_LIKE_PATTERN from sqlbuild.compiler.refactoring.models import ManualLocation @@ -37,38 +35,10 @@ def string_locations( """Return the string constants of one Python file that name a renamed model or column.""" try: - tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) - except (OSError, SyntaxError, UnicodeError): + contents: str = (project_dir / relative_path).read_text(encoding="utf-8") + except (OSError, UnicodeError): return () - locations: list[ManualLocation] = [] - node: ast.AST - for node in ast.walk(tree): - if not isinstance(node, ast.Constant) or not isinstance(node.value, str): - continue - if _flags(value=node.value, names=names, context=context): - locations.append( - ManualLocation( - path=relative_path.as_posix(), - line=node.lineno, - column=node.col_offset + 1, - reason=reason, - ) - ) - return tuple(locations) - - -def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: - sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) - if context is not None and not (sql_like and _has_word(text=value, word=context)): - return False - return any( - value.strip().lower() == name.lower() or (sql_like and _has_word(text=value, word=name)) - for name in names - ) - - -def _has_word(*, text: str, word: str) -> bool: - pattern: re.Pattern[str] = re.compile( - rf"(? dict[str, str]: '\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' ), } +ORDER_AMOUNT_CHECK: dict[str, str] = { + "python/checks/order_amounts.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import model\n\n\n" + '@check(depends_on=model("stg_orders"))\n' + "def order_amounts_present(ctx):\n" + ' """Compare with: select amount from stg_orders."""\n' + " return ctx.pass_()\n" + ), +} CUSTOMER_DECLARATIONS: dict[str, str] = { "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", "seeds/customers.yml": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index da5f81b6ae..a46b665363 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -18,6 +18,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( CENTS_MACRO, CUSTOMER_DECLARATIONS, + ORDER_AMOUNT_CHECK, ORDER_EXPORT, ORDER_SHAPE, ORDERS_MACRO, @@ -166,6 +167,14 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( expected_reason="would take macro:to_dollars along, which must stay in", expected_paths=("models/staging/_sqlbuild/_macros/cents.py",), ), + RefusedRefactorE2ETestCase( + description="column named in Python SQL text needs a manual check", + command=("rename", "column:stg_orders.amount", "revenue"), + extra_files=ORDER_AMOUNT_CHECK, + expected_status="refused", + expected_reason="Python SQL may read column amount of stg_orders", + expected_paths=("python/checks/order_amounts.py",), + ), RefusedRefactorE2ETestCase( description="rename onto an existing model", command=("rename", "model:stg_orders", "fact_orders"), From cb2a0110e57f2db0ea0cea68ce4fd74b4aacba27 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:08:55 +0100 Subject: [PATCH 06/26] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 3e2161247f..1e21e97d0c 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 136185 +remaining_lines = 136064 [[stays]] path = "executor" From ce64c0a191fe5fbde68e485120d67e91759650df Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:36:39 +0100 Subject: [PATCH 07/26] refactor: discover project inputs in the native compile driver --- .../_helpers/compile_driver/analysis.rs | 2 +- .../_helpers/compile_driver/discovery.rs | 540 ++++++++++++++++++ .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/registration.rs | 5 +- python-boundary.toml | 10 +- scripts/fensu_policy/constants.py | 1 - src/sqlbuild/_native.pyi | 14 + .../_helpers/build_planning/invocation.py | 2 +- .../cli/commands/_helpers/clone/invocation.py | 2 +- .../cli/commands/_helpers/cost/refresh.py | 2 +- .../cli/commands/_helpers/debug/checks.py | 2 +- .../cli/commands/_helpers/diff/invocation.py | 2 +- .../_helpers/janitor_runtime/invocation.py | 2 +- .../commands/_helpers/lineage/execution.py | 5 +- .../cli/commands/_helpers/lint/selection.py | 3 +- .../cli/commands/_helpers/load/invocation.py | 2 +- .../cli/commands/_helpers/plan/invocation.py | 2 +- .../cli/commands/_helpers/rules/command.py | 3 +- .../_helpers/runtime/command_invocation.py | 2 +- .../_helpers/scenario_capture/capture.py | 2 +- .../_helpers/scenario_execution/runner.py | 2 +- .../cli/commands/main/execution/_freshness.py | 3 +- .../cli/commands/main/inspection/_contract.py | 3 +- .../cli/commands/main/inspection/_query.py | 2 +- .../cli/commands/main/project/_dag.py | 2 +- .../cli/commands/main/project/_format.py | 2 +- .../compile/main/sql_expansion_context.py | 2 +- .../compiler/contract_adoption/main/write.py | 3 +- .../_helpers/filesystem/aggregation.py | 189 ------ .../compiler/discovery/main/discover.py | 106 ---- .../scopes/main/load_or_build_scope_index.py | 3 +- .../dbt/_helpers/pipeline/interop_prologue.py | 3 +- .../integrations/dbt/main/pipeline/debug.py | 2 +- src/sqlbuild/lint/_helpers/expansion.py | 2 +- .../_helpers/engine/rule_harness.py | 2 +- ... _configure_discovered_event_exporters.py} | 0 .../cli/commands/main/test_load_command.py | 2 +- .../compiler/analysis_session/helpers.py | 3 +- .../src/sqlbuild/compiler/compile/helpers.py | 3 +- .../compiler/compiled_project/helpers.py | 3 +- .../test_native_compiled_project.py | 2 +- .../test_native_parameter_boundaries.py | 3 +- .../sqlbuild/compiler/contracts/helpers.py | 3 +- .../sqlbuild/compiler/discovery/helpers.py | 2 +- .../test_native_declaration_files.py | 2 +- .../test_native_undecodable_paths.py | 2 +- .../compiler/explicit_references/helpers.py | 3 +- .../src/sqlbuild/compiler/lineage/helpers.py | 3 +- .../sqlbuild/compiler/model_config/helpers.py | 3 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 3 +- .../compiler/pipeline/test_diff_selectors.py | 2 +- .../test_macro_declaration_context.py | 2 +- .../sqlbuild/compiler/pipeline/test_main.py | 3 +- .../pipeline/test_native_binding_catalog.py | 3 +- .../test_semantic_snowflake_release.py | 3 +- .../compiler/pipeline/test_semantic_triage.py | 3 +- .../pipeline/test_set_operation_types.py | 2 +- .../test_sql_test_input_extraction.py | 2 +- .../sql_test_assembly/test_execute_chain.py | 3 +- .../planner/_helpers/test_fixture_planning.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../compiler/project_assembly/helpers.py | 3 +- .../src/sqlbuild/compiler/scopes/helpers.py | 2 +- .../compiler/semantic_checks/helpers.py | 3 +- .../compiler/sql_test_glue/helpers.py | 3 +- .../executor/build/concurrent/helpers.py | 2 +- .../concurrent/test_ordering_invariant.py | 2 +- .../sqlbuild/executor/build/custom/helpers.py | 2 +- .../src/sqlbuild/executor/build/helpers.py | 2 +- .../build/snowflake_metadata/helpers.py | 2 +- .../executor/build/test_snapshot_execution.py | 2 +- .../sqlbuild/integrations/dbt/test_graph.py | 3 +- .../integrations/dbt/test_manifest.py | 3 +- .../src/sqlbuild/lint/main/helpers.py | 2 +- .../rule_engine/host_payload/helpers.py | 2 +- .../scenario_execution/test_namespace.py | 2 +- .../main/compile/test_compile_command.py | 2 +- .../compile/_helpers/config/helpers.py | 3 +- .../compiler/compile/_helpers/helpers.py | 3 +- .../compile/_helpers/test_assembly.py | 3 +- .../compile/_helpers/test_audit_factories.py | 3 +- .../_helpers/test_declaration_scopes.py | 3 +- .../compile/_helpers/test_declarations.py | 3 +- .../compile/_helpers/test_model_schemas.py | 3 +- .../_helpers/test_rebind_declaration_scope.py | 2 +- .../compiler/compile/test_cursor_start.py | 3 +- .../sqlbuild/compiler/compile/test_main.py | 3 +- .../compile/test_resolved_connection.py | 3 +- .../sqlbuild/compiler/dag/main/test_build.py | 3 +- .../discovery/_helpers/_test_helpers.py | 4 +- .../test_filesystem_directory_snapshot.py | 2 +- .../test_filesystem_scoped_declarations.py | 2 +- .../_helpers/test_resource_identities.py | 2 +- .../_helpers/test_target_warehouses.py | 2 +- .../sqlbuild/compiler/discovery/test_main.py | 2 +- .../warehouse/test_semantic_sources.py | 2 +- .../lint/_helpers/test_project_files.py | 2 +- .../lint/test_adapter_lexical_syntax.py | 2 +- .../main/run_rules/test_run_rules.py | 3 +- 99 files changed, 659 insertions(+), 430 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs delete mode 100644 src/sqlbuild/compiler/discovery/main/discover.py rename src/sqlbuild/runtime/event_exporting/main/{configure_discovered_event_exporters.py => _configure_discovered_event_exporters.py} (100%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 3cb764a981..8d9c267653 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; -const DISCOVER_MODULE: &str = "sqlbuild.compiler.discovery.main.discover"; +const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs new file mode 100644 index 0000000000..73dde619b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -0,0 +1,540 @@ +//! Project discovery: configuration, declaration files and imported project Python, validated. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; + +const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const PROJECT_YAML_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.project"; +const TARGET_CONNECTIONS_MODULE: &str = + "sqlbuild.compiler.discovery._helpers.validation.target_connections"; +const PYTHON_PATHS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.python_paths"; +const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const EVENT_EXPORTERS_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; +const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; +const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; +const LOADERS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.integrations.loaders"; +const DECLARATION_FILES_MODULE: &str = + "sqlbuild.compiler.discovery._helpers.native.declaration_files"; +const MODEL_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.model_files"; +const SQL_TEST_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.sql_test_files"; +const YAML_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.yaml_files"; +const SNAPSHOT_MODULE: &str = "sqlbuild.compiler.discovery.classes.directory_snapshot"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; + +/// Declaration file collections counted into the declaration-parse lifecycle, in field order. +const DECLARATION_COLLECTIONS: [&str; 14] = [ + "source_files", + "model_files", + "enum_files", + "constant_files", + "model_schema_files", + "sql_function_files", + "sql_hook_files", + "python_function_files", + "schema_files", + "seed_files", + "test_files", + "scenario_files", + "audit_files", + "macro_files", +]; + +/// Discoverers of the declaration files that take only `project_dir`, with their result fields. +const FILE_DISCOVERERS: [(&str, &str, &str); 8] = [ + ("enum_files", FILESYSTEM_MODULE, "discover_enum_files"), + ( + "constant_files", + FILESYSTEM_MODULE, + "discover_constant_files", + ), + ( + "model_schema_files", + FILESYSTEM_MODULE, + "discover_model_schema_files", + ), + ( + "sql_function_files", + FILESYSTEM_MODULE, + "discover_sql_function_files", + ), + ( + "sql_hook_files", + FILESYSTEM_MODULE, + "discover_sql_hook_files", + ), + ( + "python_function_files", + FILESYSTEM_MODULE, + "discover_python_function_files", + ), + ( + "schema_files", + YAML_FILES_MODULE, + "discover_native_schema_files", + ), + ("seed_files", FILESYSTEM_MODULE, "discover_seed_files"), +]; + +/// Discoverers that run after the schema files, in their original order. +const LATE_FILE_DISCOVERERS: [(&str, &str, &str); 6] = [ + ( + "test_files", + SQL_TEST_FILES_MODULE, + "discover_native_test_files", + ), + ( + "scenario_files", + SQL_TEST_FILES_MODULE, + "discover_native_scenario_files", + ), + ("audit_files", FILESYSTEM_MODULE, "discover_audit_files"), + ("macro_files", FILESYSTEM_MODULE, "discover_macro_files"), + ("adapter_file", FILESYSTEM_MODULE, "discover_adapter_file"), + ( + "native_session", + DECLARATION_FILES_MODULE, + "retained_discovery_session", + ), +]; + +/// Discoverers of imported project Python that take `project_dir` and `providers`. +const PYTHON_DISCOVERERS: [(&str, &str, &str); 5] = [ + ( + "python_nodes", + FILESYSTEM_MODULE, + "discover_python_node_functions", + ), + ( + "materialization_files", + FILESYSTEM_MODULE, + "discover_materialization_files", + ), + ( + "hook_functions", + FILESYSTEM_MODULE, + "discover_hook_functions", + ), + ( + "event_exporters", + FILESYSTEM_MODULE, + "discover_event_exporter_functions", + ), + ( + "command_output_sinks", + SINKS_MODULE, + "discover_command_output_sink_functions", + ), +]; + +/// Python node groups whose relative paths count as imported project Python. +const PYTHON_NODE_GROUPS: [&str; 5] = ["loaders", "tasks", "assets", "checks", "audit_factories"]; + +/// `OperationLifecycle(operation_kind="project", operation_name=name)`. +fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", "project".into_pyobject(py)?.into_any()), + ("operation_name", name.into_pyobject(py)?.into_any()), + ], + ) +} + +/// The keyword arguments of one discovery, kept for the frontier stage. +struct DiscoveryRequest { + project_dir: Py, + sql_analysis_enabled_override: Py, + extract_output_column_locations: bool, +} + +/// Load all raw project inputs from disk before semantic resolution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn discover_project_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_project_inputs", + options, + ( + &["project_dir"], + &[ + "sql_analysis_enabled_override", + "extract_output_column_locations", + ], + ), + )?; + let request: DiscoveryRequest = DiscoveryRequest { + project_dir: arguments.value("project_dir")?.unbind(), + sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), + extract_output_column_locations: !arguments.passed("extract_output_column_locations")? + || arguments.flag("extract_output_column_locations")?, + }; + within(&lifecycle(py, "project_discovery")?, |_| { + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + let py: Python<'_> = arguments.py(); + within(&lifecycle(py, "discovery_project_assembly")?, |_| { + assembled_inputs(py, &request) + }) + .map(Bound::unbind) + }, + )?; + call_with( + &imported(py, FRONTIER_MODULE, "compile_frontier")?, + &[ + ( + "until", + imported(py, FRONTIER_TYPES_MODULE, "CompilerStage")? + .getattr("DISCOVERED_PROJECT_INPUTS")?, + ), + ("python_stage", stage.into_any()), + ], + ) + }) +} + +/// `_assemble_discovered_project_inputs`: configuration, its checks, discovery and validation. +fn assembled_inputs<'py>( + py: Python<'py>, + request: &DiscoveryRequest, +) -> PyResult> { + let project_dir: &Bound<'py, PyAny> = request.project_dir.bind(py); + let directory: [(&str, Bound<'py, PyAny>); 1] = [("project_dir", project_dir.clone())]; + let project_config: Bound<'py, PyAny> = call_with( + &imported(py, PROJECT_YAML_MODULE, "load_project_config")?, + &directory, + )?; + let local_config: Bound<'py, PyAny> = call_with( + &imported(py, PROJECT_YAML_MODULE, "load_local_config")?, + &directory, + )?; + let configured: [(&str, Bound<'py, PyAny>); 3] = [ + ("project_dir", project_dir.clone()), + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ]; + for (module, name) in [ + (PROJECT_YAML_MODULE, "validate_local_sql_analysis_policy"), + (PROJECT_YAML_MODULE, "validate_target_warehouses_adapter"), + (TARGET_CONNECTIONS_MODULE, "validate_target_connections"), + ] { + call_with(&imported(py, module, name)?, &configured)?; + } + call_with( + &imported(py, PYTHON_PATHS_MODULE, "validate_project_python_paths")?, + &directory, + )?; + let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); + let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { + sql_analysis_override.clone() + } else if local_config + .getattr("setting_overrides")? + .contains(imported( + py, + DISCOVERY_CONSTANTS_MODULE, + "SQL_ANALYSIS_SETTING_KEY", + )?)? + { + local_config.getattr("settings")?.getattr("sql_analysis")? + } else { + project_config + .getattr("settings")? + .getattr("sql_analysis")? + }; + let discovered: Bound<'py, PyAny> = discovered_inputs( + py, + (project_dir, &project_config, &local_config), + ( + sql_analysis_enabled.is_truthy()?, + request.extract_output_column_locations, + ), + )?; + imported(py, VALIDATION_MODULE, "validate_discovered_inputs")?.call1((&discovered,))?; + imported( + py, + EVENT_EXPORTERS_MODULE, + "configure_discovered_event_exporters", + )? + .call1((&discovered,))?; + Ok(discovered) +} + +/// Discover all project files and functions into one inputs bundle. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_discovered_project_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_discovered_project_inputs", + options, + ( + &[ + "project_dir", + "project_config", + "local_config", + "sql_analysis_enabled", + ], + &["extract_output_column_locations"], + ), + )?; + discovered_inputs( + py, + ( + &arguments.value("project_dir")?, + &arguments.value("project_config")?, + &arguments.value("local_config")?, + ), + ( + arguments.flag("sql_analysis_enabled")?, + !arguments.passed("extract_output_column_locations")? + || arguments.flag("extract_output_column_locations")?, + ), + ) +} + +/// `build_discovered_project_inputs` for a loaded configuration. +fn discovered_inputs<'py>( + py: Python<'py>, + (project_dir, project_config, local_config): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + flags: (bool, bool), +) -> PyResult> { + let declarations: Bound<'py, PyDict> = within( + &lifecycle(py, "discovery_declaration_parse")?, + |declaration_lifecycle| { + let declarations: Bound<'py, PyDict> = declaration_files(project_dir, flags)?; + let mut item_count: usize = 0; + for field in DECLARATION_COLLECTIONS { + item_count += declarations.as_any().get_item(field)?.len()?; + } + item_count += usize::from(!declarations.as_any().get_item("adapter_file")?.is_none()); + completed(declaration_lifecycle, item_count)?; + Ok(declarations) + }, + )?; + let python: Bound<'py, PyDict> = within( + &lifecycle(py, "discovery_python_import")?, + |python_lifecycle| { + let python: Bound<'py, PyDict> = imported_python(project_dir)?; + completed(python_lifecycle, imported_python_paths(&python)?)?; + Ok(python) + }, + )?; + project_inputs( + (project_dir, project_config, local_config), + &declarations, + &python, + ) +} + +/// Report a lifecycle's completion with its item count. +fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { + let py: Python<'_> = lifecycle.py(); + let metadata: Bound<'_, PyDict> = keywords( + py, + &[("item_count", item_count.into_pyobject(py)?.into_any())], + )?; + call_with( + &lifecycle.getattr("completed")?, + &[("metadata", metadata.into_any())], + )?; + Ok(()) +} + +/// `_discover_declaration_files`: every declaration file under one directory snapshot. +fn declaration_files<'py>( + project_dir: &Bound<'py, PyAny>, + (sql_analysis_enabled, extract_output_column_locations): (bool, bool), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let directory: [(&str, Bound<'py, PyAny>); 1] = [("project_dir", project_dir.clone())]; + let snapshot: Bound<'py, PyAny> = call_with( + &imported(py, SNAPSHOT_MODULE, "DirectorySnapshot")?.getattr("scope")?, + &directory, + )?; + within(&snapshot, |_| { + let files: Bound<'py, PyDict> = PyDict::new(py); + files.set_item( + "source_files", + call_with( + &imported(py, YAML_FILES_MODULE, "discover_native_source_files")?, + &directory, + )?, + )?; + files.set_item( + "model_files", + call_with( + &imported(py, MODEL_FILES_MODULE, "discover_native_model_files")?, + &[ + ("project_dir", project_dir.clone()), + ( + "extract_implicit_alias_columns", + PyBool::new(py, sql_analysis_enabled).to_owned().into_any(), + ), + ( + "extract_output_column_locations", + PyBool::new(py, extract_output_column_locations) + .to_owned() + .into_any(), + ), + ], + )?, + )?; + for (field, module, name) in FILE_DISCOVERERS.iter().chain(LATE_FILE_DISCOVERERS.iter()) { + files.set_item(*field, call_with(&imported(py, module, name)?, &directory)?)?; + } + Ok(files) + }) +} + +/// Providers, then every kind of imported project Python that may use them. +fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let providers: Bound<'py, PyAny> = call_with( + &imported(py, FILESYSTEM_MODULE, "discover_provider_classes")?, + &[("project_dir", project_dir.clone())], + )?; + let python: Bound<'py, PyDict> = PyDict::new(py); + python.set_item("providers", &providers)?; + for (field, module, name) in PYTHON_DISCOVERERS { + python.set_item( + field, + call_with( + &imported(py, module, name)?, + &[ + ("project_dir", project_dir.clone()), + ("providers", providers.clone()), + ], + )?, + )?; + } + Ok(python) +} + +/// How many distinct project Python files the imported discoveries came from. +fn imported_python_paths(python: &Bound<'_, PyDict>) -> PyResult { + let py: Python<'_> = python.py(); + let python: &Bound<'_, PyAny> = python.as_any(); + let paths: Bound<'_, PySet> = PySet::empty(py)?; + let mut groups: Vec> = vec![python.get_item("providers")?]; + let nodes: Bound<'_, PyAny> = python.get_item("python_nodes")?; + for group in PYTHON_NODE_GROUPS { + groups.push(nodes.getattr(group)?); + } + for field in [ + "materialization_files", + "hook_functions", + "event_exporters", + "command_output_sinks", + ] { + groups.push(python.get_item(field)?); + } + for group in groups { + for item in group.try_iter()? { + paths.add(item?.getattr("relative_path")?)?; + } + } + Ok(paths.len()) +} + +/// The `DiscoveredProjectInputs` of the discovered declarations and imported Python. +fn project_inputs<'py>( + (project_dir, project_config, local_config): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + declarations: &Bound<'py, PyDict>, + python: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let declarations: &Bound<'py, PyAny> = declarations.as_any(); + let python: &Bound<'py, PyAny> = python.as_any(); + let nodes: Bound<'py, PyAny> = python.get_item("python_nodes")?; + let tuple_of = |group: &str| -> PyResult> { + Ok(PyTuple::new( + py, + nodes + .getattr(group)? + .try_iter()? + .collect::>>()?, + )? + .into_any()) + }; + let integration_loaders: Bound<'py, PyAny> = + imported(py, LOADERS_MODULE, "build_integration_loader_functions")? + .call1((declarations.get_item("source_files")?,))?; + let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?.add(integration_loaders)?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("project_dir", project_dir.clone()), + ]; + for field in [ + "model_files", + "enum_files", + "constant_files", + "model_schema_files", + "sql_function_files", + "sql_hook_files", + "python_function_files", + "schema_files", + "source_files", + "seed_files", + "test_files", + "scenario_files", + "audit_files", + "macro_files", + ] { + fields.push((field, declarations.get_item(field)?)); + } + fields.push(( + "materialization_files", + python.get_item("materialization_files")?, + )); + fields.push(("loader_functions", loader_functions)); + fields.push(("task_functions", tuple_of("tasks")?)); + fields.push(("asset_functions", tuple_of("assets")?)); + fields.push(("check_functions", tuple_of("checks")?)); + fields.push(("audit_factories", tuple_of("audit_factories")?)); + for field in [ + "hook_functions", + "event_exporters", + "command_output_sinks", + "providers", + ] { + fields.push((field, python.get_item(field)?)); + } + fields.push(("adapter_file", declarations.get_item("adapter_file")?)); + fields.push(("native_session", declarations.get_item("native_session")?)); + call_with( + &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredProjectInputs")?, + &fields, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_project_inputs, module)?)?; + module.add_function(wrap_pyfunction!(build_discovered_project_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 348234e24d..e2099ed7ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod analysis; pub(crate) mod command; +pub(crate) mod discovery; pub(crate) mod inputs; pub(crate) mod phases; pub(crate) mod project; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index c5fd1c60d0..d865247142 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_driver::{analysis, command, inputs, project}; +use crate::bindings::_helpers::compile_driver::{analysis, command, discovery, inputs, project}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; command::register(module)?; project::register(module)?; - inputs::register(module) + inputs::register(module)?; + discovery::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 1e21e97d0c..44e873016c 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 136064 +remaining_lines = 135761 [[stays]] path = "executor" @@ -223,3 +223,11 @@ note = "An undecodable cached finding is a cache miss; the rules run again nativ [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." + +[[handled_native_errors]] +location = "cli/commands/_helpers/cost/refresh.py::refresh_pending_cost_run" +note = "Best-effort cost refresh keeps its existing catch-all; discovery is now entered through the native driver, and no Python implementation runs instead." + +[[handled_native_errors]] +location = "compiler/scopes/main/load_or_build_scope_index.py::load_or_build_scope_index" +note = "Project input errors from discovery and compile-input stages select the tolerant scope index as before; a native compiler failure is a RuntimeError and is not caught." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index c0eddf8ab9..e8d00f93b9 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -277,7 +277,6 @@ "src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py", "src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py", "src/sqlbuild/compiler/discovery/_helpers/yml/project.py", - "src/sqlbuild/compiler/discovery/main/discover.py", "src/sqlbuild/compiler/discovery/main/runtime_extensions.py", "src/sqlbuild/compiler/discovery/models.py", "src/sqlbuild/provider/_helpers/session.py", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 63febcc533..9dc676f797 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -972,6 +972,20 @@ def build_project_graph( on_progress: Callable[[str], None] | None = None, no_cache: bool = False, ) -> ProjectGraph: ... +def discover_project_inputs( + *, + project_dir: Path, + sql_analysis_enabled_override: bool | None = None, + extract_output_column_locations: bool = True, +) -> DiscoveredProjectInputs: ... +def build_discovered_project_inputs( + *, + project_dir: Path, + project_config: ProjectConfig, + local_config: LocalConfig, + sql_analysis_enabled: bool, + extract_output_column_locations: bool = True, +) -> DiscoveredProjectInputs: ... def build_compile_inputs( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py index f37e4ee86c..f65660587e 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config @@ -19,7 +20,6 @@ ) from sqlbuild.cli.progress.models import CommandProgressReporters from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py index 599639cc66..932fc9b368 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py @@ -6,13 +6,13 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.clone.validation import validate_clone_request from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py index b5c37ff3cf..f3c3619242 100644 --- a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py +++ b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py @@ -3,6 +3,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import ( resolve_project_connection_config, @@ -10,7 +11,6 @@ from sqlbuild.cli.commands.constants import COST_HISTORY_SELECTOR, COST_LATEST_SELECTOR from sqlbuild.cli.commands.models import CostCommandRequest from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py index 85cade7924..5bcbb73e0d 100644 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/debug/checks.py @@ -7,6 +7,7 @@ from importlib.metadata import PackageNotFoundError, version from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import ( @@ -39,7 +40,6 @@ LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, diff --git a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py index db4f00b109..fce0894b86 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py @@ -4,9 +4,9 @@ from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffIncompleteError from sqlbuild.cli.commands.models import DiffCommandRequest, DiffInvocation -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.discover_configuration import ( discover_project_configuration, ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 229b43313b..5264a0bf7b 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( @@ -11,7 +12,6 @@ JanitorInvocation, JanitorSettings, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py index 363cc8cbad..e44ccc91ae 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py @@ -106,9 +106,8 @@ def _prepare_graph( ) if cached is not None: return cached, None, parsed_depth - from sqlbuild._native import build_project_graph + from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=project_dir, @@ -216,7 +215,7 @@ def _render_lineage( if error.code == UNKNOWN_SELECTOR_ERROR_CODE and isinstance( graph, RelationLineageIndex ): - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs + from sqlbuild._native import discover_project_inputs reject_unit_test_selectors( select=request.select, diff --git a/src/sqlbuild/cli/commands/_helpers/lint/selection.py b/src/sqlbuild/cli/commands/_helpers/lint/selection.py index ef0b456211..63fe30b415 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/selection.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -12,7 +12,6 @@ from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors diff --git a/src/sqlbuild/cli/commands/_helpers/load/invocation.py b/src/sqlbuild/cli/commands/_helpers/load/invocation.py index 64ef7df106..0a522f3140 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/load/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.cli.commands._helpers.load.selection import ( select_load_entries, @@ -15,7 +16,6 @@ from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 9123cc88fc..d331b6c0aa 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -20,7 +21,6 @@ ) from sqlbuild.cli.progress.models import CommandProgressReporters from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index 752fc1e08b..b22b1aca8c 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -7,7 +7,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( @@ -17,7 +17,6 @@ ) from sqlbuild.cli.commands.models import RulesCommandRequest from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py index 768ff9be5f..180f99d001 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Protocol, TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -16,7 +17,6 @@ build_command_progress_reporters, ) from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index 6b8b7563a0..f014cde75c 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -37,7 +38,6 @@ from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index cc0f8bebc4..7f4f6f86e9 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -54,7 +55,6 @@ from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline diff --git a/src/sqlbuild/cli/commands/main/execution/_freshness.py b/src/sqlbuild/cli/commands/main/execution/_freshness.py index 6713ecbef8..93bb69164b 100644 --- a/src/sqlbuild/cli/commands/main/execution/_freshness.py +++ b/src/sqlbuild/cli/commands/main/execution/_freshness.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.observe import ( observe_source_freshness_for_command, @@ -30,7 +30,6 @@ FreshnessCommandResult, ) from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py index 08e25a3c72..545e5315f8 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ b/src/sqlbuild/cli/commands/main/inspection/_contract.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.cli.commands._helpers.contract.progress import ( inspect_contracts_with_progress, write_contract_updates_with_progress, @@ -28,7 +28,6 @@ ) from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.contract_adoption.types import ContractAction -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project_graph import build_project_graph_from_compiled_project from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph diff --git a/src/sqlbuild/cli/commands/main/inspection/_query.py b/src/sqlbuild/cli/commands/main/inspection/_query.py index 807a9eda40..4c5ad455a2 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_query.py +++ b/src/sqlbuild/cli/commands/main/inspection/_query.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import QueryResult @@ -14,7 +15,6 @@ resolve_project_connection_config, ) from sqlbuild.cli.commands.exceptions import CliUserError, QueryExecutionError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, diff --git a/src/sqlbuild/cli/commands/main/project/_dag.py b/src/sqlbuild/cli/commands/main/project/_dag.py index f42b5d8670..2502ffef2c 100644 --- a/src/sqlbuild/cli/commands/main/project/_dag.py +++ b/src/sqlbuild/cli/commands/main/project/_dag.py @@ -6,12 +6,12 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph diff --git a/src/sqlbuild/cli/commands/main/project/_format.py b/src/sqlbuild/cli/commands/main/project/_format.py index 1ff842a3de..cb4e5fb99a 100644 --- a/src/sqlbuild/cli/commands/main/project/_format.py +++ b/src/sqlbuild/cli/commands/main/project/_format.py @@ -8,6 +8,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.lint.runs import ( prepare_lint_run, @@ -16,7 +17,6 @@ ) from sqlbuild.cli.commands._helpers.lint.selection import resolve_lint_inputs from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.main.has_fixture_typed_null_candidates import ( has_fixture_typed_null_candidates, diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index ccb8a873a8..ebc498b355 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -5,6 +5,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, @@ -24,7 +25,6 @@ SqlExpansionContext, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/compiler/contract_adoption/main/write.py b/src/sqlbuild/compiler/contract_adoption/main/write.py index ae108c3b71..cbd553b741 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/write.py @@ -5,7 +5,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, @@ -25,7 +25,6 @@ ContractEvidence, ContractFinding, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.main.write_atomically import write_atomically diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index 23ce3b53b9..e3bd07712c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -5,32 +5,15 @@ from collections.abc import Callable from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( - discover_command_output_sink_functions, -) from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, - discover_event_exporter_functions, - discover_hook_functions, discover_macro_files, - discover_materialization_files, discover_model_schema_files, - discover_provider_classes, - discover_python_function_files, - discover_python_node_functions, - discover_seed_files, discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( - build_integration_loader_functions, -) -from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( - retained_discovery_session, -) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) @@ -39,196 +22,24 @@ discover_native_test_files, ) from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_schema_files, discover_native_source_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredAuditFactory, - DiscoveredCheckFunction, - DiscoveredCommandOutputSink, DiscoveredConstantFile, - DiscoveredDeclarationFiles, DiscoveredEnumFile, - DiscoveredEventExporter, - DiscoveredHookFunction, - DiscoveredLoaderFunction, DiscoveredMacroFile, - DiscoveredMaterializationFile, DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredPythonNodeFunctions, - DiscoveredSourceFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestFile, - DiscoveredTaskFunction, DiscoveryFileFault, TolerantScopeDiscovery, ) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -def build_discovered_project_inputs( - *, - project_dir: Path, - project_config: ProjectConfig, - local_config: LocalConfig, - sql_analysis_enabled: bool, - extract_output_column_locations: bool = True, -) -> DiscoveredProjectInputs: - """Discover all project files and functions into one inputs bundle.""" - - with OperationLifecycle( - operation_kind="project", operation_name="discovery_declaration_parse" - ) as declaration_lifecycle: - declarations: DiscoveredDeclarationFiles = _discover_declaration_files( - project_dir=project_dir, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - declaration_lifecycle.completed( - metadata={ - "item_count": sum( - len(files) - for files in ( - declarations.source_files, - declarations.model_files, - declarations.enum_files, - declarations.constant_files, - declarations.model_schema_files, - declarations.sql_function_files, - declarations.sql_hook_files, - declarations.python_function_files, - declarations.schema_files, - declarations.seed_files, - declarations.test_files, - declarations.scenario_files, - declarations.audit_files, - declarations.macro_files, - ) - ) - + int(declarations.adapter_file is not None) - } - ) - with OperationLifecycle( - operation_kind="project", operation_name="discovery_python_import" - ) as python_lifecycle: - providers: tuple[DiscoveredProvider, ...] = discover_provider_classes( - project_dir=project_dir - ) - python_nodes: DiscoveredPythonNodeFunctions = discover_python_node_functions( - project_dir=project_dir, - providers=providers, - ) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = ( - discover_materialization_files( - project_dir=project_dir, - providers=providers, - ) - ) - hook_functions: tuple[DiscoveredHookFunction, ...] = discover_hook_functions( - project_dir=project_dir, - providers=providers, - ) - event_exporters: tuple[DiscoveredEventExporter, ...] = discover_event_exporter_functions( - project_dir=project_dir, providers=providers - ) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ( - discover_command_output_sink_functions( - project_dir=project_dir, - providers=providers, - ) - ) - python_paths: set[Path] = {provider.relative_path for provider in providers} - python_paths.update(node.relative_path for node in python_nodes.loaders) - python_paths.update(node.relative_path for node in python_nodes.tasks) - python_paths.update(node.relative_path for node in python_nodes.assets) - python_paths.update(node.relative_path for node in python_nodes.checks) - python_paths.update(factory.relative_path for factory in python_nodes.audit_factories) - python_paths.update(file.relative_path for file in materialization_files) - python_paths.update(function.relative_path for function in hook_functions) - python_paths.update(exporter.relative_path for exporter in event_exporters) - python_paths.update(sink.relative_path for sink in command_output_sinks) - python_lifecycle.completed(metadata={"item_count": len(python_paths)}) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = tuple( - python_nodes.loaders - ) + build_integration_loader_functions(declarations.source_files) - task_functions: tuple[DiscoveredTaskFunction, ...] = tuple(python_nodes.tasks) - asset_functions: tuple[DiscoveredAssetFunction, ...] = tuple(python_nodes.assets) - check_functions: tuple[DiscoveredCheckFunction, ...] = tuple(python_nodes.checks) - audit_factories: tuple[DiscoveredAuditFactory, ...] = tuple(python_nodes.audit_factories) - return DiscoveredProjectInputs( - project_config=project_config, - local_config=local_config, - project_dir=project_dir, - model_files=declarations.model_files, - enum_files=declarations.enum_files, - constant_files=declarations.constant_files, - model_schema_files=declarations.model_schema_files, - sql_function_files=declarations.sql_function_files, - sql_hook_files=declarations.sql_hook_files, - python_function_files=declarations.python_function_files, - schema_files=declarations.schema_files, - source_files=declarations.source_files, - seed_files=declarations.seed_files, - test_files=declarations.test_files, - scenario_files=declarations.scenario_files, - audit_files=declarations.audit_files, - macro_files=declarations.macro_files, - materialization_files=materialization_files, - loader_functions=loader_functions, - task_functions=task_functions, - asset_functions=asset_functions, - check_functions=check_functions, - audit_factories=audit_factories, - hook_functions=hook_functions, - event_exporters=event_exporters, - command_output_sinks=command_output_sinks, - providers=providers, - adapter_file=declarations.adapter_file, - native_session=declarations.native_session, - ) - - -def _discover_declaration_files( - *, - project_dir: Path, - sql_analysis_enabled: bool, - extract_output_column_locations: bool, -) -> DiscoveredDeclarationFiles: - with DirectorySnapshot.scope(project_dir=project_dir): - source_files: tuple[DiscoveredSourceFile, ...] = discover_native_source_files( - project_dir=project_dir - ) - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - return DiscoveredDeclarationFiles( - source_files=source_files, - model_files=model_files, - enum_files=discover_enum_files(project_dir=project_dir), - constant_files=discover_constant_files(project_dir=project_dir), - model_schema_files=discover_model_schema_files(project_dir=project_dir), - sql_function_files=discover_sql_function_files(project_dir=project_dir), - sql_hook_files=discover_sql_hook_files(project_dir=project_dir), - python_function_files=discover_python_function_files(project_dir=project_dir), - schema_files=discover_native_schema_files(project_dir=project_dir), - seed_files=discover_seed_files(project_dir=project_dir), - test_files=discover_native_test_files(project_dir=project_dir), - scenario_files=discover_native_scenario_files(project_dir=project_dir), - audit_files=discover_audit_files(project_dir=project_dir), - macro_files=discover_macro_files(project_dir=project_dir), - adapter_file=discover_adapter_file(project_dir=project_dir), - native_session=retained_discovery_session(project_dir=project_dir), - ) - - def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py deleted file mode 100644 index 5b387ef28a..0000000000 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Discovery entrypoints.""" - -from __future__ import annotations - -from functools import partial -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( - build_discovered_project_inputs, -) -from sqlbuild.compiler.discovery._helpers.filesystem.python_paths import ( - validate_project_python_paths, -) -from sqlbuild.compiler.discovery._helpers.validation.discovery import validate_discovered_inputs -from sqlbuild.compiler.discovery._helpers.validation.target_connections import ( - validate_target_connections, -) -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, - validate_local_sql_analysis_policy, - validate_target_warehouses_adapter, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier -from sqlbuild.compiler.frontier.types import CompilerStage -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def discover_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None = None, - extract_output_column_locations: bool = True, -) -> DiscoveredProjectInputs: - """Load all raw project inputs from disk before semantic resolution.""" - - with OperationLifecycle(operation_kind="project", operation_name="project_discovery"): - return compile_frontier( - until=CompilerStage.DISCOVERED_PROJECT_INPUTS, - python_stage=partial( - _discover_project_inputs, - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ), - ) - - -def _discover_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None, - extract_output_column_locations: bool, -) -> DiscoveredProjectInputs: - with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): - return _assemble_discovered_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ) - - -def _assemble_discovered_project_inputs( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None, - extract_output_column_locations: bool, -) -> DiscoveredProjectInputs: - project_config: ProjectConfig = load_project_config(project_dir=project_dir) - local_config: LocalConfig = load_local_config(project_dir=project_dir) - validate_local_sql_analysis_policy( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_target_warehouses_adapter( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_target_connections( - project_dir=project_dir, project_config=project_config, local_config=local_config - ) - validate_project_python_paths(project_dir=project_dir) - sql_analysis_enabled: bool = ( - sql_analysis_enabled_override - if sql_analysis_enabled_override is not None - else ( - local_config.settings.sql_analysis - if SQL_ANALYSIS_SETTING_KEY in local_config.setting_overrides - else project_config.settings.sql_analysis - ) - ) - discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - validate_discovered_inputs(discovered_inputs) - from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( - configure_discovered_event_exporters, - ) - - _ = configure_discovered_event_exporters(discovered_inputs) - return discovered_inputs diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index 2228310de3..aa90f13b44 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -5,7 +5,7 @@ from functools import partial from pathlib import Path -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( @@ -13,7 +13,6 @@ ) from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 77ce4fd6b9..d25ac65540 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -8,10 +8,9 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import dbt_failure_detail from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py index 0a835ab1e6..8b49a3203f 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.config._parse_config_overrides import ( diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index d0235ec799..4c7480a36a 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -7,6 +7,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN @@ -21,7 +22,6 @@ ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 2fbbda280c..f07eda5c10 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -6,9 +6,9 @@ from pathlib import Path from tempfile import TemporaryDirectory +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine.config import load_rules_config diff --git a/src/sqlbuild/runtime/event_exporting/main/configure_discovered_event_exporters.py b/src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py similarity index 100% rename from src/sqlbuild/runtime/event_exporting/main/configure_discovered_event_exporters.py rename to src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 4cc8af7813..66259d0b1d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -13,6 +13,7 @@ from _pytest.monkeypatch import MonkeyPatch from duckdb import DuckDBPyConnection +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.load.selection import select_load_entries @@ -32,7 +33,6 @@ TestCommandRequest, ) from sqlbuild.cli.output.main._load_execution_json import format_load_execution_json -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index b5df03e7c4..f08738544b 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,7 +16,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main @@ -62,7 +62,6 @@ ModelSqlAnalysis, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompileInputsView, ModelInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from sqlbuild.sql_values.types import CollectionRendering diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 13908ab46e..0895685c2b 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -14,7 +14,7 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -25,7 +25,6 @@ MacroExpansionResult, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 4d464da3a7..34c578cd0a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -13,7 +13,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds @@ -36,7 +36,6 @@ CompileInputsView, ) from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 08cd87cdac..411527b014 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -8,12 +8,12 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index 383b963d48..f1b4bb5061 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -6,14 +6,13 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 8ead2728e1..56de61f101 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -12,7 +12,7 @@ import pytest import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -26,7 +26,6 @@ from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType from sqlbuild.spec.contracts.models import ( diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index a5413fe7f3..1917af97ec 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -20,6 +20,7 @@ from typing import Any, cast from sqlbuild import _native +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( build_tolerant_scope_discovery, ) @@ -45,7 +46,6 @@ from sqlbuild.compiler.discovery.main._model_description_inputs import ( discover_model_description_inputs, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 47249a2c5f..5e1eef9078 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -8,7 +8,7 @@ import pytest from sqlbuild import _native -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py index adef309087..784ea0cd32 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.exceptions import ProjectPathError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( UndecodableDeclarationNameTestCase, UndecodablePathToleranceTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index b2f58fbc6a..6cc274642d 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( @@ -16,7 +16,6 @@ LiteralSqlRelation, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.sql_values.types import CollectionRendering _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index a028d5803e..d9dbdbe177 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -10,12 +10,11 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( build_native_column_lineage, ) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index b65f9562fb..fa3f41315b 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -12,14 +12,13 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( resolve_effective_collection_rendering, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 0849576df7..8a1b90ec7a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,7 +8,7 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter @@ -18,7 +18,6 @@ CompiledLineageColumnFact, ) from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index 3ae7073678..b8323417ee 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.diff import ( compile_project_for_diff_environment, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index c53e80cd4f..72833e0de9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -5,12 +5,12 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index af90e57ef7..cddfc0b088 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -7,13 +7,12 @@ import pytest -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.pipeline.main import compile as compile_pipeline diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index ac470932cd..14926d931c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main @@ -15,7 +15,6 @@ from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( NativeCatalogCase, ValidationCacheCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index c4f925d99d..201b39b40c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -6,11 +6,10 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SnowflakeOutputInferenceCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index d5fc551a24..fd81e00a7d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -8,14 +8,13 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( FunctionArgumentTypeCase, SemanticTriageCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index b0641ccc5f..40b7a2d4a5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -9,12 +9,12 @@ import pytest from duckdb import sqltypes +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SetOperationLineageIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index a7d2d4464d..ad62fbbfc4 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -5,13 +5,13 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ExpectedCountTestCase diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 3ef08e7615..a008d35f29 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, @@ -16,7 +16,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index eb63b583cc..3f72f75288 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 94fec55212..f09647bc4a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import Any, cast +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -37,7 +38,6 @@ ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSchemaFile, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index e7f6608e56..9d1705a5e4 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,7 +13,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly import project as project_module from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project @@ -23,7 +23,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 451d1d21c5..fd1c905a1e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -13,6 +13,7 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope @@ -26,7 +27,6 @@ LoadedMacro, ) from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredProjectInputs, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 6bdde6b886..207414d63b 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -15,7 +15,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -25,7 +25,6 @@ from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index e1a86455e8..86af91738f 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -13,14 +13,13 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssembly, NativeSqlTestAssemblyRequest, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 886917a107..d08ac209b5 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,9 +7,9 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 9785765a12..d1a4156881 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 4447ccf2d0..58559e1169 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -425,8 +425,8 @@ def run_scheduler_build( ) -> tuple[Any, Any]: """Run a full build through the scheduler with custom materializations.""" + from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode - from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index e515555b70..aaf139bc08 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,10 +10,10 @@ import duckdb +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 011b11dff5..5e541e1b05 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,8 +13,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index ebac3b9955..80f2fc0357 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index fca78a5ce8..e3a0c63ed0 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,10 +5,9 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( build_dbt_combined_graph, diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 7aa3d7a5b4..27d3dc8921 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index 0a4772bf8f..95b17b9c0c 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -5,11 +5,11 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index f3901cd630..e66cf8ef28 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -5,10 +5,10 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine import custom_rules, fact_replay from sqlbuild.rule_engine._helpers.engine.fact_replay import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py index d88a23fb5c..af05cab4c6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py @@ -6,10 +6,10 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.exceptions import SpecConfigError from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 1a73fdec19..16f9d7d814 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -9,6 +9,7 @@ import pytest +import sqlbuild._native as discovery_entry from sqlbuild.cli.commands._helpers.compile import lineage as compile_lineage from sqlbuild.cli.commands._helpers.compile import target_writer from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters @@ -17,7 +18,6 @@ from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.discovery.main import discover as discovery_entry from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.presentation.classes import transient_status_reporter diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 2369707552..5abc2b166c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -3,7 +3,7 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.model_config import ( build_native_model_config, native_model_config_session, @@ -15,7 +15,6 @@ NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from sqlbuild.spec.contracts.models import ( DefaultsConfig, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 2d1ebf2fb0..9238a53fa7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -4,7 +4,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -49,7 +49,6 @@ CompileInputsView, ModelInput, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredMacroFile, DiscoveredProjectInputs, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index f6e1775f28..b28042ddf6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.models import ( @@ -19,7 +19,6 @@ CompiledSeedView, CompileInputsView, ) -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AssembleCompiledProjectEffectiveTargetTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index df2f5f8ca9..4facad6a36 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -7,10 +7,9 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 7e8c11d358..f0e082241b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,11 +9,10 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSourceFile, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 08a1278e14..d18c693fa4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.types import ( @@ -17,7 +17,6 @@ ModelInput, ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 741e2e9f9d..d78c7151d1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project @@ -20,7 +20,6 @@ CompileInputsView, ) from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index ff1fe2a463..2df8110bb7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -7,13 +7,13 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 281ba89744..8235239f3c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index c422832da6..fe89344a7f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,12 +7,11 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompileInputsView, FunctionInput -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, PythonHookEntry, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index 304bff465b..33024ae81d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import build_compile_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index b06ee11b11..839e69e694 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -8,7 +8,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, build_dag_json +from sqlbuild._native import build_compile_inputs, build_dag_json, discover_project_inputs from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -16,7 +16,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index 7b19f83687..fa2086eb26 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.discovery._helpers.filesystem import aggregation +import sqlbuild._native as _native from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_audit_files, discover_constant_files, @@ -168,7 +168,7 @@ def write_scoped_project(*, root: Path) -> None: def isolated_discovery_mismatches(*, project_dir: Path) -> tuple[str, ...]: """Return categories where one shared-snapshot pass differs from isolated discovery.""" - inputs: DiscoveredProjectInputs = aggregation.build_discovered_project_inputs( + inputs: DiscoveredProjectInputs = _native.build_discovered_project_inputs( project_dir=project_dir, project_config=load_project_config(project_dir=project_dir), local_config=load_local_config(project_dir=project_dir), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py index 9a02d85a88..44fe4ceeff 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_helpers import ( isolated_discovery_mismatches, model_paths, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index c329b98e23..de12cf40e1 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -8,6 +8,7 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_audit_files, discover_hook_functions, @@ -22,7 +23,6 @@ ) from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index fcf38552a4..d427cd28b2 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index a753905b74..340f686c7b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,12 +6,12 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, ) from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import TargetWarehousesConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 71d18333b1..bf96a4ee0f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.exceptions import ProjectConfigError, ProjectPythonPathError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestBlock from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.compiler.discovery._test_helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 00f0fa9c35..5b78667aeb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -10,11 +10,11 @@ import duckdb import pytest +from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( _list_source_candidates, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index c9f239e882..09c7298277 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild._native import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint._helpers.project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 6c42dc005f..5577218e17 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -10,8 +10,8 @@ import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module +from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index d67cdd7500..145414cc0e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -4,10 +4,9 @@ import pytest -from sqlbuild._native import build_project_graph +from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.rule_engine.main.load_config import load_rules_config From d2814fa7bbe177dbc9504a3876a49b45579617e8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:54:38 +0100 Subject: [PATCH 08/26] refactor: run the compile-and-plan pipeline in the native compile driver --- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/pipeline.rs | 420 ++++++++++++++++++ .../_helpers/compile_driver/registration.rs | 7 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 22 +- .../_helpers/build_planning/planning.py | 2 +- .../cli/commands/_helpers/plan/planning.py | 2 +- .../compiler/pipeline/main/_compile_phase.py | 97 ---- .../main/{compile.py => _plan_result.py} | 122 +---- .../compiler/pipeline/main/compile_only.py | 2 +- .../compiler/pipeline/main/static_command.py | 2 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 3 +- .../sqlbuild/compiler/pipeline/test_main.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 3 +- .../executor/build/concurrent/helpers.py | 2 +- .../concurrent/test_ordering_invariant.py | 2 +- .../sqlbuild/executor/build/custom/helpers.py | 2 +- .../src/sqlbuild/executor/build/helpers.py | 2 +- .../build/snowflake_metadata/helpers.py | 2 +- .../executor/build/test_snapshot_execution.py | 2 +- .../main/test_compile_phase_timings.py | 27 +- 21 files changed, 480 insertions(+), 246 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs delete mode 100644 src/sqlbuild/compiler/pipeline/main/_compile_phase.py rename src/sqlbuild/compiler/pipeline/main/{compile.py => _plan_result.py} (54%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index e2099ed7ee..6ff32c54f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod command; pub(crate) mod discovery; pub(crate) mod inputs; pub(crate) mod phases; +pub(crate) mod pipeline; pub(crate) mod project; pub(crate) mod python_calls; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs new file mode 100644 index 0000000000..482282183c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -0,0 +1,420 @@ +//! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PySet, PySetMethods}; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const GRAPH_HELPERS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.graph"; +const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; +const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; +const RULES_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; +const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; +const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; +const PLAN_MODULE: &str = "sqlbuild.compiler.pipeline.main._plan_result"; +const GENERIC_DIALECT: &str = "generic"; + +/// The current build's phase timing tracker, when one is recording. +fn timing_tracker(py: Python<'_>) -> PyResult> { + imported(py, TIMING_TRACKER_MODULE, "BuildPhaseTimingTracker")?.call_method0("current") +} + +/// Report `message` through `on_progress` when a callback is set. +fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + if !on_progress.is_none() { + on_progress.call1((message,))?; + } + Ok(()) +} + +/// Compile a project without opening a warehouse planning connection. +#[pyfunction] +#[pyo3(signature = (**options))] +fn compile_project_phase<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "compile_project_phase", + options, + ( + &["discovered_inputs", "adapter"], + &["options", "on_progress"], + ), + )?; + compiled_phase( + ( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + ), + &arguments.value("options")?, + &arguments.value("on_progress")?, + ) +} + +/// `compile_project_phase` with resolved arguments. +fn compiled_phase<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + options: &Bound<'py, PyAny>, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let options: Bound<'py, PyAny> = if options.is_truthy()? { + options.clone() + } else { + imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + }; + let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; + let connection_config: Bound<'py, PyAny> = if configured.is_none() { + call_with( + &imported( + py, + EFFECTIVE_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", options.getattr("selected_target")?), + ("cli_vars", options.getattr("cli_vars")?), + ], + )? + } else { + configured + }; + progress(on_progress, "Compiling project...")?; + let started: f64 = monotonic(py)?; + let compiled: PyResult> = (|| { + let selection: Bound<'py, PyAny> = call_with( + &imported( + py, + ANALYSIS_SELECTION_MODULE, + "resolve_compile_analysis_selection", + )?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.getattr("selected_target")?), + ("no_sql_validation", options.getattr("no_sql_validation")?), + ("cli_vars", options.getattr("cli_vars")?), + ( + "external_sql_reference_resolver", + options.getattr("external_sql_reference_resolver")?, + ), + ("resolved_connection", connection_config.clone()), + ("analysis_selection", selection), + ], + ) + })(); + let compile_seconds: f64 = monotonic(py)? - started; + let tracker: Bound<'py, PyAny> = timing_tracker(py)?; + if !tracker.is_none() { + tracker.setattr("compile_seconds", compile_seconds)?; + } + let project: Bound<'py, PyAny> = compiled?; + reject_compile_errors(&project, &options, on_progress)?; + progress( + on_progress, + &format!("Compiled project. ({compile_seconds:.2}s)"), + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompiledProjectPhaseResult")?, + &[ + ("project", project), + ("connection_config", connection_config), + ( + "compile_seconds", + compile_seconds.into_pyobject(py)?.into_any(), + ), + ], + ) +} + +/// Raise one `CompileInputError` listing every compile error the command must stop on. +fn reject_compile_errors( + project: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, + on_progress: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = project.py(); + let plan_sql_tests: bool = options.getattr("plan_sql_tests")?.is_truthy()?; + let sql_test: Bound<'_, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_TEST")?; + let mut errors: Vec> = Vec::new(); + for diagnostic in project.getattr("diagnostics")?.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? + && (plan_sql_tests || diagnostic.getattr("resource_type")?.ne(&sql_test)?) + { + errors.push(diagnostic); + } + } + let Some(first) = errors.first() else { + return Ok(()); + }; + progress(on_progress, "Project compilation failed.")?; + let mut lines: Vec = Vec::with_capacity(errors.len()); + let helps: Bound<'_, PySet> = PySet::empty(py)?; + for diagnostic in &errors { + let or_one = |name: &str| -> PyResult { + let value: Bound<'_, PyAny> = diagnostic.getattr(name)?; + Ok(if value.is_truthy()? { + value.str()?.to_string() + } else { + "1".to_owned() + }) + }; + lines.push(format!( + "[{}] {}:{}:{}: {}", + diagnostic.getattr("code")?.str()?, + diagnostic.getattr("path")?.str()?, + or_one("line")?, + or_one("column")?, + diagnostic.getattr("message")?.str()?, + )); + helps.add(diagnostic.getattr("help")?)?; + } + let help: Bound<'_, PyAny> = if helps.len() == 1 { + helps.pop().unwrap_or_else(|| py.None().into_bound(py)) + } else { + py.None().into_bound(py) + }; + let error: Bound<'_, PyAny> = imported(py, COMPILE_EXCEPTIONS_MODULE, "CompileInputError")? + .call( + (lines.join("\n"),), + Some(&keywords( + py, + &[("code", first.getattr("code")?), ("help", help)], + )?), + )?; + Err(PyErr::from_value(error)) +} + +/// Run compile inputs, assembly, planning, and manifest generation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_compile_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_compile_pipeline", + options, + (&["discovered_inputs", "adapter"], &["options", "hooks"]), + )?; + let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_SELECTION_MODULE, "apply_unit_test_selectors")?, + &[ + ("options", arguments.value("options")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let hooks: Bound<'py, PyAny> = arguments.value("hooks")?; + let hooks: Bound<'py, PyAny> = if hooks.is_none() { + imported(py, RUNTIME_MODELS_MODULE, "ConnectionHooks")?.call0()? + } else { + hooks + }; + let on_progress: Bound<'py, PyAny> = hooks.getattr("on_progress")?; + let compiled: Bound<'py, PyAny> = + compiled_phase((&discovered, &adapter), &options, &on_progress)?; + let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let graph: Bound<'py, PyAny> = + imported(py, GRAPH_HELPERS_MODULE, "build_project_graph_impl")?.call1((&project,))?; + run_rules_gate( + (&discovered, &adapter), + (&options, &on_progress), + (&project, &graph), + )?; + planned((&discovered, &adapter), (&options, &hooks), &compiled) +} + +/// Run the configured rules and stop the command when any of them finds a fault. +fn run_rules_gate<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (options, on_progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, graph): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<()> { + let py: Python<'py> = discovered.py(); + let rules_error: Bound<'py, PyAny> = imported(py, RULE_EXCEPTIONS_MODULE, "RulesError")?; + let project_dir: Bound<'py, PyAny> = discovered.getattr("project_dir")?; + if project_dir.is_none() { + return Err(PyErr::from_value( + rules_error.call1(("compiler rules require a resolved project directory",))?, + )); + } + let mut config: Bound<'py, PyAny> = call_with( + &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + if options.getattr("no_cache")?.is_truthy()? { + let cache: Bound<'py, PyAny> = call_with( + &imported(py, RULE_MODELS_MODULE, "RulesCacheConfig")?, + &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], + )?; + config = imported(py, "dataclasses", "replace")? + .call((config,), Some(&keywords(py, &[("cache", cache)])?))?; + } + progress(on_progress, "Evaluating configured rules...")?; + let started: f64 = monotonic(py)?; + let dialect: Bound<'py, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = if dialect.is_truthy()? { + dialect + } else { + GENERIC_DIALECT.into_pyobject(py)?.into_any() + }; + let selected_keys: Bound<'py, PyAny> = call_with( + &imported( + py, + ANALYSIS_SELECTION_MODULE, + "resolve_configured_rule_selection", + )?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered.clone()), + ("project", project.clone()), + ], + )?; + let result: Bound<'py, PyAny> = call_with( + &imported(py, RUN_RULES_MODULE, "run_rules")?, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", config), + ("project_dir", project_dir), + ("dialect", dialect), + ("selected_keys", selected_keys), + ("no_sql_analysis", options.getattr("no_sql_validation")?), + ], + )?; + if !on_progress.is_none() { + let elapsed: f64 = monotonic(py)? - started; + let message: Bound<'py, PyAny> = call_with( + &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, + &[ + ("elapsed_seconds", elapsed.into_pyobject(py)?.into_any()), + ("result", result.clone()), + ( + "note_skipped_rules", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + on_progress.call1((message,))?; + } + let findings: Bound<'py, PyAny> = result.getattr("findings")?; + if !findings.is_truthy()? { + return Ok(()); + } + let summary: Bound<'py, PyAny> = call_with( + &imported(py, RULE_MODELS_MODULE, "RulesResult")?, + &[ + ("findings", findings), + ("evaluated_models", result.getattr("evaluated_models")?), + ( + "unevaluated_resources", + result.getattr("unevaluated_resources")?, + ), + ("cache_hits", result.getattr("cache_hits")?), + ("cache_misses", result.getattr("cache_misses")?), + ], + )?; + let rendered: Bound<'py, PyAny> = call_with( + &imported(py, RULES_RESULT_MODULE, "format_result")?, + &[ + ("result", summary), + ( + "json_output", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + Err(PyErr::from_value(rules_error.call1((format!( + "configured compiler rules failed:\n{}", + rendered.str()? + ),))?)) +} + +/// Open the planning connection, plan, and record the compile and planning times. +fn planned<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (options, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + compiled: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let tracker: Bound<'py, PyAny> = timing_tracker(py)?; + let started: f64 = monotonic(py)?; + let mut connection: Option> = None; + let result: PyResult> = (|| { + let opened: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, + &[ + ("adapter", adapter.clone()), + ("connection_config", compiled.getattr("connection_config")?), + ("hooks", hooks.clone()), + ], + )?; + connection = Some(opened.clone()); + call_with( + &imported(py, PLAN_MODULE, "build_pipeline_result")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("connection", opened), + ("project", compiled.getattr("project")?), + ("options", options.clone()), + ("on_progress", hooks.getattr("on_progress")?), + ], + ) + })(); + if let Some(connection) = connection.filter(|connection| !connection.is_none()) { + adapter.call_method1("close", (connection,))?; + } + let planning_seconds: f64 = monotonic(py)? - started; + if !tracker.is_none() { + tracker.setattr("planning_seconds", planning_seconds)?; + } + let result: Bound<'py, PyAny> = result?; + imported(py, "dataclasses", "replace")?.call( + (result,), + Some(&keywords( + py, + &[ + ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "planning_seconds", + planning_seconds.into_pyobject(py)?.into_any(), + ), + ], + )?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_project_phase, module)?)?; + module.add_function(wrap_pyfunction!(run_compile_pipeline, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index d865247142..d82b7caa52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -2,12 +2,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_driver::{analysis, command, discovery, inputs, project}; +use crate::bindings::_helpers::compile_driver::{ + analysis, command, discovery, inputs, pipeline, project, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; command::register(module)?; project::register(module)?; inputs::register(module)?; - discovery::register(module) + discovery::register(module)?; + pipeline::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 44e873016c..ef6ca609f9 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 135761 +remaining_lines = 135550 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9dc676f797..3e4ea25cd0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -72,7 +72,12 @@ from sqlbuild.compiler.discovery.models import ( TolerantScopeDiscovery, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability -from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.pipeline.models import ( + CompiledProjectPhaseResult, + CompilePipelineOptions, + CompilePipelineResult, + ProjectGraph, +) from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @@ -104,6 +109,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -986,6 +992,20 @@ def build_discovered_project_inputs( sql_analysis_enabled: bool, extract_output_column_locations: bool = True, ) -> DiscoveredProjectInputs: ... +def compile_project_phase( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions | None = None, + on_progress: Callable[[str], None] | None = None, +) -> CompiledProjectPhaseResult: ... +def run_compile_pipeline( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions | None = None, + hooks: ConnectionHooks | None = None, +) -> CompilePipelineResult: ... def build_compile_inputs( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py b/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py index bedc7c7dd5..4d2469c00a 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import run_compile_pipeline from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -10,7 +11,6 @@ BuildInvocation, ) from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py index 41fd7ca62b..b52075934b 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/planning.py @@ -2,11 +2,11 @@ from __future__ import annotations +from sqlbuild._native import run_compile_pipeline from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import ( CompilePipelineOptions, CompilePipelineResult, diff --git a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py deleted file mode 100644 index 274d369125..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Canonical project compilation phase.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( - resolve_compile_analysis_selection, -) -from sqlbuild.compiler.pipeline.models import CompiledProjectPhaseResult, CompilePipelineOptions -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker - - -def compile_project_phase( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions | None = None, - on_progress: Callable[[str], None] | None = None, -) -> CompiledProjectPhaseResult: - """Compile a project without opening a warehouse planning connection.""" - - resolved_options: CompilePipelineOptions = options or CompilePipelineOptions() - effective_config: dict[str, object] = ( - resolved_options.connection_config - if resolved_options.connection_config is not None - else build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=resolved_options.selected_target, - cli_vars=resolved_options.cli_vars, - ) - ) - if on_progress is not None: - on_progress("Compiling project...") - compile_start: float = time.monotonic() - try: - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=resolved_options.selected_target, - no_sql_validation=resolved_options.no_sql_validation, - cli_vars=resolved_options.cli_vars, - external_sql_reference_resolver=resolved_options.external_sql_reference_resolver, - resolved_connection=effective_config, - analysis_selection=resolve_compile_analysis_selection( - options=resolved_options, - discovered_inputs=discovered_inputs, - ), - ) - finally: - compile_seconds: float = time.monotonic() - compile_start - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.compile_seconds = compile_seconds - errors: tuple[CompilerDiagnostic, ...] = tuple( - diagnostic - for diagnostic in project.diagnostics - if diagnostic.is_error - and ( - resolved_options.plan_sql_tests - or diagnostic.resource_type != CompiledResourceType.SQL_TEST - ) - ) - if errors: - if on_progress is not None: - on_progress("Project compilation failed.") - raise CompileInputError( - "\n".join( - f"[{diagnostic.code}] {diagnostic.path}:" - f"{diagnostic.line or 1}:{diagnostic.column or 1}: {diagnostic.message}" - for diagnostic in errors - ), - code=errors[0].code, - help=_shared_help(errors), - ) - if on_progress is not None: - on_progress(f"Compiled project. ({compile_seconds:.2f}s)") - return CompiledProjectPhaseResult( - project=project, - connection_config=effective_config, - compile_seconds=compile_seconds, - ) - - -def _shared_help(errors: tuple[CompilerDiagnostic, ...]) -> str | None: - """Return the help every error shares, so one remedy is shown once.""" - - helps: frozenset[str | None] = frozenset(diagnostic.help for diagnostic in errors) - return next(iter(helps)) if len(helps) == 1 else None diff --git a/src/sqlbuild/compiler/pipeline/main/compile.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py similarity index 54% rename from src/sqlbuild/compiler/pipeline/main/compile.py rename to src/sqlbuild/compiler/pipeline/main/_plan_result.py index 7500686afe..31a26dfcf8 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile.py +++ b/src/sqlbuild/compiler/pipeline/main/_plan_result.py @@ -1,11 +1,8 @@ -"""Full compile-and-plan pipeline producing CLI artifacts.""" +"""Plan a compiled project over an open warehouse connection.""" from __future__ import annotations -import time from collections.abc import Callable -from dataclasses import replace -from pathlib import Path from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,10 +13,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( - apply_unit_test_selectors, - resolve_configured_rule_selection, -) from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( build_deferred_locations, gather_deferred_relations, @@ -31,12 +24,9 @@ build_python_run_plan_outputs, python_read_source_names_for, ) -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( - CompiledProjectPhaseResult, CompilePipelineOptions, CompilePipelineResult, - ProjectGraph, PythonRunPlanOutputs, ) from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan @@ -51,116 +41,10 @@ resolve_python_sql_run_selection_from_inputs, ) from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.render_result import format_result -from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress -from sqlbuild.rule_engine.main.run_rules import run_rules -from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult, RulesRunResult -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks -from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import TargetConfig -def run_compile_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions | None = None, - hooks: ConnectionHooks | None = None, -) -> CompilePipelineResult: - """Run compile inputs, assembly, planning, and manifest generation.""" - - resolved_options: CompilePipelineOptions = apply_unit_test_selectors( - options=options, discovered_inputs=discovered_inputs - ) - resolved_hooks: ConnectionHooks = hooks if hooks is not None else ConnectionHooks() - on_progress: Callable[[str], None] | None = resolved_hooks.on_progress - compile_result: CompiledProjectPhaseResult = compile_project_phase( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=resolved_options, - on_progress=on_progress, - ) - effective_config: dict[str, object] = compile_result.connection_config - project: CompiledProjectView = compile_result.project - compile_seconds: float = compile_result.compile_seconds - graph: ProjectGraph = build_project_graph_impl(project) - project_dir: Path | None = discovered_inputs.project_dir - if project_dir is None: - raise RulesError("compiler rules require a resolved project directory") - rules_config: RulesConfig = load_rules_config(project_dir=project_dir) - if resolved_options.no_cache: - rules_config = replace(rules_config, cache=RulesCacheConfig(enabled=False)) - if on_progress is not None: - on_progress("Evaluating configured rules...") - rules_start: float = time.monotonic() - rules_result: RulesRunResult = run_rules( - graph=graph, - discovered_inputs=discovered_inputs, - config=rules_config, - project_dir=project_dir, - dialect=adapter.sql_analysis_dialect() or "generic", - selected_keys=resolve_configured_rule_selection( - options=resolved_options, - discovered_inputs=discovered_inputs, - project=project, - ), - no_sql_analysis=resolved_options.no_sql_validation, - ) - if on_progress is not None: - on_progress( - format_rules_progress( - elapsed_seconds=time.monotonic() - rules_start, - result=rules_result, - note_skipped_rules=True, - ) - ) - if rules_result.findings: - rendered: str = format_result( - result=RulesResult( - findings=rules_result.findings, - evaluated_models=rules_result.evaluated_models, - unevaluated_resources=rules_result.unevaluated_resources, - cache_hits=rules_result.cache_hits, - cache_misses=rules_result.cache_misses, - ), - json_output=False, - ) - raise RulesError(f"configured compiler rules failed:\n{rendered}") - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - planning_start: float = time.monotonic() - connection: Any | None = None - result: CompilePipelineResult | None = None - try: - connection = open_connection_with_hooks( - adapter=adapter, - connection_config=effective_config, - hooks=resolved_hooks, - ) - result = _build_result( - discovered_inputs=discovered_inputs, - adapter=adapter, - connection=connection, - project=project, - options=resolved_options, - on_progress=on_progress, - ) - finally: - if connection is not None: - adapter.close(connection) - planning_seconds: float = time.monotonic() - planning_start - if timing_tracker is not None: - timing_tracker.planning_seconds = planning_seconds - return replace( - result, - compile_seconds=compile_seconds, - planning_seconds=planning_seconds, - ) - - -def _build_result( +def build_pipeline_result( *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, @@ -169,6 +53,8 @@ def _build_result( options: CompilePipelineOptions, on_progress: Callable[[str], None] | None = None, ) -> CompilePipelineResult: + """Plan the compiled project over an open connection and attach Python node plans.""" + select: tuple[str, ...] = options.select exclude: tuple[str, ...] = options.exclude deferred_locations: dict[str, CompiledRelationLocation] | None = None diff --git a/src/sqlbuild/compiler/pipeline/main/compile_only.py b/src/sqlbuild/compiler/pipeline/main/compile_only.py index 673f35162e..ea0416fe8f 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile_only.py +++ b/src/sqlbuild/compiler/pipeline/main/compile_only.py @@ -4,9 +4,9 @@ from collections.abc import Callable +from sqlbuild._native import compile_project_phase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, diff --git a/src/sqlbuild/compiler/pipeline/main/static_command.py b/src/sqlbuild/compiler/pipeline/main/static_command.py index 3ce77cf45c..df9c0e7689 100644 --- a/src/sqlbuild/compiler/pipeline/main/static_command.py +++ b/src/sqlbuild/compiler/pipeline/main/static_command.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import compile_project_phase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -12,7 +13,6 @@ build_deferred_locations, resolve_deferred_target_config, ) -from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 8a1b90ec7a..48d12bfbcf 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,7 +8,7 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter @@ -20,7 +20,6 @@ from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index cddfc0b088..020b6eebeb 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -15,11 +15,11 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.main import compile as compile_pipeline from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.runtime.contracts.main import open_connection as compile_pipeline from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index f09647bc4a..7c839aea91 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import Any, cast -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -47,7 +47,6 @@ from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index d08ac209b5..4f67a6e85f 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -11,7 +11,7 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline +from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index d1a4156881..3410f985c4 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -13,7 +13,7 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline +from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 58559e1169..4f26ab1359 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -428,7 +428,7 @@ def run_scheduler_build( from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline + from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index aaf139bc08..bd37c6a50a 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -15,7 +15,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline +from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 5e541e1b05..310eb93c04 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -16,7 +16,7 @@ from sqlbuild._native import discover_project_inputs from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline +from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index 80f2fc0357..8d2b39277b 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -11,7 +11,7 @@ from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compile import run_compile_pipeline +from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 3629e4b28c..68bdc913c3 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -2,19 +2,23 @@ from __future__ import annotations +import time from unittest.mock import Mock import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main import _compile_phase as compile_phase_module -from sqlbuild.compiler.pipeline.main import compile as compile_module +from sqlbuild.compiler.pipeline._helpers import analysis_selection +from sqlbuild.compiler.pipeline.main import _plan_result as plan_result from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings +from sqlbuild.rule_engine.main import load_config, run_rules from sqlbuild.rule_engine.models import RulesConfig, RulesRunResult +from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -59,20 +63,19 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di project=project, plan_output=PlanOutput() ) adapter: Mock = Mock() - monkeypatch.setattr(compile_module.time, "monotonic", monotonic) - monkeypatch.setattr(compile_phase_module.time, "monotonic", monotonic) - monkeypatch.setattr(compile_phase_module, "build_compiled_project", Mock(return_value=project)) + monkeypatch.setattr(time, "monotonic", monotonic) + monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) monkeypatch.setattr( - compile_phase_module, "resolve_compile_analysis_selection", Mock(return_value=None) + analysis_selection, "resolve_compile_analysis_selection", Mock(return_value=None) ) - monkeypatch.setattr(compile_module, "open_connection_with_hooks", Mock(return_value=object())) - monkeypatch.setattr(compile_module, "_build_result", Mock(return_value=pipeline_result)) - monkeypatch.setattr(compile_module, "load_rules_config", Mock(return_value=RulesConfig())) + monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) + monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) + monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) monkeypatch.setattr( - compile_module, "resolve_configured_rule_selection", Mock(return_value=None) + analysis_selection, "resolve_configured_rule_selection", Mock(return_value=None) ) monkeypatch.setattr( - compile_module, + run_rules, "run_rules", Mock( return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) @@ -86,7 +89,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di } with tracker.scope(): - result: CompilePipelineResult = compile_module.run_compile_pipeline( + result: CompilePipelineResult = _native.run_compile_pipeline( discovered_inputs=Mock(), adapter=adapter, options=CompilePipelineOptions(connection_config={}), From a8b6b5b847845062d9d8db72b06857925085c4e1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:51:09 +0100 Subject: [PATCH 09/26] refactor: build the compile manifest natively The dbt-style manifest is built in Rust and returned as a NativeManifest whose contents() are the exact bytes json.dumps(manifest, indent=2) + "\n" produced; encoding errors still raise from the writer. Python passes only hook identities, the plan output and the generation timestamp. The Python manifest builders are removed. --- .../src/dag_artifact/_helpers/checks.rs | 3 +- .../src/dag_artifact/_helpers/edges.rs | 3 +- .../src/dag_artifact/_helpers/nodes.rs | 7 +- .../src/dag_artifact/_helpers/values.rs | 36 +- .../src/dag_artifact/main/build_dag_json.rs | 3 +- .../src/dag_artifact/models.rs | 20 +- .../src/dag_artifact/tests/helpers.rs | 7 +- .../dag_artifact/tests/test_build_dag_json.rs | 3 +- .../src/dag_artifact/types.rs | 6 +- crates/sqlbuild-analysis/src/lib.rs | 2 + .../src/manifest/_helpers/macros.rs | 134 +++++ .../src/manifest/_helpers/mod.rs | 7 + .../src/manifest/_helpers/model_nodes.rs | 176 +++++++ .../src/manifest/_helpers/node_fields.rs | 201 ++++++++ .../src/manifest/_helpers/payload.rs | 156 ++++++ .../src/manifest/_helpers/resource_nodes.rs | 172 +++++++ .../src/manifest/_helpers/test_nodes.rs | 222 ++++++++ .../src/manifest/_helpers/values.rs | 79 +++ .../src/manifest/constants.rs | 27 + .../sqlbuild-analysis/src/manifest/errors.rs | 21 + .../src/manifest/main/build_manifest_json.rs | 29 ++ .../src/manifest/main/mod.rs | 1 + crates/sqlbuild-analysis/src/manifest/mod.rs | 8 + .../sqlbuild-analysis/src/manifest/models.rs | 236 +++++++++ .../sqlbuild-analysis/src/manifest/tests.rs | 6 + .../src/manifest/tests/helpers.rs | 96 ++++ .../tests/test_build_manifest_json.rs | 141 +++++ .../src/manifest/tests/test_types.rs | 15 + .../python_values/main/encode_python_value.rs | 30 ++ .../src/python_values/main/mod.rs | 2 + .../src/python_values/main/text_key.rs | 12 + .../src/python_values/mod.rs | 2 + .../src/python_values/models.rs | 28 + .../src/bindings/_helpers/dag_artifact/mod.rs | 1 - .../bindings/_helpers/dag_artifact/rows.rs | 10 +- .../src/bindings/_helpers/manifest/builder.rs | 126 +++++ .../src/bindings/_helpers/manifest/mod.rs | 3 + .../_helpers/manifest/registration.rs | 9 + .../src/bindings/_helpers/manifest/rows.rs | 484 ++++++++++++++++++ .../src/bindings/_helpers/mod.rs | 2 + .../bindings/_helpers/python_values/mod.rs | 1 + .../{dag_artifact => python_values}/values.rs | 65 ++- .../src/bindings/main/register.rs | 3 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 20 +- .../_helpers/build_planning/compile_target.py | 3 +- .../_helpers/compile/target_writer.py | 11 +- .../classes/prepared_compile_artifacts.py | 3 +- .../compiler/manifest/_helpers/__init__.py | 0 .../compiler/manifest/_helpers/macros.py | 114 ----- .../compiler/manifest/_helpers/model_nodes.py | 160 ------ .../compiler/manifest/_helpers/payload.py | 166 ------ .../compiler/manifest/_helpers/seeds.py | 79 --- .../compiler/manifest/_helpers/shared.py | 156 ------ .../compiler/manifest/_helpers/sources.py | 65 --- .../compiler/manifest/_helpers/tests.py | 180 ------- src/sqlbuild/compiler/manifest/constants.py | 22 - .../manifest/main/_build_graph_manifest.py | 16 +- src/sqlbuild/compiler/manifest/main/build.py | 8 +- .../sqlbuild/cli/commands/main/_test_types.py | 9 + .../cli/commands/main/test_build_manifest.py | 64 +++ .../src/sqlbuild/compiler/pipeline/helpers.py | 20 +- .../compile/test_compile_target_writer.py | 15 +- .../compiler/compile/_helpers/helpers.py | 17 + .../compile/_helpers/test_model_schemas.py | 14 +- .../sqlbuild/compiler/manifest/_test_types.py | 7 + .../src/sqlbuild/compiler/manifest/helpers.py | 6 +- .../sqlbuild/compiler/manifest/test_main.py | 63 ++- 68 files changed, 2730 insertions(+), 1085 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/_helpers/values.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/constants.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/errors.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/mod.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/models.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/tests.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs create mode 100644 crates/sqlbuild-analysis/src/manifest/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs create mode 100644 crates/sqlbuild-analysis/src/python_values/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_values/main/text_key.rs create mode 100644 crates/sqlbuild-analysis/src/python_values/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_values/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs rename crates/sqlbuild-python/src/bindings/_helpers/{dag_artifact => python_values}/values.rs (62%) delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/macros.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/model_nodes.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/payload.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/seeds.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/shared.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/sources.py delete mode 100644 src/sqlbuild/compiler/manifest/_helpers/tests.py delete mode 100644 src/sqlbuild/compiler/manifest/constants.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs index 3be61a3c2f..5f2e72c092 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/checks.rs @@ -11,8 +11,9 @@ use crate::dag_artifact::constants::{ use crate::dag_artifact::errors::DagArtifactError; use crate::dag_artifact::models::{ DagAuditInput, DagProjectInput, DagPythonGraphInput, DagPythonNodeInput, DagScenarioInput, - DagSqlTestInput, DagValue, + DagSqlTestInput, }; +use crate::dag_artifact::types::DagValue; /// Every `DagCheck` field, `None` or empty where a check kind leaves it out. #[derive(Default)] diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs index 1c73520b42..092e82a5c7 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs @@ -6,7 +6,8 @@ use crate::dag_artifact::_helpers::nodes::{ loader_or_source_id, node_id, python_node_id, sources_by_loader, }; use crate::dag_artifact::_helpers::values::{object, text}; -use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use crate::dag_artifact::types::DagValue; use crate::graph::models::ProjectGraph; /// Each Python node name's kind, as `nodes_by_name` maps it. diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs index 312b15f3fd..fbbd296122 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs @@ -16,9 +16,10 @@ use crate::dag_artifact::errors::DagArtifactError; use crate::dag_artifact::models::{ DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, DagProjectInput, DagPythonNodeInput, DagRelationInput, DagSeedInput, DagSourceEntryInput, DagSourceInput, - DagValue, }; use crate::dag_artifact::types::DagColumnLineage; +use crate::dag_artifact::types::DagValue; +use crate::python_values::main::text_key::text_key; /// Every `DagNode` field, `None` or empty where a node kind leaves it out. #[derive(Default)] @@ -68,7 +69,7 @@ impl NodeFields { DagValue::Object( self.column_lineage .iter() - .map(|(name, refs)| (Ok(name.clone()), lineage_refs(refs))) + .map(|(name, refs)| (text_key(name), lineage_refs(refs))) .collect(), ), ), @@ -283,7 +284,7 @@ pub(crate) fn loader_node( ))), path: Some(loader.path.clone()), meta: Some(DagValue::Object(vec![( - Ok(LOADER_NODE_META_KEY.to_owned()), + text_key(LOADER_NODE_META_KEY), DagValue::Bool(true), )])), columns: columns(&loader.columns), diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs index d9c45cb611..aa8c96dd3e 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/values.rs @@ -4,15 +4,18 @@ use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use crate::dag_artifact::constants::REQUIRED_COLLECTION_KEYS; use crate::dag_artifact::errors::DagArtifactError; -use crate::dag_artifact::models::{DagColumnInput, DagValue}; -use crate::dag_artifact::types::DagKey; +use crate::dag_artifact::models::DagColumnInput; +use crate::dag_artifact::types::DagValue; +use crate::python_values::main::encode_python_value::encode_python_value; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::PythonKey; /// An object whose keys are all field names. pub(crate) fn object(entries: Vec<(&str, DagValue)>) -> DagValue { DagValue::Object( entries .into_iter() - .map(|(key, value)| (Ok(key.to_owned()), value)) + .map(|(key, value)| (text_key(key), value)) .collect(), ) } @@ -62,8 +65,9 @@ pub(crate) fn typed_column((name, data_type): &(String, Option)) -> DagV ]) } -fn is_required_collection(key: &DagKey) -> bool { - key.as_ref() +fn is_required_collection(key: &PythonKey) -> bool { + key.text + .as_ref() .is_ok_and(|key| REQUIRED_COLLECTION_KEYS.contains(&key.as_str())) } @@ -93,25 +97,5 @@ pub(crate) fn drop_none(value: DagValue) -> DagValue { /// The value `json.dumps` encodes, or the `TypeError` it raises first in output order. pub(crate) fn encoded(value: DagValue) -> Result { - Ok(match value { - DagValue::Null => JsonValue::Null, - DagValue::Bool(flag) => JsonValue::Bool(flag), - DagValue::Integer(number) => JsonValue::Integer(number), - DagValue::Float(number) => JsonValue::Float(number), - DagValue::String(text) => JsonValue::String(text), - DagValue::Utf16(units) => JsonValue::Utf16(units), - DagValue::Array(items) => { - JsonValue::Array(items.into_iter().map(encoded).collect::>()?) - } - DagValue::Object(entries) => JsonValue::Object( - entries - .into_iter() - .map(|(key, item)| { - let key: String = key.map_err(DagArtifactError::Unencodable)?; - Ok((key, encoded(item)?)) - }) - .collect::>()?, - ), - DagValue::Unencodable(message) => return Err(DagArtifactError::Unencodable(message)), - }) + encode_python_value(value).map_err(DagArtifactError::Unencodable) } diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs index 0e2c34a039..8ec9694e35 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/build_dag_json.rs @@ -12,7 +12,8 @@ use crate::dag_artifact::_helpers::nodes::{ use crate::dag_artifact::_helpers::values::{drop_none, encoded, integer, object, text}; use crate::dag_artifact::constants::{DAG_INDENT, DAG_VERSION, LOADER_KIND}; use crate::dag_artifact::errors::DagArtifactError; -use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use crate::dag_artifact::types::DagValue; use crate::graph::models::ProjectGraph; fn nodes( diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs index 157309cb8e..660a118be2 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/models.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -1,25 +1,7 @@ //! The compiled project, Python node graph and authored values the DAG artifact is built from. -use sqlbuild_core::json::models::JsonInteger; - use crate::assembly::project::types::ObjectKey; -use crate::dag_artifact::types::{DagColumnLineage, DagKey}; - -/// A Python value as `dataclasses.asdict` copies it, before `json.dumps` encodes it. -#[derive(Clone, Debug, PartialEq)] -pub enum DagValue { - Null, - Bool(bool), - Integer(JsonInteger), - Float(f64), - String(String), - /// A `str` holding lone surrogates, as UTF-16 code units. - Utf16(Vec), - Array(Vec), - Object(Vec<(DagKey, DagValue)>), - /// A value `json.dumps` refuses, with the message of its `TypeError`. - Unencodable(String), -} +use crate::dag_artifact::types::{DagColumnLineage, DagValue}; /// One authored column: name, type, nullability, description and meta. #[derive(Clone, Debug, PartialEq)] diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs index ec04a5e978..c41a2a4f7f 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -3,10 +3,11 @@ use crate::dag_artifact::errors::DagArtifactError; use crate::dag_artifact::main::build_dag_json::build_dag_json; use crate::dag_artifact::models::{ DagColumnInput, DagLoaderInput, DagPythonNodeInput, DagSourceEntryInput, DagSourceInput, - DagValue, }; use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::types::DagValue; use crate::graph::models::ProjectGraph; +use crate::python_values::main::text_key::text_key; /// A `(resource type, name)` key. pub(crate) fn key(kind: &str, name: &str) -> ObjectKey { @@ -91,7 +92,7 @@ pub(crate) fn built(test_case: &BuildDagJsonTestCase) -> Result DagValue { DagValue::Object(vec![ - (Ok("owner".to_owned()), DagValue::Null), - (Ok("team".to_owned()), value), + (text_key("owner"), DagValue::Null), + (text_key("team"), value), ]) } diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs index 003d133324..1c540c5ae9 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -1,7 +1,8 @@ use crate::dag_artifact::errors::DagArtifactError; -use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput, DagValue}; +use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; use crate::dag_artifact::tests::helpers::{built, key, loader, python_node, source, team_meta}; use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::types::DagValue; const EMPTY_JSON: &str = "{ \"version\": 1, diff --git a/crates/sqlbuild-analysis/src/dag_artifact/types.rs b/crates/sqlbuild-analysis/src/dag_artifact/types.rs index 3154b438ad..a23241346a 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/types.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/types.rs @@ -1,7 +1,9 @@ //! Keys and lineage shapes of the DAG artifact inputs. -/// A mapping key as `json.dumps` writes it, or the message of the `TypeError` it raises. -pub type DagKey = Result; +use crate::python_values::models::PythonValue; + +/// A Python value as `dataclasses.asdict` copies it, before `json.dumps` encodes it. +pub type DagValue = PythonValue; /// Column to `(upstream node, upstream column)` references. pub type DagColumnLineage = Vec<(String, Vec<(String, String)>)>; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5e0d31c0c9..d7d585de55 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -13,6 +13,8 @@ pub mod dag_artifact; pub mod explicit_references; pub mod graph; pub mod lineage; +pub mod manifest; +pub mod python_values; pub mod query_analysis; pub mod relation_names; pub mod required_columns; diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs new file mode 100644 index 0000000000..bd806d5b13 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/macros.rs @@ -0,0 +1,134 @@ +//! dbt-compatible macro nodes of project macros and SQL hook definitions. + +use std::collections::HashMap; + +use crate::manifest::_helpers::values::{ + docs, empty_array, keyed, object, optional_text, text, text_or_empty, zero_float, +}; +use crate::manifest::constants::{RESOURCE_TYPE_MACRO, SQL_HOOK_PACKAGE_SUFFIX}; +use crate::manifest::models::{ + ManifestInput, ManifestMacroDeclarationInput, ManifestMacroInput, ManifestSqlHookInput, +}; +use crate::python_values::models::PythonValue; + +fn macro_node( + loaded: &ManifestMacroInput, + project_name: &str, + declaration: (Option<&Vec>, Option<&ManifestMacroDeclarationInput>), +) -> PythonValue { + let (dependencies, record) = declaration; + let path: &str = &loaded.relative_path; + object(vec![ + ("name", text(&loaded.name)), + ("resource_type", text(RESOURCE_TYPE_MACRO)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_MACRO}.{project_name}.{}", + loaded.name + )), + ), + ("macro_sql", text(&loaded.raw_source)), + ( + "depends_on", + object(vec![( + "macros", + PythonValue::Array( + dependencies + .map(Vec::as_slice) + .unwrap_or_default() + .iter() + .map(|name| text(&format!("{RESOURCE_TYPE_MACRO}.{project_name}.{name}"))) + .collect(), + ), + )]), + ), + ("description", text("")), + ( + "meta", + object(vec![ + ( + "sqlbuild_visibility", + text(record.map_or("global", |record| record.scope.as_str())), + ), + ( + "sqlbuild_scope_path", + optional_text(record.and_then(|record| record.owning_path.as_deref())), + ), + ]), + ), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("arguments", empty_array()), + ("created_at", zero_float()), + ("supported_languages", PythonValue::Null), + ]) +} + +fn sql_hook_node(hook: &ManifestSqlHookInput, project_name: &str) -> PythonValue { + let package: String = format!("{project_name}{SQL_HOOK_PACKAGE_SUFFIX}"); + object(vec![ + ("name", text(&hook.name)), + ("resource_type", text(RESOURCE_TYPE_MACRO)), + ("package_name", text(&package)), + ("path", text(&hook.relative_path)), + ("original_file_path", text(&hook.relative_path)), + ( + "unique_id", + text(&format!("{RESOURCE_TYPE_MACRO}.{package}.{}", hook.name)), + ), + ("macro_sql", text(&hook.sql_body)), + ("depends_on", object(vec![("macros", empty_array())])), + ("description", text_or_empty(hook.description.as_deref())), + ( + "meta", + object(vec![ + ("sqlbuild_resource_type", text("sql_hook")), + ("sqlbuild_hook_name", text(&hook.name)), + ]), + ), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("arguments", empty_array()), + ("created_at", zero_float()), + ("supported_languages", PythonValue::Null), + ]) +} + +/// Every macro node: project macros, then SQL hook definitions. +pub(crate) fn macro_nodes(input: &ManifestInput) -> PythonValue { + let project_name: &str = &input.project_name; + let mut dependencies: HashMap<&str, &Vec> = HashMap::new(); + let mut records: HashMap<&str, &ManifestMacroDeclarationInput> = HashMap::new(); + for record in &input.macro_declarations { + if let Some(names) = &record.dependencies { + dependencies.insert(&record.name, names); + } + records.insert(&record.name, record); + } + let mut nodes: Vec<(String, PythonValue)> = Vec::new(); + for loaded in &input.macros { + let name: &str = &loaded.key; + nodes.push(( + format!("{RESOURCE_TYPE_MACRO}.{project_name}.{name}"), + macro_node( + loaded, + project_name, + (dependencies.get(name).copied(), records.get(name).copied()), + ), + )); + } + for hook in &input.sql_hooks { + nodes.push(( + format!( + "{RESOURCE_TYPE_MACRO}.{project_name}{SQL_HOOK_PACKAGE_SUFFIX}.{}", + hook.name + ), + sql_hook_node(hook, project_name), + )); + } + keyed(nodes) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs new file mode 100644 index 0000000000..cf93695262 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod macros; +pub(crate) mod model_nodes; +pub(crate) mod node_fields; +pub(crate) mod payload; +pub(crate) mod resource_nodes; +pub(crate) mod test_nodes; +pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs new file mode 100644 index 0000000000..ec47ec1936 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/model_nodes.rs @@ -0,0 +1,176 @@ +//! The dbt-compatible model node. + +use crate::manifest::_helpers::node_fields::{ + columns, config, config_tags, depends_on, fqn, lifecycle_hooks, +}; +use crate::manifest::_helpers::values::{ + docs, empty_object, inserted, object, optional_text, text, text_or_empty, +}; +use crate::manifest::constants::{CHECKSUM_HASH_NAME, RESOURCE_TYPE_MODEL, SQLBUILD_META_KEY}; +use crate::manifest::models::{ManifestModelInput, ManifestPythonHookInput}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +/// The model node's inputs besides the model. +pub(crate) struct ModelNodeContext<'a> { + pub(crate) project_name: &'a str, + pub(crate) compiled_code: Option<&'a str>, + pub(crate) query_hash: &'a str, + pub(crate) python_hooks: &'a [ManifestPythonHookInput], +} + +fn meta_entry(meta: &[(PythonKey, PythonValue)], key: &str) -> PythonValue { + meta.iter() + .find(|(existing, _)| *existing == text_key(key)) + .map_or(PythonValue::Null, |(_, value)| value.clone()) +} + +/// The `str`-keyed entries of an authored `sqlbuild` dict, or its non-dict value as `authored_value`. +fn authored_sqlbuild(authored: PythonValue) -> Vec<(PythonKey, PythonValue)> { + match authored { + PythonValue::Object(entries) => entries + .into_iter() + .filter(|(key, _)| key.is_str) + .fold(Vec::new(), |fields, (key, value)| { + inserted(fields, key, value) + }), + PythonValue::Null => Vec::new(), + other => vec![(text_key("authored_value"), other)], + } +} + +fn with_sqlbuild_fact( + meta: Vec<(PythonKey, PythonValue)>, + (field, value): (&str, PythonValue), +) -> Vec<(PythonKey, PythonValue)> { + let fields: Vec<(PythonKey, PythonValue)> = inserted( + authored_sqlbuild(meta_entry(&meta, SQLBUILD_META_KEY)), + text_key(field), + value, + ); + inserted( + meta, + text_key(SQLBUILD_META_KEY), + PythonValue::Object(fields), + ) +} + +fn meta(model: &ManifestModelInput, python_hooks: &[ManifestPythonHookInput]) -> PythonValue { + let mut meta: Vec<(PythonKey, PythonValue)> = Vec::new(); + if let Some(PythonValue::Object(authored)) = + model.schema.as_ref().and_then(|schema| schema.meta.clone()) + { + for (key, value) in authored { + meta = inserted(meta, key, value); + } + } + let mut hooks: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, entries) in [ + ("pre_hooks", model.config.pre_hooks.as_deref()), + ("post_hooks", model.config.post_hooks.as_deref()), + ] { + let serialized: Vec = lifecycle_hooks(entries, python_hooks); + if !serialized.is_empty() { + hooks.push((text_key(key), PythonValue::Array(serialized))); + } + } + if !hooks.is_empty() { + meta = with_sqlbuild_fact(meta, ("lifecycle_hooks", PythonValue::Object(hooks))); + } + let families = model + .schema + .as_ref() + .map(|schema| schema.dynamic_columns.as_slice()) + .unwrap_or_default(); + if !families.is_empty() { + let families: Vec = families + .iter() + .map(|family| { + object(vec![ + ("name", family.name.clone()), + ("pivot_column", family.pivot_column.clone()), + ("value_column", family.value_column.clone()), + ("aggregate", family.aggregate.clone()), + ("type", family.data_type.clone()), + ("name_pattern", family.name_pattern.clone()), + ]) + }) + .collect(); + meta = with_sqlbuild_fact(meta, ("dynamic_columns", PythonValue::Array(families))); + } + PythonValue::Object(meta) +} + +/// One dbt-compatible model node. +pub(crate) fn model_node( + model: &ManifestModelInput, + context: &ModelNodeContext<'_>, +) -> PythonValue { + let project_name: &str = context.project_name; + let path: &str = &model.path.text; + object(vec![ + ( + "database", + optional_text(model.destination.database.as_deref()), + ), + ("schema", optional_text(model.destination.schema.as_deref())), + ("name", text(&model.name)), + ("resource_type", text(RESOURCE_TYPE_MODEL)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_MODEL}.{project_name}.{}", + model.name + )), + ), + ("fqn", fqn(project_name, &model.path)), + ("alias", optional_text(model.destination.name.as_deref())), + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text(context.query_hash)), + ]), + ), + ("config", config(&model.config)), + ("tags", config_tags(&model.config)), + ( + "description", + text_or_empty( + model + .schema + .as_ref() + .and_then(|schema| schema.description.as_deref()), + ), + ), + ( + "columns", + model + .schema + .as_ref() + .map_or_else(empty_object, |schema| columns(&schema.columns)), + ), + ("meta", meta(model, context.python_hooks)), + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", PythonValue::Float(0.0)), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ( + "relation_name", + optional_text(model.destination.qualified_name.as_deref()), + ), + ("raw_code", text(&model.query_sql)), + ( + "compiled_code", + text(context.compiled_code.unwrap_or(&model.query_sql)), + ), + ("depends_on", depends_on(&model.deps, project_name)), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs new file mode 100644 index 0000000000..8c9aed05f2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/node_fields.rs @@ -0,0 +1,201 @@ +//! Shared manifest node pieces: fqn, columns, config, hooks and dependency ids. + +use crate::assembly::project::types::ObjectKey; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, keyed, object, optional_text, text, text_or_empty, texts, +}; +use crate::manifest::constants::{DEFAULT_RESOURCE_PREFIX, RESOURCE_TYPE_PREFIXES}; +use crate::manifest::models::{ + ManifestColumnInput, ManifestConfigInput, ManifestHookInput, ManifestPathInput, + ManifestPythonHookInput, +}; +use crate::python_values::models::PythonValue; + +/// `[project, *relative_path.parent.parts, relative_path.stem]`. +pub(crate) fn fqn(project_name: &str, path: &ManifestPathInput) -> PythonValue { + texts( + std::iter::once(project_name) + .chain(path.parent_parts.iter().map(String::as_str)) + .chain(std::iter::once(path.stem.as_str())), + ) +} + +/// The dbt-style columns dict, a later column replacing an earlier one of the same name. +pub(crate) fn columns(columns: &[ManifestColumnInput]) -> PythonValue { + keyed( + columns + .iter() + .map(|column| { + ( + column.name.clone(), + object(vec![ + ("name", text(&column.name)), + ("description", text_or_empty(column.description.as_deref())), + ("meta", column.meta.clone().unwrap_or_else(empty_object)), + ("data_type", optional_text(column.data_type.as_deref())), + ("constraints", empty_array()), + ("quote", PythonValue::Null), + ("tags", empty_array()), + ]), + ) + }) + .collect(), + ) +} + +/// `prefix.project.name` of a compiled key. +pub(crate) fn unique_id((resource_type, name): &ObjectKey, project_name: &str) -> String { + let prefix: &str = RESOURCE_TYPE_PREFIXES + .iter() + .find(|(kind, _)| kind == resource_type) + .map_or(DEFAULT_RESOURCE_PREFIX, |(_, prefix)| prefix); + format!("{prefix}.{project_name}.{name}") +} + +/// The unique ids of `keys`. +pub(crate) fn unique_ids(keys: &[ObjectKey], project_name: &str) -> PythonValue { + PythonValue::Array( + keys.iter() + .map(|key| text(&unique_id(key, project_name))) + .collect(), + ) +} + +fn dbt_sql_hooks(hooks: Option<&[ManifestHookInput]>) -> PythonValue { + let statements = hooks + .unwrap_or_default() + .iter() + .filter_map(|hook| match hook { + ManifestHookInput::Sql { statement, .. } => Some(statement), + _ => None, + }); + PythonValue::Array( + statements + .enumerate() + .map(|(index, statement)| { + object(vec![ + ("sql", text(statement)), + ("transaction", PythonValue::Bool(true)), + ( + "index", + PythonValue::Integer(sqlbuild_core::json::models::JsonInteger::from( + i64::try_from(index).unwrap_or(i64::MAX), + )), + ), + ]) + }) + .collect(), + ) +} + +/// The config tags, or an empty list. +pub(crate) fn config_tags(config: &ManifestConfigInput) -> PythonValue { + texts(config.tags.iter().flatten().map(String::as_str)) +} + +/// The dbt-compatible config dict of a model. +pub(crate) fn config(config: &ManifestConfigInput) -> PythonValue { + object(vec![ + ("enabled", config.enabled.clone()), + ("alias", config.alias.clone()), + ("schema", config.schema.clone()), + ("database", config.database.clone()), + ("tags", config_tags(config)), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text(&config.materialized)), + ("incremental_strategy", config.incremental_strategy.clone()), + ( + "merge_exclude_columns", + config.merge_exclude_columns.clone(), + ), + ("persist_docs", empty_object()), + ("post-hook", dbt_sql_hooks(config.post_hooks.as_deref())), + ("pre-hook", dbt_sql_hooks(config.pre_hooks.as_deref())), + ("quoting", empty_object()), + ("column_types", empty_object()), + ("full_refresh", config.full_refresh.clone()), + ("unique_key", config.unique_key.clone()), + ("on_schema_change", config.on_schema_change.clone()), + ("on_configuration_change", text("apply")), + ("grants", empty_object()), + ("packages", empty_array()), + ("docs", docs()), + ( + "contract", + object(vec![ + ("enforced", PythonValue::Bool(false)), + ("alias_types", PythonValue::Bool(true)), + ]), + ), + ]) +} + +fn optional_field(name: &str, value: Option) -> Option<(String, PythonValue)> { + value.map(|value| (name.to_owned(), value)) +} + +/// `serialize_hook_entries` with every SQL hook output field and the Python hook identities. +pub(crate) fn lifecycle_hooks( + hooks: Option<&[ManifestHookInput]>, + python_hooks: &[ManifestPythonHookInput], +) -> Vec { + let mut serialized: Vec = Vec::new(); + for hook in hooks.unwrap_or_default() { + match hook { + ManifestHookInput::Sql { + statement, + name, + relative_path, + definition_sql, + kwargs, + description, + } => { + let fields: Vec<(String, PythonValue)> = [ + Some(("type".to_owned(), text("sql"))), + Some(("statement".to_owned(), text(statement))), + optional_field("name", name.as_deref().map(text)), + optional_field("relative_path", relative_path.as_deref().map(text)), + optional_field("definition_sql", definition_sql.as_deref().map(text)), + optional_field( + "kwargs", + (*kwargs != PythonValue::Null).then(|| kwargs.clone()), + ), + optional_field("description", description.as_deref().map(text)), + ] + .into_iter() + .flatten() + .collect(); + serialized.push(keyed(fields)); + } + ManifestHookInput::Python { name, kwargs } => { + let mut fields: Vec<(String, PythonValue)> = vec![ + ("type".to_owned(), text("python")), + ("name".to_owned(), text(name)), + ("kwargs".to_owned(), kwargs.clone()), + ]; + if let Some(identity) = python_hooks.iter().rev().find(|hook| hook.name == *name) { + fields.extend([ + ("relative_path".to_owned(), text(&identity.relative_path)), + ( + "definition_hash".to_owned(), + text(&identity.definition_hash), + ), + ("version_hash".to_owned(), text(&identity.version_hash)), + ]); + } + serialized.push(keyed(fields)); + } + ManifestHookInput::Other => {} + } + } + serialized +} + +/// `{"macros": [], "nodes": [...]}`. +pub(crate) fn depends_on(deps: &[ObjectKey], project_name: &str) -> PythonValue { + object(vec![ + ("macros", empty_array()), + ("nodes", unique_ids(deps, project_name)), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs new file mode 100644 index 0000000000..32087c8f0e --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/payload.rs @@ -0,0 +1,156 @@ +//! The complete manifest payload. + +use std::collections::{HashMap, HashSet}; + +use crate::graph::main::unique_id_edges::unique_id_edges; +use crate::graph::models::{EdgeDirection, ProjectGraph}; +use crate::manifest::_helpers::macros::macro_nodes; +use crate::manifest::_helpers::model_nodes::{ModelNodeContext, model_node}; +use crate::manifest::_helpers::resource_nodes::{seed_node, source_node}; +use crate::manifest::_helpers::test_nodes::{audit_test_node, sql_test_node}; +use crate::manifest::_helpers::values::{empty_object, keyed, object, text}; +use crate::manifest::constants::{ + DBT_MANIFEST_SCHEMA_VERSION, DBT_VERSION, RESOURCE_TYPE_MODEL, RESOURCE_TYPE_PREFIXES, + RESOURCE_TYPE_SEED, RESOURCE_TYPE_SOURCE, +}; +use crate::manifest::errors::ManifestError; +use crate::manifest::models::{ManifestInput, ManifestPlanInput}; +use crate::python_values::models::PythonValue; +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// Each planned model's resolved SQL; a later entry of the same name wins. +fn resolved_sql(plan: &ManifestPlanInput) -> HashMap<&str, &str> { + plan.models + .iter() + .map(|(name, sql)| (name.as_str(), sql.as_str())) + .collect() +} + +fn id_list(ids: &[String]) -> PythonValue { + PythonValue::Array(ids.iter().map(|id| text(id)).collect()) +} + +fn nodes(input: &ManifestInput) -> Result { + let project_name: &str = &input.project_name; + let plan: Option<&ManifestPlanInput> = input.plan.as_ref(); + let resolved: HashMap<&str, &str> = plan.map(resolved_sql).unwrap_or_default(); + let mut nodes: Vec<(String, PythonValue)> = Vec::new(); + for model in &input.models { + let query_hash: String = + compiler_query_fingerprint(&model.query_sql, input.sql_analysis_dialect.as_deref()) + .map_err(ManifestError::QueryFingerprint)?; + nodes.push(( + format!("{RESOURCE_TYPE_MODEL}.{project_name}.{}", model.name), + model_node( + model, + &ModelNodeContext { + project_name, + compiled_code: resolved.get(model.name.as_str()).copied(), + query_hash: &query_hash, + python_hooks: &input.python_hooks, + }, + ), + )); + } + for seed in &input.seeds { + nodes.push(( + format!("{RESOURCE_TYPE_SEED}.{project_name}.{}", seed.name), + seed_node(seed, project_name), + )); + } + if let Some(plan) = plan { + nodes.extend( + plan.audits + .iter() + .map(|audit| audit_test_node(audit, project_name)), + ); + for test in &plan.sql_tests { + nodes.push(sql_test_node(test, project_name)?); + } + } + Ok(keyed(nodes)) +} + +fn sources(input: &ManifestInput) -> PythonValue { + let project_name: &str = &input.project_name; + let loaded: HashSet<&str> = input + .plan + .as_ref() + .map(|plan| plan.source_load_names.iter().map(String::as_str).collect()) + .unwrap_or_default(); + keyed( + input + .sources + .iter() + .map(|source| { + ( + format!("{RESOURCE_TYPE_SOURCE}.{project_name}.{}", source.name), + source_node(source, project_name, loaded.contains(source.name.as_str())), + ) + }) + .collect(), + ) +} + +fn id_map(graph: &ProjectGraph, direction: EdgeDirection, project_name: &str) -> PythonValue { + let prefixes: HashMap = RESOURCE_TYPE_PREFIXES + .iter() + .map(|(kind, prefix)| ((*kind).to_owned(), (*prefix).to_owned())) + .collect(); + keyed( + unique_id_edges(graph, direction, project_name, &prefixes) + .into_iter() + .map(|(id, neighbours)| (id, id_list(&neighbours))) + .collect(), + ) +} + +fn metadata(input: &ManifestInput) -> PythonValue { + object(vec![ + ("dbt_schema_version", text(DBT_MANIFEST_SCHEMA_VERSION)), + ("dbt_version", text(DBT_VERSION)), + ("generated_at", text(&input.generated_at)), + ("invocation_id", text(&input.run_id)), + ("env", empty_object()), + ("project_name", text(&input.project_name)), + ("project_id", PythonValue::Null), + ("user_id", PythonValue::Null), + ("send_anonymous_usage_stats", PythonValue::Bool(false)), + ("adapter_type", text(&input.adapter_type)), + ]) +} + +/// The manifest dict `build_manifest_payload` returned. +pub(crate) fn payload( + input: &ManifestInput, + graph: &ProjectGraph, +) -> Result { + let macros: PythonValue = macro_nodes(input); + let nodes: PythonValue = nodes(input)?; + let sources: PythonValue = sources(input); + let project_name: &str = &input.project_name; + Ok(object(vec![ + ("metadata", metadata(input)), + ("nodes", nodes), + ("sources", sources), + ("macros", macros), + ("exposures", empty_object()), + ("metrics", empty_object()), + ("groups", empty_object()), + ("selectors", empty_object()), + ("disabled", empty_object()), + ( + "parent_map", + id_map(graph, EdgeDirection::Upstream, project_name), + ), + ( + "child_map", + id_map(graph, EdgeDirection::Downstream, project_name), + ), + ("group_map", empty_object()), + ("docs", empty_object()), + ("saved_queries", empty_object()), + ("semantic_models", empty_object()), + ("unit_tests", empty_object()), + ])) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs new file mode 100644 index 0000000000..cf82fa841e --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/resource_nodes.rs @@ -0,0 +1,172 @@ +//! The dbt-compatible source and seed nodes. + +use crate::manifest::_helpers::node_fields::{columns, fqn}; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, inserted, object, optional_text, text, text_or_empty, texts, + zero_float, +}; +use crate::manifest::constants::{ + CHECKSUM_HASH_NAME, RESOURCE_TYPE_SEED, RESOURCE_TYPE_SOURCE, SQLBUILD_META_KEY, +}; +use crate::manifest::models::{ManifestSeedInput, ManifestSourceInput}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +fn object_entries(value: Option) -> Vec<(PythonKey, PythonValue)> { + match value { + Some(PythonValue::Object(entries)) => entries, + _ => Vec::new(), + } +} + +fn source_meta(source: &ManifestSourceInput, auto_load: bool) -> PythonValue { + let meta: Vec<(PythonKey, PythonValue)> = object_entries(source.meta.clone()); + let Some(loader) = &source.loader else { + return PythonValue::Object(meta); + }; + let mut fields: Vec<(&str, PythonValue)> = vec![ + ("loader", text(loader)), + ("auto_load", PythonValue::Bool(auto_load)), + ]; + if let Some(strategy) = &source.write_strategy { + fields.push(("write_strategy", text(strategy))); + } + if let Some(cursor) = &source.cursor_column { + fields.push(("cursor_column", cursor.clone())); + } + if let Some(unique_key) = &source.unique_key { + fields.push(("unique_key", unique_key.clone())); + } + PythonValue::Object(inserted(meta, text_key(SQLBUILD_META_KEY), object(fields))) +} + +/// One dbt-compatible source node. +pub(crate) fn source_node( + source: &ManifestSourceInput, + project_name: &str, + auto_load: bool, +) -> PythonValue { + let name: &str = &source.entry_name; + object(vec![ + ("database", optional_text(source.database.as_deref())), + ("schema", text_or_empty(source.schema.as_deref())), + ("name", text(name)), + ("resource_type", text(RESOURCE_TYPE_SOURCE)), + ("package_name", text(project_name)), + ("path", text(&source.path)), + ("original_file_path", text(&source.path)), + ( + "unique_id", + text(&format!("{RESOURCE_TYPE_SOURCE}.{project_name}.{name}")), + ), + ("fqn", texts([project_name, name])), + ("source_name", text(name)), + ("source_description", text("")), + ("loader", text("")), + ("identifier", text(source.table.as_deref().unwrap_or(name))), + ("quoting", empty_object()), + ("loaded_at_field", PythonValue::Null), + ("freshness", PythonValue::Null), + ("external", PythonValue::Null), + ("description", text_or_empty(source.description.as_deref())), + ("columns", columns(&source.columns)), + ("meta", source_meta(source, auto_load)), + ("source_meta", empty_object()), + ("tags", empty_array()), + ("config", empty_object()), + ("patch_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("relation_name", PythonValue::Null), + ("created_at", zero_float()), + ]) +} + +fn seed_config(seed: &ManifestSeedInput) -> PythonValue { + object(vec![ + ("enabled", PythonValue::Bool(true)), + ("alias", PythonValue::Null), + ("schema", optional_text(seed.destination.schema.as_deref())), + ( + "database", + optional_text(seed.destination.database.as_deref()), + ), + ("tags", texts(seed.tags.iter().map(String::as_str))), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text("seed")), + ("persist_docs", empty_object()), + ("post-hook", empty_array()), + ("pre-hook", empty_array()), + ("quoting", empty_object()), + ("column_types", empty_object()), + ("full_refresh", PythonValue::Null), + ("unique_key", PythonValue::Null), + ("on_schema_change", text("ignore")), + ("on_configuration_change", text("apply")), + ("grants", empty_object()), + ("packages", empty_array()), + ("docs", docs()), + ( + "contract", + object(vec![ + ("enforced", PythonValue::Bool(false)), + ("alias_types", PythonValue::Bool(true)), + ]), + ), + ("delimiter", text(",")), + ("quote_columns", PythonValue::Null), + ]) +} + +/// One dbt-compatible seed node. +pub(crate) fn seed_node(seed: &ManifestSeedInput, project_name: &str) -> PythonValue { + let path: &str = &seed.path.text; + object(vec![ + ( + "database", + optional_text(seed.destination.database.as_deref()), + ), + ("schema", optional_text(seed.destination.schema.as_deref())), + ("name", text(&seed.name)), + ("resource_type", text(RESOURCE_TYPE_SEED)), + ("package_name", text(project_name)), + ("path", text(path)), + ("original_file_path", text(path)), + ( + "unique_id", + text(&format!( + "{RESOURCE_TYPE_SEED}.{project_name}.{}", + seed.name + )), + ), + ("fqn", fqn(project_name, &seed.path)), + ("alias", optional_text(seed.destination.name.as_deref())), + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text("")), + ]), + ), + ("config", seed_config(seed)), + ("tags", texts(seed.tags.iter().map(String::as_str))), + ("description", text_or_empty(seed.description.as_deref())), + ("columns", columns(&seed.columns)), + ("meta", seed.meta.clone().unwrap_or_else(empty_object)), + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", zero_float()), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ( + "relation_name", + optional_text(seed.destination.qualified_name.as_deref()), + ), + ("raw_code", text("")), + ("root_path", PythonValue::Null), + ("depends_on", object(vec![("macros", empty_array())])), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs new file mode 100644 index 0000000000..9dea3f2dea --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/test_nodes.rs @@ -0,0 +1,222 @@ +//! dbt-compatible generic and singular test nodes of planned audits and SQL tests. + +use std::collections::HashMap; + +use crate::manifest::_helpers::node_fields::{unique_id, unique_ids}; +use crate::manifest::_helpers::values::{ + docs, empty_array, empty_object, keyed, object, optional_text, text, texts, zero_float, +}; +use crate::manifest::constants::{CHECKSUM_HASH_NAME, RESOURCE_TYPE_TEST}; +use crate::manifest::errors::ManifestError; +use crate::manifest::models::{ManifestAuditPlanInput, ManifestSqlTestPlanInput}; +use crate::python_values::models::PythonValue; + +fn test_config() -> PythonValue { + object(vec![ + ("enabled", PythonValue::Bool(true)), + ("alias", PythonValue::Null), + ("schema", PythonValue::Null), + ("database", PythonValue::Null), + ("tags", empty_array()), + ("meta", empty_object()), + ("group", PythonValue::Null), + ("materialized", text("test")), + ("severity", text("ERROR")), + ("store_failures", PythonValue::Null), + ("store_failures_as", PythonValue::Null), + ("where", PythonValue::Null), + ("limit", PythonValue::Null), + ("fail_calc", text("count(*)")), + ("warn_if", text("!= 0")), + ("error_if", text("!= 0")), + ]) +} + +/// The test node fields shared by audits and SQL tests, after `fqn` and before `checksum`. +struct TestNodeHead<'a> { + unique_id: &'a str, + name: &'a str, + project_name: &'a str, +} + +fn head(node: &TestNodeHead<'_>) -> Vec<(&'static str, PythonValue)> { + vec![ + ("database", PythonValue::Null), + ("schema", text("")), + ("name", text(node.name)), + ("resource_type", text(RESOURCE_TYPE_TEST)), + ("package_name", text(node.project_name)), + ("path", text("")), + ("original_file_path", text("")), + ("unique_id", text(node.unique_id)), + ("fqn", texts([node.project_name, node.name])), + ("alias", text(node.name)), + ] +} + +fn tail(compiled_code: &str, depends_on: PythonValue) -> Vec<(&'static str, PythonValue)> { + vec![ + ("group", PythonValue::Null), + ("docs", docs()), + ("patch_path", PythonValue::Null), + ("build_path", PythonValue::Null), + ("unrendered_config", empty_object()), + ("created_at", zero_float()), + ("config_call_dict", empty_object()), + ("unrendered_config_call_dict", empty_object()), + ("relation_name", PythonValue::Null), + ("raw_code", text(compiled_code)), + ("compiled_code", text(compiled_code)), + ( + "depends_on", + object(vec![("macros", empty_array()), ("nodes", depends_on)]), + ), + ] +} + +/// The generic test node of one planned audit, keyed by its unique id. +pub(crate) fn audit_test_node( + audit: &ManifestAuditPlanInput, + project_name: &str, +) -> (String, PythonValue) { + let id: String = format!("{RESOURCE_TYPE_TEST}.{project_name}.{}", audit.name); + let mut kwargs: Vec<(String, PythonValue)> = Vec::new(); + if let Some(target) = &audit.attached_target_name { + kwargs.push(("model".to_owned(), text(target))); + } + if let Some(column) = &audit.attached_column_name { + kwargs.push(("column_name".to_owned(), text(column))); + } + let mut fields: Vec<(&str, PythonValue)> = head(&TestNodeHead { + unique_id: &id, + name: &audit.name, + project_name, + }); + fields.extend([ + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ("checksum", text("")), + ]), + ), + ("config", test_config()), + ("tags", empty_array()), + ("description", text("")), + ("columns", empty_object()), + ("meta", object(vec![("sqlbuild_test_type", text("audit"))])), + ]); + fields.extend(tail( + &audit.resolved_sql, + unique_ids(&audit.scope_deps, project_name), + )); + fields.extend([ + ( + "test_metadata", + object(vec![("name", text(&audit.name)), ("kwargs", keyed(kwargs))]), + ), + ( + "column_name", + optional_text(audit.attached_column_name.as_deref()), + ), + ]); + (id, object(fields)) +} + +fn sql_test_meta(test: &ManifestSqlTestPlanInput) -> Result { + let mut fields: Vec<(&str, PythonValue)> = vec![("sqlbuild_test_type", text("sql_native"))]; + let (Some(case_name), Some(source_path)) = (&test.case_name, &test.source_path) else { + return Ok(object(fields)); + }; + let types: HashMap<&str, &str> = test + .parameter_schema + .iter() + .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) + .collect(); + let parameters: Vec = test + .parameters + .iter() + .map(|(name, value)| { + let value_type: &str = types + .get(name.as_str()) + .ok_or_else(|| ManifestError::MissingParameterType(name.clone()))?; + Ok(object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("value", value.clone()), + ])) + }) + .collect::>()?; + fields.extend([ + ("sqlbuild_source_path", text(source_path)), + ("sqlbuild_block_index", test.block_index.clone()), + ("sqlbuild_parent_name", test.parent_name.clone()), + ("sqlbuild_case_name", text(case_name)), + ("sqlbuild_case_index", test.case_index.clone()), + ( + "sqlbuild_case_fingerprint", + optional_text(test.case_fingerprint.as_deref()), + ), + ( + "sqlbuild_parameter_schema", + PythonValue::Array( + test.parameter_schema + .iter() + .map(|(name, value_type, nullable)| { + object(vec![ + ("name", text(name)), + ("type", text(value_type)), + ("nullable", nullable.clone()), + ]) + }) + .collect(), + ), + ), + ("sqlbuild_parameters", PythonValue::Array(parameters)), + ]); + Ok(object(fields)) +} + +/// The singular test node of one planned SQL test, keyed by its unique id. +pub(crate) fn sql_test_node( + test: &ManifestSqlTestPlanInput, + project_name: &str, +) -> Result<(String, PythonValue), ManifestError> { + let id: String = unique_id(&test.key, project_name); + let mut parts: Vec = Vec::new(); + for (model_name, resolved_sql) in &test.chain { + if resolved_sql.is_empty() { + continue; + } + parts.push(format!("-- step: {model_name}")); + parts.push(resolved_sql.clone()); + } + let compiled_code: String = parts.join("\n\n"); + let mut fields: Vec<(&str, PythonValue)> = head(&TestNodeHead { + unique_id: &id, + name: &test.name, + project_name, + }); + fields.extend([ + ( + "checksum", + object(vec![ + ("name", text(CHECKSUM_HASH_NAME)), + ( + "checksum", + text(test.case_fingerprint.as_deref().unwrap_or_default()), + ), + ]), + ), + ("config", test_config()), + ("tags", empty_array()), + ("description", text("")), + ("columns", empty_object()), + ("meta", sql_test_meta(test)?), + ]); + fields.extend(tail( + &compiled_code, + unique_ids(&test.scope_deps, project_name), + )); + Ok((id, object(fields))) +} diff --git a/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs b/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs new file mode 100644 index 0000000000..0c2575f9f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/_helpers/values.rs @@ -0,0 +1,79 @@ +//! Building manifest values as Python dicts, lists and strings. + +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +/// A dict of field names, later duplicate keys replacing earlier values in place. +pub(crate) fn object(entries: Vec<(&str, PythonValue)>) -> PythonValue { + let mut fields: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, value) in entries { + fields = inserted(fields, text_key(key), value); + } + PythonValue::Object(fields) +} + +/// `d[key] = value`: replace the value in place, or append the entry. +pub(crate) fn inserted( + entries: Vec<(PythonKey, PythonValue)>, + key: PythonKey, + value: PythonValue, +) -> Vec<(PythonKey, PythonValue)> { + let mut entries: Vec<(PythonKey, PythonValue)> = entries; + match entries.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => entries.push((key, value)), + } + entries +} + +/// A dict of `str` keys built entry by entry, as a dict comprehension or `update` builds it. +pub(crate) fn keyed(entries: Vec<(String, PythonValue)>) -> PythonValue { + let mut fields: Vec<(PythonKey, PythonValue)> = Vec::new(); + for (key, value) in entries { + fields = inserted(fields, text_key(&key), value); + } + PythonValue::Object(fields) +} + +/// A string. +pub(crate) fn text(value: &str) -> PythonValue { + PythonValue::String(value.to_owned()) +} + +/// A string or `None`. +pub(crate) fn optional_text(value: Option<&str>) -> PythonValue { + value.map_or(PythonValue::Null, text) +} + +/// `value or ""`. +pub(crate) fn text_or_empty(value: Option<&str>) -> PythonValue { + text(value.unwrap_or_default()) +} + +/// A list of strings. +pub(crate) fn texts<'a>(values: impl IntoIterator) -> PythonValue { + PythonValue::Array(values.into_iter().map(text).collect()) +} + +/// An empty dict. +pub(crate) fn empty_object() -> PythonValue { + PythonValue::Object(Vec::new()) +} + +/// An empty list. +pub(crate) fn empty_array() -> PythonValue { + PythonValue::Array(Vec::new()) +} + +/// `0.0`. +pub(crate) fn zero_float() -> PythonValue { + PythonValue::Float(0.0) +} + +/// `{"show": True, "node_color": None}`. +pub(crate) fn docs() -> PythonValue { + object(vec![ + ("show", PythonValue::Bool(true)), + ("node_color", PythonValue::Null), + ]) +} diff --git a/crates/sqlbuild-analysis/src/manifest/constants.rs b/crates/sqlbuild-analysis/src/manifest/constants.rs new file mode 100644 index 0000000000..e472e9c44d --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/constants.rs @@ -0,0 +1,27 @@ +//! dbt manifest v12 constants. + +pub(crate) const DBT_MANIFEST_SCHEMA_VERSION: &str = + "https://schemas.getdbt.com/dbt/manifest/v12.json"; +pub(crate) const DBT_VERSION: &str = "1.12.0"; +pub(crate) const RESOURCE_TYPE_MODEL: &str = "model"; +pub(crate) const RESOURCE_TYPE_SOURCE: &str = "source"; +pub(crate) const RESOURCE_TYPE_SEED: &str = "seed"; +pub(crate) const RESOURCE_TYPE_TEST: &str = "test"; +pub(crate) const RESOURCE_TYPE_MACRO: &str = "macro"; +pub(crate) const CHECKSUM_HASH_NAME: &str = "sha256"; +/// Compiled resource type to its dbt unique id prefix; any other type uses `model`. +pub(crate) const RESOURCE_TYPE_PREFIXES: [(&str, &str); 6] = [ + ("model", "model"), + ("source", "source"), + ("seed", "seed"), + ("dbt_ref", "model"), + ("audit", "test"), + ("sql_test", "test"), +]; +pub(crate) const DEFAULT_RESOURCE_PREFIX: &str = "model"; +/// The package suffix SQL hook macros are published under. +pub(crate) const SQL_HOOK_PACKAGE_SUFFIX: &str = "__sqlbuild_hooks"; +/// The meta key SQLBuild-owned facts are nested under. +pub(crate) const SQLBUILD_META_KEY: &str = "sqlbuild"; +/// `json.dumps(..., indent=2)`. +pub(crate) const MANIFEST_INDENT: usize = 2; diff --git a/crates/sqlbuild-analysis/src/manifest/errors.rs b/crates/sqlbuild-analysis/src/manifest/errors.rs new file mode 100644 index 0000000000..7067fc17d1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/errors.rs @@ -0,0 +1,21 @@ +//! Why the manifest could not be built or encoded, named after the Python error each reproduces. + +use sqlbuild_core::json::errors::JsonEmitError; + +/// A failure while building the manifest, raised where Python's builder raised it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ManifestError { + /// `QueryFingerprintError` of a model query that cannot be fingerprinted. + QueryFingerprint(String), + /// `KeyError` for a bound SQL test parameter missing from the parameter schema. + MissingParameterType(String), +} + +/// A failure encoding the built manifest, raised where `json.dumps` raised it: on writing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ManifestEncodeError { + /// `TypeError` for the first value `json.dumps` cannot encode. + Unencodable(String), + /// The emitter refused the encoded manifest. + Emit(JsonEmitError), +} diff --git a/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs b/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs new file mode 100644 index 0000000000..e4648fdb49 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/main/build_manifest_json.rs @@ -0,0 +1,29 @@ +//! The manifest bytes `json.dumps(manifest, indent=2) + "\n"` wrote, or the error encoding raised. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::graph::models::ProjectGraph; +use crate::manifest::_helpers::payload::payload; +use crate::manifest::constants::MANIFEST_INDENT; +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::models::ManifestInput; +use crate::python_values::main::encode_python_value::encode_python_value; + +/// Build the manifest; its encoding outcome is kept for the writer, where `json.dumps` raised. +pub fn build_manifest_json( + input: &ManifestInput, + graph: &ProjectGraph, +) -> Result, ManifestError> { + let manifest = payload(input, graph)?; + Ok(encode_python_value(manifest) + .map_err(ManifestEncodeError::Unencodable) + .and_then(|value: JsonValue| { + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(MANIFEST_INDENT))), + ) + .map_err(ManifestEncodeError::Emit) + }) + .map(|text| text + "\n")) +} diff --git a/crates/sqlbuild-analysis/src/manifest/main/mod.rs b/crates/sqlbuild-analysis/src/manifest/main/mod.rs new file mode 100644 index 0000000000..3d510c80d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/main/mod.rs @@ -0,0 +1 @@ +pub mod build_manifest_json; diff --git a/crates/sqlbuild-analysis/src/manifest/mod.rs b/crates/sqlbuild-analysis/src/manifest/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/manifest/models.rs b/crates/sqlbuild-analysis/src/manifest/models.rs new file mode 100644 index 0000000000..7065b8efcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/models.rs @@ -0,0 +1,236 @@ +//! The compiled project, plan entries and Python-only facts a dbt-style manifest is built from. + +use crate::assembly::project::types::ObjectKey; +use crate::python_values::models::PythonValue; + +/// A compiled relation location. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestRelationInput { + pub database: Option, + pub schema: Option, + pub name: Option, + pub qualified_name: Option, +} + +/// A path's `str()`, its parent's parts and its stem. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestPathInput { + pub text: String, + pub parent_parts: Vec, + pub stem: String, +} + +/// An authored column. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestColumnInput { + pub name: String, + pub description: Option, + /// `dict(column.meta)` when the meta is truthy. + pub meta: Option, + pub data_type: Option, +} + +/// One resolved lifecycle hook entry. +#[derive(Clone, Debug, PartialEq)] +pub enum ManifestHookInput { + Sql { + statement: String, + name: Option, + /// `relative_path.as_posix()`. + relative_path: Option, + definition_sql: Option, + kwargs: PythonValue, + description: Option, + }, + Python { + name: String, + kwargs: PythonValue, + }, + /// Any other entry, which serializations skip. + Other, +} + +/// The model config values the manifest reads, each as `values.get(key, default)` returned it. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestConfigInput { + pub enabled: PythonValue, + pub alias: PythonValue, + pub schema: PythonValue, + pub database: PythonValue, + /// `str()` of each tag when the `tags` value is a list or tuple. + pub tags: Option>, + /// `str()` of the `materialized` value, `view` when absent or `None`. + pub materialized: String, + pub incremental_strategy: PythonValue, + pub merge_exclude_columns: PythonValue, + pub full_refresh: PythonValue, + pub unique_key: PythonValue, + pub on_schema_change: PythonValue, + /// The entries when the hooks value is a list or tuple. + pub pre_hooks: Option>, + pub post_hooks: Option>, +} + +/// One dynamic column family of a model schema. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestDynamicColumnInput { + pub name: PythonValue, + pub pivot_column: PythonValue, + pub value_column: PythonValue, + pub aggregate: PythonValue, + pub data_type: PythonValue, + pub name_pattern: PythonValue, +} + +/// A model's attached schema entry. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestModelSchemaInput { + pub description: Option, + /// `dict(schema_entry.meta)` when the meta is not `None`. + pub meta: Option, + pub columns: Vec, + pub dynamic_columns: Vec, +} + +/// A compiled model. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestModelInput { + pub name: String, + pub path: ManifestPathInput, + pub destination: ManifestRelationInput, + pub query_sql: String, + pub config: ManifestConfigInput, + pub schema: Option, + pub deps: Vec, +} + +/// A compiled source. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSourceInput { + /// The compiled source's name, which keys the `sources` map. + pub name: String, + pub path: String, + pub entry_name: String, + pub database: Option, + pub schema: Option, + pub table: Option, + pub description: Option, + /// `dict(entry.meta)` when the meta is truthy. + pub meta: Option, + pub loader: Option, + pub write_strategy: Option, + pub cursor_column: Option, + /// The unique key, when truthy. + pub unique_key: Option, + pub columns: Vec, +} + +/// A compiled seed. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSeedInput { + pub name: String, + pub path: ManifestPathInput, + pub destination: ManifestRelationInput, + pub tags: Vec, + pub description: Option, + /// `dict(schema_entry.meta)` when the meta is truthy. + pub meta: Option, + pub columns: Vec, +} + +/// A loaded macro, keyed by its registry name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestMacroInput { + /// The registry name the macro is loaded under. + pub key: String, + pub name: String, + pub relative_path: String, + pub raw_source: String, +} + +/// A macro declaration of the scope index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestMacroDeclarationInput { + pub name: String, + pub scope: String, + pub owning_path: Option, + /// The macro's dependency names, when the record carries macro facts. + pub dependencies: Option>, +} + +/// A SQL hook definition file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestSqlHookInput { + pub name: String, + pub relative_path: String, + pub sql_body: String, + pub description: Option, +} + +/// A Python hook's path and source identity hashes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestPythonHookInput { + pub name: String, + pub relative_path: String, + pub definition_hash: String, + pub version_hash: String, +} + +/// A planned audit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestAuditPlanInput { + pub name: String, + pub resolved_sql: String, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub scope_deps: Vec, +} + +/// A planned SQL test. +#[derive(Clone, Debug, PartialEq)] +pub struct ManifestSqlTestPlanInput { + pub key: ObjectKey, + pub name: String, + /// `(model name, resolved SQL)` of each chain step. + pub chain: Vec<(String, String)>, + pub scope_deps: Vec, + pub source_path: Option, + pub block_index: PythonValue, + pub parent_name: PythonValue, + pub case_name: Option, + pub case_index: PythonValue, + pub case_fingerprint: Option, + /// `(name, value type, nullable)` of each declared parameter. + pub parameter_schema: Vec<(String, String, PythonValue)>, + /// `(name, value)` of each bound parameter, decimals already as text. + pub parameters: Vec<(String, PythonValue)>, +} + +/// The plan entries a build manifest reads. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ManifestPlanInput { + /// `(model name, resolved SQL)` of each model entry. + pub models: Vec<(String, String)>, + pub source_load_names: Vec, + pub audits: Vec, + pub sql_tests: Vec, +} + +/// Everything the manifest is built from besides the graph. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ManifestInput { + pub project_name: String, + pub adapter_type: String, + pub run_id: String, + /// `datetime.now(tz=UTC).isoformat()`. + pub generated_at: String, + pub sql_analysis_dialect: Option, + pub models: Vec, + pub sources: Vec, + pub seeds: Vec, + pub macros: Vec, + pub macro_declarations: Vec, + pub sql_hooks: Vec, + pub python_hooks: Vec, + pub plan: Option, +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests.rs b/crates/sqlbuild-analysis/src/manifest/tests.rs new file mode 100644 index 0000000000..4045696844 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_manifest_json.rs"] +mod build_manifest_json; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs b/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs new file mode 100644 index 0000000000..8523893f3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/helpers.rs @@ -0,0 +1,96 @@ +use crate::manifest::models::{ + ManifestConfigInput, ManifestHookInput, ManifestInput, ManifestModelInput, + ManifestModelSchemaInput, ManifestPathInput, ManifestRelationInput, ManifestSqlTestPlanInput, +}; +use crate::python_values::main::text_key::text_key; +use crate::python_values::models::{PythonKey, PythonValue}; + +pub(crate) fn config() -> ManifestConfigInput { + ManifestConfigInput { + enabled: PythonValue::Bool(true), + alias: PythonValue::Null, + schema: PythonValue::Null, + database: PythonValue::Null, + tags: None, + materialized: "table".to_owned(), + incremental_strategy: PythonValue::Null, + merge_exclude_columns: PythonValue::Null, + full_refresh: PythonValue::Null, + unique_key: PythonValue::Null, + on_schema_change: PythonValue::String("ignore".to_owned()), + pre_hooks: None, + post_hooks: None, + } +} + +pub(crate) fn sql_hook() -> ManifestHookInput { + ManifestHookInput::Sql { + statement: "select 1".to_owned(), + name: Some("audit_orders".to_owned()), + relative_path: None, + definition_sql: None, + kwargs: PythonValue::Null, + description: Some(String::new()), + } +} + +pub(crate) fn meta(entries: Vec<(PythonKey, PythonValue)>) -> Option { + Some(PythonValue::Object(entries)) +} + +pub(crate) fn model(meta: Option, config: ManifestConfigInput) -> ManifestModelInput { + ManifestModelInput { + name: "orders".to_owned(), + path: ManifestPathInput { + text: "models/orders.sql".to_owned(), + parent_parts: vec!["models".to_owned()], + stem: "orders".to_owned(), + }, + destination: ManifestRelationInput::default(), + query_sql: "SELECT 1 AS order_id".to_owned(), + config, + schema: Some(ManifestModelSchemaInput { + description: None, + meta, + columns: Vec::new(), + dynamic_columns: Vec::new(), + }), + deps: Vec::new(), + } +} + +pub(crate) fn project(models: Vec) -> ManifestInput { + ManifestInput { + project_name: "shop".to_owned(), + adapter_type: "duckdb".to_owned(), + run_id: "run".to_owned(), + generated_at: "2026-10-12T00:00:00+00:00".to_owned(), + models, + ..ManifestInput::default() + } +} + +pub(crate) fn authored(key: &str, value: PythonValue) -> (PythonKey, PythonValue) { + (text_key(key), value) +} + +pub(crate) fn sql_test(parameter: &str) -> ManifestSqlTestPlanInput { + ManifestSqlTestPlanInput { + key: ("sql_test".to_owned(), "orders_case".to_owned()), + name: "orders_case".to_owned(), + chain: Vec::new(), + scope_deps: Vec::new(), + source_path: Some("tests/orders.sql".to_owned()), + block_index: PythonValue::Null, + parent_name: PythonValue::Null, + case_name: Some("first".to_owned()), + case_index: PythonValue::Null, + case_fingerprint: None, + parameter_schema: vec![( + "amount".to_owned(), + "decimal".to_owned(), + PythonValue::Bool(false), + )], + parameters: vec![(parameter.to_owned(), PythonValue::String("1.50".to_owned()))], + } +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs b/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs new file mode 100644 index 0000000000..b68a4031e1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/test_build_manifest_json.rs @@ -0,0 +1,141 @@ +use crate::graph::models::ProjectGraph; +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::main::build_manifest_json::build_manifest_json; +use crate::manifest::models::ManifestPlanInput; +use crate::manifest::tests::helpers::{authored, config, meta, model, project, sql_hook, sql_test}; +use crate::manifest::tests::test_types::{BuildManifestJsonTestCase, ManifestFailureTestCase}; +use crate::python_values::models::{PythonKey, PythonValue}; + +#[test] +fn given_project_when_building_manifest_json_then_python_payload_agrees() { + let hooked = || { + let mut hooked = config(); + hooked.post_hooks = Some(vec![sql_hook()]); + hooked + }; + let test_cases = [ + BuildManifestJsonTestCase { + description: "an empty project keeps metadata first and ends with a newline", + input: project(Vec::new()), + expected_fragments: vec![ + "{\n \"metadata\": {\n \"dbt_schema_version\": \"https://schemas.getdbt.com/dbt/manifest/v12.json\",\n \"dbt_version\": \"1.12.0\",\n \"generated_at\": \"2026-10-12T00:00:00+00:00\",\n \"invocation_id\": \"run\",", + " \"unit_tests\": {}\n}\n", + ], + }, + BuildManifestJsonTestCase { + description: "an authored sqlbuild scalar is kept beside lifecycle hooks", + input: project(vec![model( + meta(vec![authored( + "sqlbuild", + PythonValue::String("team".to_owned()), + )]), + hooked(), + )]), + expected_fragments: vec![ + "\"sqlbuild\": {\n \"authored_value\": \"team\",\n \"lifecycle_hooks\": {\n \"post_hooks\": [", + "\"type\": \"sql\",\n \"statement\": \"select 1\",\n \"name\": \"audit_orders\",\n \"description\": \"\"\n", + "\"post-hook\": [\n {\n \"sql\": \"select 1\",\n \"transaction\": true,\n \"index\": 0\n", + "\"fqn\": [\n \"shop\",\n \"models\",\n \"orders\"\n ]", + "\"created_at\": 0.0,", + ], + }, + BuildManifestJsonTestCase { + description: "non-str keys of an authored sqlbuild dict are dropped", + input: project(vec![model( + meta(vec![authored( + "sqlbuild", + PythonValue::Object(vec![ + ( + PythonKey { + text: Ok("1".to_owned()), + is_str: false, + }, + PythonValue::Null, + ), + authored("owner", PythonValue::String("ops".to_owned())), + ]), + )]), + hooked(), + )]), + expected_fragments: vec![ + "\"sqlbuild\": {\n \"owner\": \"ops\",\n \"lifecycle_hooks\"", + ], + }, + ]; + for test_case in test_cases { + let json: String = build_manifest_json(&test_case.input, &ProjectGraph::default()) + .expect("manifest builds") + .expect("manifest encodes"); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| json.contains(fragment)), + "{}: missing a fragment of\n{json}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_values_when_building_manifest_json_then_python_error_agrees() { + let unencodable = + PythonValue::Unencodable("Object of type set is not JSON serializable".to_owned()); + let test_cases = [ + ManifestFailureTestCase { + description: "an unencodable meta value fails on encoding", + input: project(vec![model( + meta(vec![authored("owners", unencodable.clone())]), + config(), + )]), + expected_error: Ok(Some(ManifestEncodeError::Unencodable( + "Object of type set is not JSON serializable".to_owned(), + ))), + }, + ManifestFailureTestCase { + description: "the first unencodable value in output order is reported", + input: { + let mut early = config(); + early.alias = PythonValue::Unencodable( + "Object of type frozenset is not JSON serializable".to_owned(), + ); + project(vec![model(meta(vec![authored("owners", unencodable)]), early)]) + }, + expected_error: Ok(Some(ManifestEncodeError::Unencodable( + "Object of type frozenset is not JSON serializable".to_owned(), + ))), + }, + ManifestFailureTestCase { + description: "a bound parameter missing from the schema fails while building", + input: { + let mut input = project(Vec::new()); + input.plan = Some(ManifestPlanInput { + sql_tests: vec![sql_test("other")], + ..ManifestPlanInput::default() + }); + input + }, + expected_error: Err(ManifestError::MissingParameterType("other".to_owned())), + }, + ManifestFailureTestCase { + description: "a query that cannot be fingerprinted fails while building", + input: { + let mut broken = model(None, config()); + broken.query_sql = "SELECT 'unterminated".to_owned(); + project(vec![broken]) + }, + expected_error: Err(ManifestError::QueryFingerprint( + "SQL tokenization failed: Tokenization error at line 1, column 21: Unterminated string".to_owned(), + )), + }, + ]; + for test_case in test_cases { + let actual: Result, ManifestError> = + build_manifest_json(&test_case.input, &ProjectGraph::default()).map(Result::err); + assert_eq!( + actual, test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs b/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs new file mode 100644 index 0000000000..d0ad2b87e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/manifest/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::manifest::errors::{ManifestEncodeError, ManifestError}; +use crate::manifest::models::ManifestInput; + +pub(crate) struct BuildManifestJsonTestCase { + pub(crate) description: &'static str, + pub(crate) input: ManifestInput, + pub(crate) expected_fragments: Vec<&'static str>, +} + +pub(crate) struct ManifestFailureTestCase { + pub(crate) description: &'static str, + pub(crate) input: ManifestInput, + /// The building error, or the encoding error kept for the writer. + pub(crate) expected_error: Result, ManifestError>, +} diff --git a/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs b/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs new file mode 100644 index 0000000000..23ee2aa843 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/encode_python_value.rs @@ -0,0 +1,30 @@ +//! The JSON value `json.dumps` encodes, or the `TypeError` it raises first in output order. + +use sqlbuild_core::json::models::JsonValue; + +use crate::python_values::models::PythonValue; + +/// The encoded value, or the message of the first `TypeError` in output order. +pub fn encode_python_value(value: PythonValue) -> Result { + Ok(match value { + PythonValue::Null => JsonValue::Null, + PythonValue::Bool(flag) => JsonValue::Bool(flag), + PythonValue::Integer(number) => JsonValue::Integer(number), + PythonValue::Float(number) => JsonValue::Float(number), + PythonValue::String(text) => JsonValue::String(text), + PythonValue::Utf16(units) => JsonValue::Utf16(units), + PythonValue::Array(items) => JsonValue::Array( + items + .into_iter() + .map(encode_python_value) + .collect::>()?, + ), + PythonValue::Object(entries) => JsonValue::Object( + entries + .into_iter() + .map(|(key, item)| Ok((key.text?, encode_python_value(item)?))) + .collect::>()?, + ), + PythonValue::Unencodable(message) => return Err(message), + }) +} diff --git a/crates/sqlbuild-analysis/src/python_values/main/mod.rs b/crates/sqlbuild-analysis/src/python_values/main/mod.rs new file mode 100644 index 0000000000..ca5fee3739 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/mod.rs @@ -0,0 +1,2 @@ +pub mod encode_python_value; +pub mod text_key; diff --git a/crates/sqlbuild-analysis/src/python_values/main/text_key.rs b/crates/sqlbuild-analysis/src/python_values/main/text_key.rs new file mode 100644 index 0000000000..b898b05c9e --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/main/text_key.rs @@ -0,0 +1,12 @@ +//! A `str` dict key. + +use crate::python_values::models::PythonKey; + +/// The key of a `str` field name. +#[must_use] +pub fn text_key(text: &str) -> PythonKey { + PythonKey { + text: Ok(text.to_owned()), + is_str: true, + } +} diff --git a/crates/sqlbuild-analysis/src/python_values/mod.rs b/crates/sqlbuild-analysis/src/python_values/mod.rs new file mode 100644 index 0000000000..5355003ff2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/mod.rs @@ -0,0 +1,2 @@ +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-analysis/src/python_values/models.rs b/crates/sqlbuild-analysis/src/python_values/models.rs new file mode 100644 index 0000000000..a5bfa2bdc3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_values/models.rs @@ -0,0 +1,28 @@ +//! Python values on their way to `json.dumps`, keeping what Python's type checks see. + +use sqlbuild_core::json::models::JsonInteger; + +/// A Python value as `json.dumps` would encode it. +#[derive(Clone, Debug, PartialEq)] +pub enum PythonValue { + Null, + Bool(bool), + Integer(JsonInteger), + Float(f64), + String(String), + /// A `str` holding lone surrogates, as UTF-16 code units. + Utf16(Vec), + /// A `list` or `tuple`. + Array(Vec), + /// A `dict`, in insertion order. + Object(Vec<(PythonKey, PythonValue)>), + /// A value `json.dumps` refuses, with the message of its `TypeError`. + Unencodable(String), +} + +/// A `dict` key: the text `json.dumps` writes, or its `TypeError` message, and whether it is a `str`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonKey { + pub text: Result, + pub is_str: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index c3b3b90372..96eb544955 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod artifact; pub(crate) mod registration; pub(crate) mod rows; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs index 0509429a49..1cb78168a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs @@ -7,12 +7,13 @@ use sqlbuild_analysis::dag_artifact::models::{ DagAuditInput, DagColumnInput, DagFunctionInput, DagLoaderInput, DagModelInput, DagModelSchemaInput, DagProjectInput, DagPythonAssetInput, DagPythonGraphInput, DagPythonNodeInput, DagRelationInput, DagScenarioInput, DagSeedInput, DagSourceEntryInput, - DagSourceInput, DagSqlTestCaseInput, DagSqlTestInput, DagValue, + DagSourceInput, DagSqlTestCaseInput, DagSqlTestInput, }; +use sqlbuild_analysis::dag_artifact::types::DagValue; use sqlbuild_analysis::dag_artifact::types::DagColumnLineage; -use crate::bindings::_helpers::dag_artifact::values::dag_value; +use crate::bindings::_helpers::python_values::values::python_value; const AUDIT_SEVERITY_MODULE: &str = "sqlbuild.compiler.auditing.types"; const DECIMAL_KIND: &str = "decimal"; @@ -454,3 +455,8 @@ pub(crate) fn python_graph_input(graph: &Bound<'_, PyAny>) -> PyResult) -> PyResult { + python_value(value, true) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs new file mode 100644 index 0000000000..a4bfb862b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -0,0 +1,126 @@ +//! `build_manifest`: the dbt-style manifest bytes, whose encoding error is raised on writing. + +use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyBytes; +use pyo3::{FromPyObject, PyRef, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::manifest::errors::{ManifestEncodeError, ManifestError}; +use sqlbuild_analysis::manifest::main::build_manifest_json::build_manifest_json; +use sqlbuild_analysis::manifest::models::ManifestInput; +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::manifest::rows::{ + ProjectRows, plan_input, project_rows, python_hooks, +}; + +const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; + +/// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeManifest { + contents: Result, ManifestEncodeError>, +} + +#[pymethods] +impl NativeManifest { + /// The bytes `json.dumps(manifest, indent=2) + "\n"` encoded, raising what it raised. + fn contents<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.contents { + Ok(bytes) => Ok(PyBytes::new(py, bytes)), + Err(ManifestEncodeError::Unencodable(message)) => { + Err(PyTypeError::new_err(message.clone())) + } + Err(ManifestEncodeError::Emit(JsonEmitError::NestingTooDeep)) => Err( + PyRecursionError::new_err("maximum recursion depth exceeded"), + ), + Err(ManifestEncodeError::Emit(error)) => { + Err(PyValueError::new_err(format!("{error:?}"))) + } + } + } +} + +fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { + match error { + ManifestError::QueryFingerprint(message) => py + .import(FINGERPRINT_EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("QueryFingerprintError")) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + ManifestError::MissingParameterType(name) => PyKeyError::new_err(name), + } +} + +/// The manifest facts only Python knows: names, the generation stamp and hook identities. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ManifestFacts<'py> { + project_name: String, + adapter_type: String, + /// `datetime.now(tz=UTC).isoformat()`. + generated_at: String, + /// `build_hook_identities(project.hook_functions)`. + hook_identities: Bound<'py, PyAny>, +} + +/// The manifest of `project` and the optional plan. +#[pyfunction] +#[pyo3(signature = (*, project, plan_output, graph, facts))] +pub(crate) fn build_manifest( + project: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + graph: PyRef<'_, NativeProjectGraph>, + facts: ManifestFacts<'_>, +) -> PyResult> { + compiler_guard(|| { + let py: Python<'_> = project.py(); + let ManifestFacts { + project_name, + adapter_type, + generated_at, + hook_identities, + } = facts; + let python_hooks = python_hooks(project, &hook_identities)?; + let ProjectRows { + run_id, + sql_analysis_dialect, + models, + sources, + seeds, + macros, + macro_declarations, + sql_hooks, + } = project_rows(project)?; + let input: ManifestInput = ManifestInput { + project_name, + adapter_type, + run_id, + generated_at, + sql_analysis_dialect, + models, + sources, + seeds, + macros, + macro_declarations, + sql_hooks, + python_hooks, + plan: plan_input(plan_output)?, + }; + let graph: &ProjectGraph = graph.graph(); + let contents = py + .detach(|| build_manifest_json(&input, graph)) + .map_err(|error| python_error(error, py))? + .map(String::into_bytes); + Py::new(py, NativeManifest { contents }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_manifest, module)?)?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs new file mode 100644 index 0000000000..c50b252ca8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod builder; +pub(crate) mod registration; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs new file mode 100644 index 0000000000..f2c51d9e32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs @@ -0,0 +1,9 @@ +//! Register the manifest builder. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::manifest::builder; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + builder::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs new file mode 100644 index 0000000000..5d7f5d9752 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs @@ -0,0 +1,484 @@ +//! Read the compiled project, plan output and hook identities into manifest inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::manifest::models::{ + ManifestAuditPlanInput, ManifestColumnInput, ManifestConfigInput, ManifestDynamicColumnInput, + ManifestHookInput, ManifestMacroDeclarationInput, ManifestMacroInput, ManifestModelInput, + ManifestModelSchemaInput, ManifestPathInput, ManifestPlanInput, ManifestPythonHookInput, + ManifestRelationInput, ManifestSeedInput, ManifestSourceInput, ManifestSqlHookInput, + ManifestSqlTestPlanInput, +}; +use sqlbuild_analysis::python_values::models::PythonValue; + +use crate::bindings::_helpers::python_values::values::python_value; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DECIMAL_KIND: &str = "decimal"; +const MACRO_KIND: &str = "macro"; +const DEFAULT_MATERIALIZATION: &str = "view"; + +fn attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attribute(value, name)?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + attribute(value, name)?.extract() +} + +/// The value as `json.dumps` encodes it. +fn json(value: &Bound<'_, PyAny>) -> PyResult { + python_value(value, false) +} + +fn enum_value(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let member: Bound<'_, PyAny> = attribute(value, name)?; + if member.is_none() { + return Ok(None); + } + member.getattr("value")?.extract().map(Some) +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + attribute(value, name)?.try_iter()?.collect() +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)? + .iter() + .map(PyAnyMethods::extract) + .collect() +} + +/// `relative_path.as_posix()` of an optional path attribute. +fn optional_posix(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let path: Bound<'_, PyAny> = attribute(value, name)?; + if path.is_none() { + return Ok(None); + } + path.call_method0("as_posix")?.extract().map(Some) +} + +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(( + attribute(key, "resource_type")?.str()?.to_string(), + text(key, "name")?, + )) +} + +fn keys(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(object_key).collect() +} + +/// `dict(value)` when `truthy_only` is unset or the value is truthy, as JSON. +fn dict_copy(value: &Bound<'_, PyAny>, truthy_only: bool) -> PyResult> { + if value.is_none() || (truthy_only && !value.is_truthy()?) { + return Ok(None); + } + let copied: Bound<'_, PyAny> = value + .py() + .import("builtins")? + .getattr("dict")? + .call1((value,))?; + json(&copied).map(Some) +} + +fn path(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestPathInput { + text: value.str()?.to_string(), + parent_parts: attribute(&attribute(value, "parent")?, "parts")?.extract()?, + stem: text(value, "stem")?, + }) +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestRelationInput { + database: optional_text(value, "database")?, + schema: optional_text(value, "schema")?, + name: optional_text(value, "name")?, + qualified_name: optional_text(value, "qualified_name")?, + }) +} + +fn column(column: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestColumnInput { + name: text(column, "name")?, + description: optional_text(column, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(column, "meta")?, true)?, + data_type: optional_text(column, "type")?, + }) +} + +fn columns(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(column).collect() +} + +fn hook( + entry: &Bound<'_, PyAny>, + classes: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let (sql_class, python_class) = classes; + if entry.is_instance(sql_class)? { + return Ok(ManifestHookInput::Sql { + statement: text(entry, "statement")?, + name: optional_text(entry, "name")?, + relative_path: optional_posix(entry, "relative_path")?, + definition_sql: optional_text(entry, "definition_sql")?, + kwargs: json(&attribute(entry, "kwargs")?)?, + description: optional_text(entry, "description")?, + }); + } + if entry.is_instance(python_class)? { + return Ok(ManifestHookInput::Python { + name: text(entry, "name")?, + kwargs: json(&attribute(entry, "kwargs")?)?, + }); + } + Ok(ManifestHookInput::Other) +} + +/// The entries when the hooks value is a list or tuple. +fn hooks(value: &Bound<'_, PyAny>) -> PyResult>> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + let module: Bound<'_, PyAny> = value.py().import(DISCOVERY_MODELS_MODULE)?.into_any(); + let sql_class: Bound<'_, PyAny> = module.getattr("SqlHookEntry")?; + let python_class: Bound<'_, PyAny> = module.getattr("PythonHookEntry")?; + value + .try_iter()? + .map(|entry| hook(&entry?, (&sql_class, &python_class))) + .collect::>() + .map(Some) +} + +fn tags(value: &Bound<'_, PyAny>) -> PyResult>> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + value + .try_iter()? + .map(|tag| Ok(tag?.str()?.to_string())) + .collect::>() + .map(Some) +} + +fn config_value<'py>( + values: &Bound<'py, PyAny>, + key: &str, + default: Bound<'py, PyAny>, +) -> PyResult> { + values.call_method1("get", (key, default)) +} + +fn config(values: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = values.py(); + let get = |key: &str| values.call_method1("get", (key,)); + let materialized: Bound<'_, PyAny> = config_value( + values, + "materialized", + PyString::new(py, DEFAULT_MATERIALIZATION).into_any(), + )?; + Ok(ManifestConfigInput { + enabled: json(&config_value( + values, + "enabled", + PyBool::new(py, true).to_owned().into_any(), + )?)?, + alias: json(&get("alias")?)?, + schema: json(&get("schema")?)?, + database: json(&get("database")?)?, + tags: tags(&get("tags")?)?, + materialized: if materialized.is_none() { + DEFAULT_MATERIALIZATION.to_owned() + } else { + materialized.str()?.to_string() + }, + incremental_strategy: json(&get("incremental_strategy")?)?, + merge_exclude_columns: json(&get("merge_exclude_columns")?)?, + full_refresh: json(&get("full_refresh")?)?, + unique_key: json(&get("unique_key")?)?, + on_schema_change: json(&config_value( + values, + "on_schema_change", + PyString::new(py, "ignore").into_any(), + )?)?, + pre_hooks: hooks(&get("pre_hooks")?)?, + post_hooks: hooks(&get("post_hooks")?)?, + }) +} + +fn dynamic_column(family: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestDynamicColumnInput { + name: json(&attribute(family, "name")?)?, + pivot_column: json(&attribute(family, "pivot_column")?)?, + value_column: json(&attribute(family, "value_column")?)?, + aggregate: json(&attribute(family, "aggregate")?)?, + data_type: json(&attribute(family, "type")?)?, + name_pattern: json(&attribute(family, "name_pattern")?)?, + }) +} + +fn model_schema(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + Ok(Some(ManifestModelSchemaInput { + description: optional_text(entry, "description")?, + meta: dict_copy(&attribute(entry, "meta")?, false)?, + columns: columns(entry, "columns")?, + dynamic_columns: items(entry, "dynamic_columns")? + .iter() + .map(dynamic_column) + .collect::>()?, + })) +} + +fn model(model: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestModelInput { + name: text(model, "name")?, + path: path(&attribute(model, "relative_path")?)?, + destination: relation(&attribute(model, "destination")?)?, + query_sql: text(model, "query_sql")?, + config: config(&attribute(&attribute(model, "config")?, "values")?)?, + schema: model_schema(&attribute(model, "schema_entry")?)?, + deps: keys(model, "deps")?, + }) +} + +fn source(source: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(source, "source_entry")?; + let cursor: Bound<'_, PyAny> = attribute(&entry, "cursor_column")?; + let unique_key: Bound<'_, PyAny> = attribute(&entry, "unique_key")?; + Ok(ManifestSourceInput { + name: text(source, "name")?, + path: attribute(&attribute(source, "source_file")?, "relative_path")? + .str()? + .to_string(), + entry_name: text(&entry, "name")?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?.filter(|text| !text.is_empty()), + table: optional_text(&entry, "table")?, + description: optional_text(&entry, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(&entry, "meta")?, true)?, + loader: optional_text(&entry, "loader")?, + write_strategy: enum_value(&entry, "write_strategy")?, + cursor_column: if cursor.is_none() { + None + } else { + Some(json(&cursor)?) + }, + unique_key: if unique_key.is_truthy()? { + Some(json(&unique_key)?) + } else { + None + }, + columns: columns(&entry, "columns")?, + }) +} + +fn seed(seed: &Bound<'_, PyAny>) -> PyResult { + let entry: Bound<'_, PyAny> = attribute(seed, "schema_entry")?; + Ok(ManifestSeedInput { + name: text(seed, "name")?, + path: path(&attribute(&attribute(seed, "seed_file")?, "relative_path")?)?, + destination: relation(&attribute(seed, "destination")?)?, + tags: texts(&entry, "tags")?, + description: optional_text(&entry, "description")?.filter(|text| !text.is_empty()), + meta: dict_copy(&attribute(&entry, "meta")?, true)?, + columns: columns(&entry, "columns")?, + }) +} + +fn macros(project: &Bound<'_, PyAny>) -> PyResult> { + let loaded: Bound<'_, PyAny> = attribute(project, "loaded_macros")?; + let mut macros: Vec = Vec::new(); + for (key, loaded_macro) in loaded.downcast::()?.iter() { + macros.push(ManifestMacroInput { + key: key.extract()?, + name: text(&loaded_macro, "name")?, + relative_path: optional_posix(&loaded_macro, "relative_path")?.unwrap_or_default(), + raw_source: text(&loaded_macro, "raw_source")?, + }); + } + Ok(macros) +} + +fn macro_declarations(project: &Bound<'_, PyAny>) -> PyResult> { + let scope_index: Bound<'_, PyAny> = attribute(project, "scope_index")?; + if scope_index.is_none() { + return Ok(Vec::new()); + } + let mut declarations: Vec = Vec::new(); + for record in items(&scope_index, "declarations")? { + let identity: Bound<'_, PyAny> = attribute(&record, "identity")?; + if enum_value(&identity, "kind")?.as_deref() != Some(MACRO_KIND) { + continue; + } + let facts: Bound<'_, PyAny> = attribute(&record, "macro")?; + declarations.push(ManifestMacroDeclarationInput { + name: text(&identity, "name")?, + scope: enum_value(&record, "scope")?.unwrap_or_default(), + owning_path: optional_text(&record, "owning_path")?, + dependencies: if facts.is_none() { + None + } else { + Some( + items(&facts, "dependencies")? + .iter() + .map(|dependency| text(dependency, "name")) + .collect::>()?, + ) + }, + }); + } + Ok(declarations) +} + +fn sql_hook(hook: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestSqlHookInput { + name: text(hook, "name")?, + relative_path: optional_posix(hook, "relative_path")?.unwrap_or_default(), + sql_body: text(hook, "sql_body")?, + description: optional_text(hook, "description")?.filter(|text| !text.is_empty()), + }) +} + +/// Each discovered Python hook's path and identity hashes. +pub(crate) fn python_hooks( + project: &Bound<'_, PyAny>, + identities: &Bound<'_, PyAny>, +) -> PyResult> { + items(project, "hook_functions")? + .iter() + .map(|hook| { + let name: String = text(hook, "name")?; + let identity: Bound<'_, PyAny> = identities.get_item(&name)?; + Ok(ManifestPythonHookInput { + relative_path: optional_posix(hook, "relative_path")?.unwrap_or_default(), + definition_hash: text(&identity, "definition_hash")?, + version_hash: text(&identity, "version_hash")?, + name, + }) + }) + .collect() +} + +fn audit_plan(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(ManifestAuditPlanInput { + name: text(entry, "name")?, + resolved_sql: text(entry, "resolved_sql")?, + attached_target_name: optional_text(entry, "attached_target_name")?, + attached_column_name: optional_text(entry, "attached_column_name")?, + scope_deps: keys(entry, "scope_deps")?, + }) +} + +fn sql_test_plan(entry: &Bound<'_, PyAny>) -> PyResult { + let mut parameters: Vec<(String, PythonValue)> = Vec::new(); + for pair in items(entry, "parameter_values")? { + let (name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let raw: Bound<'_, PyAny> = attribute(&value, "value")?; + let converted: PythonValue = if enum_value(&value, "kind")?.as_deref() == Some(DECIMAL_KIND) + { + PythonValue::String(raw.str()?.to_string()) + } else { + json(&raw)? + }; + parameters.push((name, converted)); + } + Ok(ManifestSqlTestPlanInput { + key: object_key(&attribute(entry, "key")?)?, + name: text(entry, "name")?, + chain: items(entry, "chain")? + .iter() + .map(|step| Ok((text(step, "model_name")?, text(step, "resolved_sql")?))) + .collect::>()?, + scope_deps: keys(entry, "scope_deps")?, + source_path: optional_posix(entry, "source_path")?, + block_index: json(&attribute(entry, "block_index")?)?, + parent_name: json(&attribute(entry, "parent_name")?)?, + case_name: optional_text(entry, "case_name")?, + case_index: json(&attribute(entry, "case_index")?)?, + case_fingerprint: optional_text(entry, "case_fingerprint")?, + parameter_schema: items(entry, "parameter_schema")? + .iter() + .map(|parameter| { + Ok(( + text(parameter, "name")?, + enum_value(parameter, "value_type")?.unwrap_or_default(), + json(&attribute(parameter, "nullable")?)?, + )) + }) + .collect::>()?, + parameters, + }) +} + +/// The plan entries the manifest reads. +pub(crate) fn plan_input(plan_output: &Bound<'_, PyAny>) -> PyResult> { + if plan_output.is_none() { + return Ok(None); + } + Ok(Some(ManifestPlanInput { + models: items(plan_output, "model_entries")? + .iter() + .map(|entry| Ok((text(entry, "name")?, text(entry, "resolved_sql")?))) + .collect::>()?, + source_load_names: items(plan_output, "source_load_entries")? + .iter() + .map(|entry| text(entry, "name")) + .collect::>()?, + audits: items(plan_output, "audit_entries")? + .iter() + .map(audit_plan) + .collect::>()?, + sql_tests: items(plan_output, "test_entries")? + .iter() + .map(sql_test_plan) + .collect::>()?, + })) +} + +/// The project's models, sources, seeds, macros and hooks. +pub(crate) struct ProjectRows { + pub(crate) run_id: String, + pub(crate) sql_analysis_dialect: Option, + pub(crate) models: Vec, + pub(crate) sources: Vec, + pub(crate) seeds: Vec, + pub(crate) macros: Vec, + pub(crate) macro_declarations: Vec, + pub(crate) sql_hooks: Vec, +} + +/// Read the compiled project. +pub(crate) fn project_rows(project: &Bound<'_, PyAny>) -> PyResult { + Ok(ProjectRows { + run_id: text(project, "run_id")?, + sql_analysis_dialect: optional_text(project, "sql_analysis_dialect")?, + models: items(project, "models")? + .iter() + .map(model) + .collect::>()?, + sources: items(project, "sources")? + .iter() + .map(source) + .collect::>()?, + seeds: items(project, "seeds")? + .iter() + .map(seed) + .collect::>()?, + macros: macros(project)?, + macro_declarations: macro_declarations(project)?, + sql_hooks: items(project, "sql_hook_files")? + .iter() + .map(sql_hook) + .collect::>()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index fac37883a5..ec29b33bb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -11,8 +11,10 @@ pub(crate) mod dag_artifact; pub(crate) mod discovery; pub(crate) mod graph; pub(crate) mod lineage; +pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod project_assembly; +pub(crate) mod python_values; pub(crate) mod refactor; pub(crate) mod render; pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs new file mode 100644 index 0000000000..4b9235503b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs @@ -0,0 +1 @@ +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs rename to crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs index 0c7750f27d..7ec29a1a92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs @@ -1,19 +1,18 @@ -//! Python values as `dataclasses.asdict` copies them and `json.dumps` would encode them. +//! Python values as `json.dumps` would encode them, optionally after `dataclasses.asdict`. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyStringMethods, PyTuple, PyTypeMethods, }; -use sqlbuild_analysis::dag_artifact::models::DagValue; -use sqlbuild_analysis::dag_artifact::types::DagKey; +use sqlbuild_analysis::python_values::models::{PythonKey, PythonValue}; use sqlbuild_core::json::models::JsonInteger; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_outputs::json_report::float_key; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; -const SURROGATE_KEY: &str = "a DAG artifact mapping key holds a lone surrogate, which the native \ +const SURROGATE_KEY: &str = "a JSON artifact mapping key holds a lone surrogate, which the native \ encoder does not write"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -22,14 +21,14 @@ fn type_name(value: &Bound<'_, PyAny>) -> PyResult { } /// A `str`, as UTF-16 code units when it holds lone surrogates. -pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { +pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { if let Ok(text) = text.to_str() { - return Ok(DagValue::String(text.to_owned())); + return Ok(PythonValue::String(text.to_owned())); } let encoded: Bound<'_, PyBytes> = text .call_method1("encode", ("utf-16-le", "surrogatepass"))? .downcast_into::()?; - Ok(DagValue::Utf16( + Ok(PythonValue::Utf16( encoded .as_bytes() .as_chunks::<2>() @@ -40,25 +39,35 @@ pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { )) } -fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { +fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(signed) = value.extract::() { - return Ok(DagValue::Integer(JsonInteger::from(signed))); + return Ok(PythonValue::Integer(JsonInteger::from(signed))); } if let Ok(unsigned) = value.extract::() { - return Ok(DagValue::Integer(JsonInteger::from(unsigned))); + return Ok(PythonValue::Integer(JsonInteger::from(unsigned))); } let decimal: String = value.str()?.to_str()?.to_owned(); - Ok(JsonInteger::parse(&decimal).map_or(DagValue::Null, DagValue::Integer)) + Ok(JsonInteger::parse(&decimal).map_or(PythonValue::Null, PythonValue::Integer)) } /// A mapping key as `json.dumps` writes it, or its `TypeError` message. -fn key_value(key: &Bound<'_, PyAny>) -> PyResult { +fn key_value(key: &Bound<'_, PyAny>) -> PyResult { if let Ok(text) = key.downcast::() { return text .to_str() - .map(|text| Ok(text.to_owned())) + .map(|text| PythonKey { + text: Ok(text.to_owned()), + is_str: true, + }) .map_err(|_| NativeCompilerError::new_err(SURROGATE_KEY)); } + Ok(PythonKey { + text: scalar_key_text(key)?, + is_str: false, + }) +} + +fn scalar_key_text(key: &Bound<'_, PyAny>) -> PyResult> { if key.is_none() { return Ok(Ok("null".to_owned())); } @@ -77,58 +86,58 @@ fn key_value(key: &Bound<'_, PyAny>) -> PyResult { ))) } -fn dataclass_value(value: &Bound<'_, PyAny>) -> PyResult { +fn dataclass_value(value: &Bound<'_, PyAny>) -> PyResult { let fields: Bound<'_, PyAny> = value .py() .import(DATACLASSES_MODULE)? .getattr("fields")? .call1((value,))?; - let mut entries: Vec<(DagKey, DagValue)> = Vec::new(); + let mut entries: Vec<(PythonKey, PythonValue)> = Vec::new(); for field in fields.try_iter()? { let name: Bound<'_, PyAny> = field?.getattr("name")?; let item: Bound<'_, PyAny> = value.getattr(name.downcast::()?)?; - entries.push((key_value(&name)?, dag_value(&item)?)); + entries.push((key_value(&name)?, python_value(&item, true)?)); } - Ok(DagValue::Object(entries)) + Ok(PythonValue::Object(entries)) } -/// The value `asdict` copies into the artifact; refused leaves keep `json.dumps`'s message. -pub(crate) fn dag_value(value: &Bound<'_, PyAny>) -> PyResult { +/// The value `json.dumps` encodes, copied by `asdict` first when `dataclasses` is set. +pub(crate) fn python_value(value: &Bound<'_, PyAny>, dataclasses: bool) -> PyResult { if value.is_none() { - return Ok(DagValue::Null); + return Ok(PythonValue::Null); } - if value.get_type().hasattr(DATACLASS_FIELDS)? { + if dataclasses && value.get_type().hasattr(DATACLASS_FIELDS)? { return dataclass_value(value); } if value.is_instance_of::() || value.is_instance_of::() { - return Ok(DagValue::Array( + return Ok(PythonValue::Array( value .try_iter()? - .map(|item| dag_value(&item?)) + .map(|item| python_value(&item?, dataclasses)) .collect::>()?, )); } if let Ok(mapping) = value.downcast::() { - return Ok(DagValue::Object( + return Ok(PythonValue::Object( mapping .iter() - .map(|(key, item)| Ok((key_value(&key)?, dag_value(&item)?))) + .map(|(key, item)| Ok((key_value(&key)?, python_value(&item, dataclasses)?))) .collect::>()?, )); } if let Ok(flag) = value.downcast::() { - return Ok(DagValue::Bool(flag.is_true())); + return Ok(PythonValue::Bool(flag.is_true())); } if value.is_instance_of::() { return integer_value(value); } if value.is_instance_of::() { - return Ok(DagValue::Float(value.extract::()?)); + return Ok(PythonValue::Float(value.extract::()?)); } if let Ok(text) = value.downcast::() { return text_value(text); } - Ok(DagValue::Unencodable(format!( + Ok(PythonValue::Unencodable(format!( "Object of type {} is not JSON serializable", type_name(value)? ))) diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 09b575c735..2f3f2d11da 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,7 +5,7 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, dag_artifact, discovery, graph, lineage, model_config, + compiled_project, contracts, dag_artifact, discovery, graph, lineage, manifest, model_config, project_assembly, refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, type_system, }; @@ -59,5 +59,6 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { compiled_project::registration::register(module)?; compile_driver::registration::register(module)?; dag_artifact::registration::register(module)?; + manifest::registration::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index ef6ca609f9..ed6a047b38 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 135550 +remaining_lines = 134615 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3e4ea25cd0..d216eca47c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -78,7 +78,8 @@ from sqlbuild.compiler.pipeline.models import ( CompilePipelineResult, ProjectGraph, ) -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( @@ -794,6 +795,23 @@ def build_dag_json( *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None ) -> str: ... +class _ManifestFacts(TypedDict): + project_name: str + adapter_type: str + generated_at: str + hook_identities: Mapping[str, PythonNodeIdentity] + +class NativeManifest: + def contents(self) -> bytes: ... + +def build_manifest( + *, + project: CompiledProjectView, + plan_output: PlanOutput | None, + graph: NativeProjectGraph, + facts: _ManifestFacts, +) -> NativeManifest: ... + class NativeProjectGraph: @staticmethod def from_resources( diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py index 6bbc59afbd..c9d62648a6 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlbuild._native as _native from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target from sqlbuild.cli.commands.models import ( BuildCommandRequest, @@ -19,7 +20,7 @@ def write_build_compile_target( ) -> None: """Write the compile target artifacts with the optional manifest payload.""" - manifest_payload: dict[str, object] | None = None + manifest_payload: _native.NativeManifest | None = None if request.manifest: manifest_payload = build_manifest( project=pipeline_result.project, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index bc1be4841d..52ae0b17d3 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from collections.abc import Callable from functools import partial @@ -78,7 +77,7 @@ def write_compile_target( target_dir: Path, adapter: BaseAdapter, plan_output: PlanOutput, - manifest: dict[str, object] | None = None, + manifest: _native.NativeManifest | None = None, ) -> WrittenTarget: """Write compiled output files under target_dir.""" @@ -115,7 +114,7 @@ def write_static_compile_target( target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView, - manifest: dict[str, object] | None = None, + manifest: _native.NativeManifest | None = None, planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, ) -> WrittenTarget: """Write offline compiled output files under target_dir, reusing SQL tests planned ahead.""" @@ -166,7 +165,7 @@ def publish_static_compile_target( *, prepared: WrittenTarget, target_dir: Path, - manifest: dict[str, object] | None, + manifest: _native.NativeManifest | None, expected_files: frozenset[str], ) -> WrittenTarget: """Publish staged files with the same unchanged-file and stale-file semantics.""" @@ -583,12 +582,12 @@ def _sql_test_artifact_diagnostics( ) -def _write_manifest(*, target_dir: Path, manifest: dict[str, object]) -> None: +def _write_manifest(*, target_dir: Path, manifest: _native.NativeManifest) -> None: """Write manifest.json.""" manifest_path: Path = target_dir / _MANIFEST_FILE batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=True) - batch.queue(path=manifest_path, contents=(json.dumps(manifest, indent=2) + "\n").encode()) + batch.queue(path=manifest_path, contents=manifest.contents()) batch.flush() diff --git a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py index 83d9d6b5f3..eedd5ec7d3 100644 --- a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py +++ b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py @@ -12,6 +12,7 @@ from filelock import FileLock, Timeout +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import ( plan_static_sql_tests, @@ -148,7 +149,7 @@ def prepare() -> _PreparedResult: return True def publish( - self, *, target_dir: Path, manifest: dict[str, object] | None + self, *, target_dir: Path, manifest: _native.NativeManifest | None ) -> WrittenTarget | None: if self._planned_tests is not None and self._planning_inputs is not None: project, adapter, _ = self._planning_inputs diff --git a/src/sqlbuild/compiler/manifest/_helpers/__init__.py b/src/sqlbuild/compiler/manifest/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/manifest/_helpers/macros.py b/src/sqlbuild/compiler/manifest/_helpers/macros.py deleted file mode 100644 index 4f157bd70a..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/macros.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Macro node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_MACRO -from sqlbuild.compiler.scopes.models import DeclarationIdentity, DeclarationRecord, ScopeIndex -from sqlbuild.compiler.scopes.types import DeclarationKind - - -def build_macro_node( - *, - loaded_macro: LoadedMacro, - project_name: str, - dependencies: tuple[DeclarationIdentity, ...] = (), - scope_record: DeclarationRecord | None = None, -) -> dict[str, object]: - """Build one dbt-compatible macro node dict.""" - - unique_id: str = f"macro.{project_name}.{loaded_macro.name}" - - return { - "name": loaded_macro.name, - "resource_type": RESOURCE_TYPE_MACRO, - "package_name": project_name, - "path": loaded_macro.relative_path.as_posix(), - "original_file_path": loaded_macro.relative_path.as_posix(), - "unique_id": unique_id, - "macro_sql": loaded_macro.raw_source, - "depends_on": { - "macros": [f"macro.{project_name}.{dependency.name}" for dependency in dependencies] - }, - "description": "", - "meta": { - "sqlbuild_visibility": ( - scope_record.scope.value if scope_record is not None else "global" - ), - "sqlbuild_scope_path": (scope_record.owning_path if scope_record is not None else None), - }, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "arguments": [], - "created_at": 0.0, - "supported_languages": None, - } - - -def build_sql_hook_macro_node( - *, hook_file: DiscoveredSqlHookFile, project_name: str -) -> dict[str, object]: - """Expose one SQL hook definition through a dbt-compatible macro node.""" - - hook_package_name: str = f"{project_name}__sqlbuild_hooks" - return { - "name": hook_file.name, - "resource_type": RESOURCE_TYPE_MACRO, - "package_name": hook_package_name, - "path": hook_file.relative_path.as_posix(), - "original_file_path": hook_file.relative_path.as_posix(), - "unique_id": f"macro.{hook_package_name}.{hook_file.name}", - "macro_sql": hook_file.sql_body, - "depends_on": {"macros": []}, - "description": hook_file.description or "", - "meta": { - "sqlbuild_resource_type": "sql_hook", - "sqlbuild_hook_name": hook_file.name, - }, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "arguments": [], - "created_at": 0.0, - "supported_languages": None, - } - - -def build_manifest_macro_nodes( - *, - loaded_macros: dict[str, LoadedMacro], - sql_hook_files: tuple[DiscoveredSqlHookFile, ...], - project_name: str, - scope_index: ScopeIndex | None = None, -) -> dict[str, dict[str, object]]: - """Build all dbt-compatible macro entries, including SQL hook definitions.""" - - macros: dict[str, dict[str, object]] = {} - dependency_map: dict[str, tuple[DeclarationIdentity, ...]] = { - record.identity.name: record.macro.dependencies - for record in (() if scope_index is None else scope_index.declarations) - if record.identity.kind is DeclarationKind.MACRO and record.macro is not None - } - scope_records: dict[str, DeclarationRecord] = { - record.identity.name: record - for record in (() if scope_index is None else scope_index.declarations) - if record.identity.kind is DeclarationKind.MACRO - } - macro_name: str - loaded_macro: LoadedMacro - for macro_name, loaded_macro in loaded_macros.items(): - unique_id: str = f"macro.{project_name}.{macro_name}" - macros[unique_id] = build_macro_node( - loaded_macro=loaded_macro, - project_name=project_name, - dependencies=dependency_map.get(macro_name, ()), - scope_record=scope_records.get(macro_name), - ) - hook_file: DiscoveredSqlHookFile - for hook_file in sql_hook_files: - unique_id = f"macro.{project_name}__sqlbuild_hooks.{hook_file.name}" - macros[unique_id] = build_sql_hook_macro_node( - hook_file=hook_file, - project_name=project_name, - ) - return macros diff --git a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py deleted file mode 100644 index 095a59dc26..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Model node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS -from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash -from sqlbuild.compiler.manifest._helpers.shared import ( - build_columns_dict, - build_config_dict, - build_depends_on, - build_fqn, -) -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_MODEL -from sqlbuild.compiler.planner.models import ModelPlanEntry - - -def build_model_node( - *, - model: CompiledModelView, - plan_entry: ModelPlanEntry | None, - project_name: str, - python_hook_metadata: dict[str, dict[str, object]] | None = None, - sql_analysis_dialect: str | None = None, -) -> dict[str, object]: - """Build one dbt-compatible model node dict.""" - - unique_id: str = f"model.{project_name}.{model.name}" - relative_path: Path = model.relative_path - raw_code: str = model.query_sql - compiled_code: str = plan_entry.resolved_sql if plan_entry is not None else raw_code - query_hash: str = compute_query_hash(query_sql=raw_code, dialect=sql_analysis_dialect) - - return { - "database": model.destination.database, - "schema": model.destination.schema, - "name": model.name, - "resource_type": RESOURCE_TYPE_MODEL, - "package_name": project_name, - "path": str(relative_path), - "original_file_path": str(relative_path), - "unique_id": unique_id, - "fqn": build_fqn(project_name=project_name, relative_path=relative_path), - "alias": model.destination.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": query_hash, - }, - "config": build_config_dict(model.config.values), - "tags": _extract_tags(model), - "description": _extract_description(model), - "columns": build_columns_dict(model.schema_entry), - "meta": _extract_meta( - model=model, - python_hook_metadata=python_hook_metadata or {}, - ), - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": model.destination.qualified_name, - "raw_code": raw_code, - "compiled_code": compiled_code, - "depends_on": build_depends_on( - deps=model.deps, - project_name=project_name, - ), - } - - -def _extract_tags(model: CompiledModelView) -> list[str]: - """Extract tags from model config.""" - - raw: object | None = model.config.values.get("tags") - if isinstance(raw, list): - return [str(item) for item in raw] - if isinstance(raw, tuple): - return [str(item) for item in raw] - return [] - - -def _extract_description(model: CompiledModelView) -> str: - """Extract description from schema entry.""" - - if model.schema_entry is not None and model.schema_entry.description is not None: - return model.schema_entry.description - return "" - - -def _extract_meta( - *, model: CompiledModelView, python_hook_metadata: dict[str, dict[str, object]] -) -> dict[str, object]: - """Extract meta from schema entry.""" - - meta: dict[str, object] = {} - if model.schema_entry is not None and model.schema_entry.meta is not None: - meta.update(model.schema_entry.meta) - lifecycle_hooks: dict[str, list[dict[str, object]]] = {} - hook_key: str - for hook_key in ("pre_hooks", "post_hooks"): - serialized: list[dict[str, object]] = _serialize_hooks( - value=model.config.values.get(hook_key), - python_hook_metadata=python_hook_metadata, - ) - if serialized: - lifecycle_hooks[hook_key] = serialized - if lifecycle_hooks: - authored_sqlbuild_meta: object | None = meta.get("sqlbuild") - sqlbuild_meta: dict[str, object] = {} - if isinstance(authored_sqlbuild_meta, dict): - authored_key: object - authored_value: object - for authored_key, authored_value in authored_sqlbuild_meta.items(): - if isinstance(authored_key, str): - sqlbuild_meta[authored_key] = authored_value - elif authored_sqlbuild_meta is not None: - sqlbuild_meta["authored_value"] = authored_sqlbuild_meta - sqlbuild_meta["lifecycle_hooks"] = lifecycle_hooks - meta["sqlbuild"] = sqlbuild_meta - if model.schema_entry is not None and model.schema_entry.dynamic_columns: - authored_sqlbuild_meta = meta.get("sqlbuild") - sqlbuild_meta: dict[str, object] = {} - if isinstance(authored_sqlbuild_meta, dict): - authored_key: object - authored_value: object - for authored_key, authored_value in authored_sqlbuild_meta.items(): - if isinstance(authored_key, str): - sqlbuild_meta[authored_key] = authored_value - if authored_sqlbuild_meta is not None and not isinstance(authored_sqlbuild_meta, dict): - sqlbuild_meta["authored_value"] = authored_sqlbuild_meta - sqlbuild_meta["dynamic_columns"] = [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in model.schema_entry.dynamic_columns - ] - meta["sqlbuild"] = sqlbuild_meta - return meta - - -def _serialize_hooks( - *, value: object, python_hook_metadata: dict[str, dict[str, object]] -) -> list[dict[str, object]]: - return serialize_hook_entries( - value=value, - sql_fields=SQL_HOOK_OUTPUT_FIELDS, - python_hook_fields=python_hook_metadata, - ) diff --git a/src/sqlbuild/compiler/manifest/_helpers/payload.py b/src/sqlbuild/compiler/manifest/_helpers/payload.py deleted file mode 100644 index e68790fa21..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/payload.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Build a dbt-compatible manifest.json payload from compiled project state.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.compiler.manifest._helpers.macros import ( - build_manifest_macro_nodes, -) -from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node -from sqlbuild.compiler.manifest._helpers.seeds import build_seed_node -from sqlbuild.compiler.manifest._helpers.sources import build_source_node -from sqlbuild.compiler.manifest._helpers.tests import ( - build_audit_test_nodes, - build_sql_test_nodes, -) -from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION, RESOURCE_TYPE_PREFIX -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ModelPlanEntry, - PlanOutput, - SqlTestPlanEntry, -) -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity - - -def build_manifest_payload( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None, - project_name: str, - adapter_type: str, - graph: _native.NativeProjectGraph, -) -> dict[str, object]: - """A full dbt v12-compatible manifest; parent and child maps come from the native graph.""" - - model_plan_map: dict[str, ModelPlanEntry] = {} - source_load_names: frozenset[str] = frozenset() - if plan_output is not None: - model_plan_map = {entry.name: entry for entry in plan_output.model_entries} - source_load_names = frozenset(entry.name for entry in plan_output.source_load_entries) - - nodes: dict[str, dict[str, object]] = {} - sources: dict[str, dict[str, object]] = {} - macros: dict[str, dict[str, object]] = build_manifest_macro_nodes( - loaded_macros=project.loaded_macros, - sql_hook_files=project.sql_hook_files, - project_name=project_name, - scope_index=project.scope_index, - ) - python_hook_metadata: dict[str, dict[str, object]] = _build_python_hook_metadata(project) - - model: CompiledModelView - for model in project.models: - plan_entry: ModelPlanEntry | None = model_plan_map.get(model.name) - unique_id: str = f"model.{project_name}.{model.name}" - nodes[unique_id] = build_model_node( - model=model, - plan_entry=plan_entry, - project_name=project_name, - python_hook_metadata=python_hook_metadata, - sql_analysis_dialect=project.sql_analysis_dialect, - ) - - source: CompiledSourceView - for source in project.sources: - unique_id = f"source.{project_name}.{source.name}" - sources[unique_id] = build_source_node( - source=source, - project_name=project_name, - auto_load=source.name in source_load_names, - ) - - seed: CompiledSeedView - for seed in project.seeds: - unique_id = f"seed.{project_name}.{seed.name}" - nodes[unique_id] = build_seed_node( - seed=seed, - project_name=project_name, - ) - - if plan_output is not None: - audit_entry: AuditPlanEntry - for audit_entry in plan_output.audit_entries: - audit_nodes: dict[str, dict[str, object]] = build_audit_test_nodes( - audit_entry=audit_entry, - project_name=project_name, - ) - nodes.update(audit_nodes) - - test_entry: SqlTestPlanEntry - for test_entry in plan_output.test_entries: - test_nodes: dict[str, dict[str, object]] = build_sql_test_nodes( - test_entry=test_entry, - project_name=project_name, - ) - nodes.update(test_nodes) - - parent_rows, child_rows = graph.unique_id_maps(project_name, dict(RESOURCE_TYPE_PREFIX)) - parent_map: dict[str, list[str]] = dict(parent_rows) - child_map: dict[str, list[str]] = dict(child_rows) - - return { - "metadata": _build_metadata( - project_name=project_name, - adapter_type=adapter_type, - run_id=project.run_id, - ), - "nodes": nodes, - "sources": sources, - "macros": macros, - "exposures": {}, - "metrics": {}, - "groups": {}, - "selectors": {}, - "disabled": {}, - "parent_map": parent_map, - "child_map": child_map, - "group_map": {}, - "docs": {}, - "saved_queries": {}, - "semantic_models": {}, - "unit_tests": {}, - } - - -def _build_python_hook_metadata(project: CompiledProjectView) -> dict[str, dict[str, object]]: - hook_identities: dict[str, PythonNodeIdentity] = build_hook_identities(project.hook_functions) - return { - hook.name: { - "relative_path": hook.relative_path.as_posix(), - "definition_hash": hook_identities[hook.name].definition_hash, - "version_hash": hook_identities[hook.name].version_hash, - } - for hook in project.hook_functions - } - - -def _build_metadata( - *, - project_name: str, - adapter_type: str, - run_id: str, -) -> dict[str, object]: - """Build the top-level manifest metadata block.""" - - return { - "dbt_schema_version": DBT_MANIFEST_SCHEMA_VERSION, - "dbt_version": "1.12.0", - "generated_at": datetime.now(tz=UTC).isoformat(), - "invocation_id": run_id, - "env": {}, - "project_name": project_name, - "project_id": None, - "user_id": None, - "send_anonymous_usage_stats": False, - "adapter_type": adapter_type, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/seeds.py b/src/sqlbuild/compiler/manifest/_helpers/seeds.py deleted file mode 100644 index abe698b282..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/seeds.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Seed node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.manifest._helpers.shared import build_columns_dict, build_fqn -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_SEED - - -def build_seed_node( - *, - seed: CompiledSeedView, - project_name: str, -) -> dict[str, object]: - """Build one dbt-compatible seed node dict.""" - - unique_id: str = f"seed.{project_name}.{seed.name}" - relative_path_str: str = str(seed.seed_file.relative_path) - - return { - "database": seed.destination.database, - "schema": seed.destination.schema, - "name": seed.name, - "resource_type": RESOURCE_TYPE_SEED, - "package_name": project_name, - "path": relative_path_str, - "original_file_path": relative_path_str, - "unique_id": unique_id, - "fqn": build_fqn( - project_name=project_name, - relative_path=seed.seed_file.relative_path, - ), - "alias": seed.destination.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": "", - }, - "config": { - "enabled": True, - "alias": None, - "schema": seed.destination.schema, - "database": seed.destination.database, - "tags": list(seed.schema_entry.tags), - "meta": {}, - "group": None, - "materialized": "seed", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "quoting": {}, - "column_types": {}, - "full_refresh": None, - "unique_key": None, - "on_schema_change": "ignore", - "on_configuration_change": "apply", - "grants": {}, - "packages": [], - "docs": {"show": True, "node_color": None}, - "contract": {"enforced": False, "alias_types": True}, - "delimiter": ",", - "quote_columns": None, - }, - "tags": list(seed.schema_entry.tags), - "description": seed.schema_entry.description or "", - "columns": build_columns_dict(seed.schema_entry), - "meta": dict(seed.schema_entry.meta) if seed.schema_entry.meta else {}, - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": seed.destination.qualified_name, - "raw_code": "", - "root_path": None, - "depends_on": {"macros": []}, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/shared.py b/src/sqlbuild/compiler/manifest/_helpers/shared.py deleted file mode 100644 index d9bdd0aa70..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/shared.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Shared serialization helpers for manifest node building.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_PREFIX -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, -) - - -def build_fqn(*, project_name: str, relative_path: Path) -> list[str]: - """Build a dbt-style fully qualified name list from project name and path.""" - - parts: list[str] = [project_name] - parent: Path = relative_path.parent - part: str - for part in parent.parts: - parts.append(part) - stem: str = relative_path.stem - parts.append(stem) - return parts - - -def build_columns_dict( - schema_entry: SchemaModelEntry | SchemaSeedEntry | None, -) -> dict[str, dict[str, object]]: - """Build a dbt-compatible columns dict from schema entry columns.""" - - if schema_entry is None: - return {} - result: dict[str, dict[str, object]] = {} - column: SchemaColumn - for column in schema_entry.columns: - result[column.name] = { - "name": column.name, - "description": column.description or "", - "meta": dict(column.meta) if column.meta else {}, - "data_type": column.type, - "constraints": [], - "quote": None, - "tags": [], - } - return result - - -def build_source_columns_dict( - columns: tuple[SourceColumnEntry, ...], -) -> dict[str, dict[str, object]]: - """Build a dbt-compatible columns dict from source column entries.""" - - result: dict[str, dict[str, object]] = {} - column: SourceColumnEntry - for column in columns: - result[column.name] = { - "name": column.name, - "description": column.description or "", - "meta": dict(column.meta) if column.meta else {}, - "data_type": column.type, - "constraints": [], - "quote": None, - "tags": [], - } - return result - - -def build_config_dict(values: dict[str, object]) -> dict[str, object]: - """Build a dbt-compatible config dict from model config values.""" - - materialized: object = values.get("materialized", "view") - result: dict[str, object] = { - "enabled": values.get("enabled", True), - "alias": values.get("alias"), - "schema": values.get("schema"), - "database": values.get("database"), - "tags": _config_tags(values), - "meta": {}, - "group": None, - "materialized": str(materialized) if materialized is not None else "view", - "incremental_strategy": values.get("incremental_strategy"), - "merge_exclude_columns": values.get("merge_exclude_columns"), - "persist_docs": {}, - "post-hook": _dbt_sql_hooks(values.get("post_hooks")), - "pre-hook": _dbt_sql_hooks(values.get("pre_hooks")), - "quoting": {}, - "column_types": {}, - "full_refresh": values.get("full_refresh"), - "unique_key": values.get("unique_key"), - "on_schema_change": values.get("on_schema_change", "ignore"), - "on_configuration_change": "apply", - "grants": {}, - "packages": [], - "docs": {"show": True, "node_color": None}, - "contract": {"enforced": False, "alias_types": True}, - } - return result - - -def _dbt_sql_hooks(value: object) -> list[dict[str, object]]: - if not isinstance(value, list | tuple): - return [] - hooks: list[dict[str, object]] = [] - entry: object - for entry in value: - if not isinstance(entry, SqlHookEntry): - continue - hooks.append( - { - "sql": entry.statement, - "transaction": True, - "index": len(hooks), - } - ) - return hooks - - -def build_depends_on( - *, - deps: tuple[CompiledObjectKey, ...], - project_name: str, -) -> dict[str, list[str]]: - """Build a dbt-compatible depends_on dict from compiled deps.""" - - node_ids: list[str] = [] - dep: CompiledObjectKey - for dep in deps: - dep_id: str = _key_to_unique_id(key=dep, project_name=project_name) - node_ids.append(dep_id) - return { - "macros": [], - "nodes": node_ids, - } - - -def _key_to_unique_id(*, key: CompiledObjectKey, project_name: str) -> str: - """Convert a CompiledObjectKey to a dbt-style unique_id.""" - - resource_prefix: str = RESOURCE_TYPE_PREFIX.get(str(key.resource_type), "model") - return f"{resource_prefix}.{project_name}.{key.name}" - - -def _config_tags(values: dict[str, object]) -> list[str]: - """Extract tags from config values as a list.""" - - raw: object | None = values.get("tags") - if isinstance(raw, list): - return [str(item) for item in raw] - if isinstance(raw, tuple): - return [str(item) for item in raw] - return [] diff --git a/src/sqlbuild/compiler/manifest/_helpers/sources.py b/src/sqlbuild/compiler/manifest/_helpers/sources.py deleted file mode 100644 index 07c107a786..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/sources.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Source node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledSourceView -from sqlbuild.compiler.manifest._helpers.shared import build_source_columns_dict -from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_SOURCE -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_source_node( - *, - source: CompiledSourceView, - project_name: str, - auto_load: bool = False, -) -> dict[str, object]: - """Build one dbt-compatible source node dict.""" - - entry: SourceEntry = source.source_entry - unique_id: str = f"source.{project_name}.{entry.name}" - identifier: str = entry.table if entry.table is not None else entry.name - - meta: dict[str, object] = dict(entry.meta) if entry.meta else {} - if entry.loader is not None: - sqlbuild_meta: dict[str, object] = { - "loader": entry.loader, - "auto_load": auto_load, - } - if entry.write_strategy is not None: - sqlbuild_meta["write_strategy"] = entry.write_strategy.value - if entry.cursor_column is not None: - sqlbuild_meta["cursor_column"] = entry.cursor_column - if entry.unique_key: - sqlbuild_meta["unique_key"] = entry.unique_key - meta["sqlbuild"] = sqlbuild_meta - - return { - "database": entry.database, - "schema": entry.schema or "", - "name": entry.name, - "resource_type": RESOURCE_TYPE_SOURCE, - "package_name": project_name, - "path": str(source.source_file.relative_path), - "original_file_path": str(source.source_file.relative_path), - "unique_id": unique_id, - "fqn": [project_name, entry.name], - "source_name": entry.name, - "source_description": "", - "loader": "", - "identifier": identifier, - "quoting": {}, - "loaded_at_field": None, - "freshness": None, - "external": None, - "description": entry.description or "", - "columns": build_source_columns_dict(entry.columns), - "meta": meta, - "source_meta": {}, - "tags": [], - "config": {}, - "patch_path": None, - "unrendered_config": {}, - "relation_name": None, - "created_at": 0.0, - } diff --git a/src/sqlbuild/compiler/manifest/_helpers/tests.py b/src/sqlbuild/compiler/manifest/_helpers/tests.py deleted file mode 100644 index baa2ea9c83..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/tests.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Test node serialization for dbt-compatible manifest.""" - -from __future__ import annotations - -from sqlbuild.compiler.manifest._helpers.shared import _key_to_unique_id -from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_TEST -from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, SqlTestPlanEntry -from sqlbuild.sql_values.types import SqlValueKind - -_TEST_CONFIG: dict[str, object] = { - "enabled": True, - "alias": None, - "schema": None, - "database": None, - "tags": [], - "meta": {}, - "group": None, - "materialized": "test", - "severity": "ERROR", - "store_failures": None, - "store_failures_as": None, - "where": None, - "limit": None, - "fail_calc": "count(*)", - "warn_if": "!= 0", - "error_if": "!= 0", -} - - -def build_audit_test_nodes( - *, - audit_entry: AuditPlanEntry, - project_name: str, -) -> dict[str, dict[str, object]]: - """Build dbt-compatible GenericTest nodes from one audit plan entry.""" - - unique_id: str = f"test.{project_name}.{audit_entry.name}" - compiled_code: str = audit_entry.resolved_sql - - test_metadata: dict[str, object] = {"name": audit_entry.name} - kwargs: dict[str, object] = {} - if audit_entry.attached_target_name is not None: - kwargs["model"] = audit_entry.attached_target_name - if audit_entry.attached_column_name is not None: - kwargs["column_name"] = audit_entry.attached_column_name - test_metadata["kwargs"] = kwargs - - depends_on_nodes: list[str] = [ - _key_to_unique_id(key=dep, project_name=project_name) for dep in audit_entry.scope_deps - ] - - node: dict[str, object] = { - "database": None, - "schema": "", - "name": audit_entry.name, - "resource_type": RESOURCE_TYPE_TEST, - "package_name": project_name, - "path": "", - "original_file_path": "", - "unique_id": unique_id, - "fqn": [project_name, audit_entry.name], - "alias": audit_entry.name, - "checksum": {"name": CHECKSUM_HASH_NAME, "checksum": ""}, - "config": dict(_TEST_CONFIG), - "tags": [], - "description": "", - "columns": {}, - "meta": {"sqlbuild_test_type": "audit"}, - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": None, - "raw_code": compiled_code, - "compiled_code": compiled_code, - "depends_on": {"macros": [], "nodes": depends_on_nodes}, - "test_metadata": test_metadata, - "column_name": audit_entry.attached_column_name, - } - return {unique_id: node} - - -def build_sql_test_nodes( - *, - test_entry: SqlTestPlanEntry, - project_name: str, -) -> dict[str, dict[str, object]]: - """Build dbt-compatible SingularTest nodes from one SQL test plan entry.""" - - unique_id: str = _key_to_unique_id(key=test_entry.key, project_name=project_name) - - compiled_parts: list[str] = [] - step: ChainStep - for step in test_entry.chain: - if not step.resolved_sql: - continue - compiled_parts.append(f"-- step: {step.model_name}") - compiled_parts.append(step.resolved_sql) - compiled_code: str = "\n\n".join(compiled_parts) - - depends_on_nodes: list[str] = [ - _key_to_unique_id(key=dep, project_name=project_name) for dep in test_entry.scope_deps - ] - - node: dict[str, object] = { - "database": None, - "schema": "", - "name": test_entry.name, - "resource_type": RESOURCE_TYPE_TEST, - "package_name": project_name, - "path": "", - "original_file_path": "", - "unique_id": unique_id, - "fqn": [project_name, test_entry.name], - "alias": test_entry.name, - "checksum": { - "name": CHECKSUM_HASH_NAME, - "checksum": test_entry.case_fingerprint or "", - }, - "config": dict(_TEST_CONFIG), - "tags": [], - "description": "", - "columns": {}, - "meta": _sql_test_meta(test_entry), - "group": None, - "docs": {"show": True, "node_color": None}, - "patch_path": None, - "build_path": None, - "unrendered_config": {}, - "created_at": 0.0, - "config_call_dict": {}, - "unrendered_config_call_dict": {}, - "relation_name": None, - "raw_code": compiled_code, - "compiled_code": compiled_code, - "depends_on": {"macros": [], "nodes": depends_on_nodes}, - } - return {unique_id: node} - - -def _sql_test_meta(test_entry: SqlTestPlanEntry) -> dict[str, object]: - meta: dict[str, object] = {"sqlbuild_test_type": "sql_native"} - if test_entry.case_name is None or test_entry.source_path is None: - return meta - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test_entry.parameter_schema - } - meta.update( - { - "sqlbuild_source_path": test_entry.source_path.as_posix(), - "sqlbuild_block_index": test_entry.block_index, - "sqlbuild_parent_name": test_entry.parent_name, - "sqlbuild_case_name": test_entry.case_name, - "sqlbuild_case_index": test_entry.case_index, - "sqlbuild_case_fingerprint": test_entry.case_fingerprint, - "sqlbuild_parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test_entry.parameter_schema - ], - "sqlbuild_parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": ( - str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value - ), - } - for name, value in test_entry.parameter_values - ], - } - ) - return meta diff --git a/src/sqlbuild/compiler/manifest/constants.py b/src/sqlbuild/compiler/manifest/constants.py deleted file mode 100644 index 6c70425595..0000000000 --- a/src/sqlbuild/compiler/manifest/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Manifest output constants aligned with dbt manifest v12 schema.""" - -from sqlbuild.compiler.compile.types import CompiledResourceType - -DBT_MANIFEST_SCHEMA_VERSION: str = "https://schemas.getdbt.com/dbt/manifest/v12.json" - -RESOURCE_TYPE_MODEL: str = "model" -RESOURCE_TYPE_SOURCE: str = "source" -RESOURCE_TYPE_SEED: str = "seed" -RESOURCE_TYPE_TEST: str = "test" -RESOURCE_TYPE_MACRO: str = "macro" - -CHECKSUM_HASH_NAME: str = "sha256" - -RESOURCE_TYPE_PREFIX: dict[str, str] = { - CompiledResourceType.MODEL: "model", - CompiledResourceType.SOURCE: "source", - CompiledResourceType.SEED: "seed", - CompiledResourceType.DBT_REF: "model", - CompiledResourceType.AUDIT: "test", - CompiledResourceType.SQL_TEST: "test", -} diff --git a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py index e43dec7fcb..6a2386322b 100644 --- a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py +++ b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py @@ -2,10 +2,12 @@ from __future__ import annotations +from datetime import UTC, datetime + import sqlbuild._native as _native from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities def build_graph_manifest( @@ -15,13 +17,17 @@ def build_graph_manifest( project_name: str, adapter_type: str, graph: _native.NativeProjectGraph, -) -> dict[str, object]: +) -> _native.NativeManifest: """Build the manifest with parent and child maps read from the native lineage graph.""" - return build_manifest_payload( + return _native.build_manifest( project=project, plan_output=plan_output, - project_name=project_name, - adapter_type=adapter_type, graph=graph, + facts={ + "project_name": project_name, + "adapter_type": adapter_type, + "generated_at": datetime.now(tz=UTC).isoformat(), + "hook_identities": build_hook_identities(project.hook_functions), + }, ) diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py index 913ee1df6c..9c6c173269 100644 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ b/src/sqlbuild/compiler/manifest/main/build.py @@ -6,7 +6,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph.main._native_graph_from_edges import native_graph_from_edges -from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload +from sqlbuild.compiler.manifest.main._build_graph_manifest import build_graph_manifest from sqlbuild.compiler.planner.models import PlanOutput @@ -18,13 +18,13 @@ def build_manifest( adapter_type: str, upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[str, object]: - """Build a full dbt v12-compatible manifest dictionary from caller-built edge maps.""" +) -> _native.NativeManifest: + """Build a full dbt v12-compatible manifest from caller-built edge maps.""" graph: _native.NativeProjectGraph = native_graph_from_edges( upstream=upstream_deps, downstream=downstream_deps ) - return build_manifest_payload( + return build_graph_manifest( project=project, plan_output=plan_output, project_name=project_name, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 383fe8c38c..7aa8240085 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1071,3 +1071,12 @@ class InferredFreshnessCliTestCase: description: str sources_yaml: str expected_sources: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class BuildManifestTestCase: + """One real `sqb build --manifest` run and the manifest nodes it must publish.""" + + description: str + model_count: int + expected_test_type: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py b/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py new file mode 100644 index 0000000000..f83b1fc74c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_build_manifest.py @@ -0,0 +1,64 @@ +"""Integration coverage for the manifest a real build publishes.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import BuildManifestTestCase +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + run_build, + write_test_planning_project, +) + + +@pytest.mark.parametrize( + "test_case", + ( + BuildManifestTestCase( + description="planned models and SQL tests reach the build manifest", + model_count=3, + expected_test_type="sql_native", + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_project_when_building_with_manifest_then_plan_nodes_are_published( + test_case: BuildManifestTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project: Path = tmp_path / "orders" + write_test_planning_project( + project_dir=project, + model_count=test_case.model_count, + rules_config=f'\n[connection]\ndatabase = "{project / "orders.duckdb"}"\n', + ) + + exit_code, output = run_build(project_dir=project, flags=("--manifest",), capsys=capsys) + + assert exit_code == 0, output + contents: bytes = (project / "target" / "manifest.json").read_bytes() + manifest: dict[str, Any] = json.loads(contents) + assert contents.endswith(b"}\n") + nodes: dict[str, Any] = manifest["nodes"] + model_ids: list[str] = [ + f"model.orders.orders_{index:03}" for index in range(test_case.model_count) + ] + assert Counter( + (node["resource_type"], node["meta"].get("sqlbuild_test_type")) for node in nodes.values() + ) == { + ("model", None): test_case.model_count, + ("test", test_case.expected_test_type): test_case.model_count - 1, + } + assert nodes[model_ids[1]]["compiled_code"].startswith("SELECT") + assert model_ids[0] in manifest["parent_map"][model_ids[1]] + assert model_ids[1] in manifest["child_map"][model_ids[0]] + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 48d12bfbcf..f7e076a056 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -117,15 +117,17 @@ def build_manifest_for_pipeline_result( project_name: str, adapter_type: str, ) -> dict[str, object]: - """Build the manifest artifact from a compile pipeline result.""" - - return build_manifest( - project=result.project, - plan_output=result.plan_output, - project_name=project_name, - adapter_type=adapter_type, - upstream_deps=result.plan_output.upstream_deps, - downstream_deps=result.plan_output.downstream_deps, + """Build the manifest artifact from a compile pipeline result and parse its JSON.""" + + return json.loads( + build_manifest( + project=result.project, + plan_output=result.plan_output, + project_name=project_name, + adapter_type=adapter_type, + upstream_deps=result.plan_output.upstream_deps, + downstream_deps=result.plan_output.downstream_deps, + ).contents() ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index b9d26f4f90..8382d5bb4b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os from dataclasses import replace from pathlib import Path @@ -8,6 +7,7 @@ import pytest +import sqlbuild._native as _native from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module from sqlbuild.cli.commands._helpers.compile.target_writer import ( @@ -26,6 +26,7 @@ ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -41,6 +42,7 @@ build_target_writer_plan_output, read_target_files, ) +from tests.unit.src.sqlbuild.compiler.manifest.helpers import build_test_project @pytest.mark.parametrize( @@ -75,7 +77,14 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( expected_test_sql: str = ( build_sql_test_comparison_sql(test_entry=plan_output.test_entries[0]) + "\n" ) - manifest: dict[str, object] = {"metadata": {"project_name": "demo"}} + manifest: _native.NativeManifest = build_manifest( + project=build_test_project(), + project_name="demo", + adapter_type="duckdb", + upstream_deps={}, + downstream_deps={}, + ) + manifest_contents: bytes = manifest.contents() expected_files: dict[str, str] = dict(test_case.expected_files) expected_files["compiled/tests/_chain_/orders__stg_orders/orders_chain.sql"] = expected_test_sql @@ -97,7 +106,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( ) assert read_target_files(tmp_path / "target", expected_files) == expected_files - assert json.loads(manifest_path.read_text()) == manifest + assert manifest_path.read_bytes() == manifest_contents assert manifest_path.stat().st_mtime_ns == unchanged_mtime_ns assert not (tmp_path / "target" / "run").exists() diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 9238a53fa7..6c528656b8 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,4 +1,5 @@ import itertools +import json from collections.abc import Callable from pathlib import Path from types import MappingProxyType @@ -59,6 +60,7 @@ ColumnTransformKind, InferredNullability, ) +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( build_native_declaration_contexts, ) @@ -482,3 +484,18 @@ def reanalysed_columns( profile=inference_profile or ExpressionInferenceProfile(), column_types_by_table=column_types_by_table, )[0] + + +def manifest_model_node(*, project: CompiledProjectView, name: str) -> dict[str, object]: + """Build the project's manifest and return one model node from its published JSON.""" + + manifest: dict[str, dict[str, dict[str, object]]] = json.loads( + build_manifest( + project=project, + project_name="demo", + adapter_type="duckdb", + upstream_deps={}, + downstream_deps={}, + ).contents() + ) + return manifest["nodes"][f"model.demo.{name}"] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index d78c7151d1..d693e298df 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -21,7 +21,6 @@ ) from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( build_model_version_identity_metadata_json, ) @@ -38,6 +37,7 @@ from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, compile_first_model, + manifest_model_node, ) _PROJECT_FILE: str = """ @@ -304,11 +304,7 @@ def test_given_inherited_model_schemas_when_compiling_then_attaches_resolved_con ], } assert evaluate_model_contracts(project=project, dialect="duckdb").diagnostics == () - manifest_node: dict[str, object] = build_model_node( - model=superset, - plan_entry=None, - project_name="demo", - ) + manifest_node: dict[str, object] = manifest_model_node(project=project, name="order_superset") assert manifest_node["description"] == "Canonical order schema" manifest_columns: object = manifest_node["columns"] assert isinstance(manifest_columns, dict) @@ -489,11 +485,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac "name_pattern": None, } ] - manifest_node: dict[str, object] = build_model_node( - model=model, - plan_entry=None, - project_name="demo", - ) + manifest_node: dict[str, object] = manifest_model_node(project=project, name=model.name) manifest_meta: dict[str, object] = cast(dict[str, object], manifest_node["meta"]) sqlbuild_meta: dict[str, object] = cast(dict[str, object], manifest_meta["sqlbuild"]) assert sqlbuild_meta["dynamic_columns"] == contract_identity["dynamic_columns"] diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index a459ffea88..1974575b1c 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -138,3 +138,10 @@ class FqnTestCase: project_name: str relative_path: str expected_fqn: list[str] + + +@dataclass(frozen=True) +class ManifestEncodingErrorTestCase: + description: str + meta: dict[object, object] + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index b689ed3aae..9dfcd8b46e 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any, cast @@ -69,10 +70,9 @@ def source_key(name: str) -> CompiledObjectKey: def run_manifest(**kwargs: Any) -> dict[str, Any]: - """Build a manifest and return it with Any typing for test convenience.""" + """Build a manifest and return its parsed `manifest.json` contents.""" - result: dict[str, object] = build_manifest(**kwargs) - return result # type: ignore[return-value] + return json.loads(build_manifest(**kwargs).contents()) def manifest_nodes(result: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 485555ada0..7a549c4bce 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -1,11 +1,13 @@ from __future__ import annotations +import re from dataclasses import replace from pathlib import Path -from typing import Any +from typing import Any, cast import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( CompiledObjectKey, LoadedMacro, @@ -18,8 +20,7 @@ ) from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry -from sqlbuild.compiler.manifest._helpers.shared import build_fqn -from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION +from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, @@ -33,6 +34,7 @@ from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, + ManifestEncodingErrorTestCase, ManifestMacroNodeTestCase, ManifestModelNodeTestCase, ManifestParentMapTestCase, @@ -61,6 +63,7 @@ _PROJECT: str = "demo" _ADAPTER: str = "duckdb" +_DBT_MANIFEST_SCHEMA_VERSION: str = "https://schemas.getdbt.com/dbt/manifest/v12.json" _MODEL_ORDERS: CompiledModelView = build_test_model( name="orders", @@ -158,7 +161,7 @@ expected_macro_count=0, expected_metadata_project_name=_PROJECT, expected_metadata_adapter_type=_ADAPTER, - expected_metadata_schema_version=DBT_MANIFEST_SCHEMA_VERSION, + expected_metadata_schema_version=_DBT_MANIFEST_SCHEMA_VERSION, ), ManifestTopLevelTestCase( description="produces correct counts for multi-resource project", @@ -179,7 +182,7 @@ expected_macro_count=0, expected_metadata_project_name=_PROJECT, expected_metadata_adapter_type=_ADAPTER, - expected_metadata_schema_version=DBT_MANIFEST_SCHEMA_VERSION, + expected_metadata_schema_version=_DBT_MANIFEST_SCHEMA_VERSION, ), ], ids=lambda case: case.description, @@ -608,15 +611,23 @@ def test_given_seed_when_building_manifest_then_produces_correct_node( ], ids=lambda case: case.description, ) -def test_given_relative_path_when_building_fqn_then_returns_expected_parts( +def test_given_relative_path_when_building_manifest_then_fqn_has_expected_parts( test_case: FqnTestCase, ) -> None: - result: list[str] = build_fqn( + model: CompiledModelView = build_test_model( + name="orders", relative_path=test_case.relative_path + ) + + result: dict[str, Any] = run_manifest( + project=build_test_project(models=(model,)), project_name=test_case.project_name, - relative_path=Path(test_case.relative_path), + adapter_type=_ADAPTER, + upstream_deps={}, + downstream_deps={}, ) - assert result == test_case.expected_fqn + node: dict[str, Any] = manifest_nodes(result)[f"model.{test_case.project_name}.orders"] + assert node["fqn"] == test_case.expected_fqn @pytest.mark.parametrize( @@ -1038,3 +1049,37 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc "sqlbuild_resource_type": "sql_hook", "sqlbuild_hook_name": "record_access", } + + +@pytest.mark.parametrize( + "test_case", + [ + ManifestEncodingErrorTestCase( + description="unencodable meta value raises when the manifest is written", + meta={"owners": {"orders"}}, + expected_message="Object of type set is not JSON serializable", + ), + ManifestEncodingErrorTestCase( + description="unencodable meta key raises when the manifest is written", + meta={("orders", 1): "owner"}, + expected_message="keys must be str, int, float, bool or None, not tuple", + ), + ], + ids=lambda case: case.description, +) +def test_given_unencodable_meta_when_writing_manifest_then_raises_json_type_error( + test_case: ManifestEncodingErrorTestCase, +) -> None: + model: CompiledModelView = build_test_model( + name="orders", meta=cast(dict[str, object], test_case.meta) + ) + manifest: _native.NativeManifest = build_manifest( + project=build_test_project(models=(model,)), + project_name=_PROJECT, + adapter_type=_ADAPTER, + upstream_deps={}, + downstream_deps={}, + ) + + with pytest.raises(TypeError, match=re.escape(test_case.expected_message)): + manifest.contents() From dc13b8564a28604882fe38211ab8be8f59c3ff9e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:07:24 +0100 Subject: [PATCH 10/26] refactor: compile projects through the native entry directly --- python-boundary.toml | 10 ++++++- .../_helpers/janitor_runtime/compilation.py | 4 +-- .../compiler/pipeline/main/project.py | 30 ------------------- src/sqlbuild/lint/_helpers/fixes.py | 12 +++++--- .../_helpers/engine/rule_harness.py | 5 ++-- .../compiler/compiled_project/helpers.py | 9 +++--- .../test_native_compiled_project.py | 5 ++-- .../src/sqlbuild/compiler/pipeline/helpers.py | 12 +++++--- .../test_macro_declaration_context.py | 5 ++-- .../sqlbuild/compiler/pipeline/test_main.py | 9 +++--- .../pipeline/test_set_operation_types.py | 7 ++--- .../test_sql_test_input_extraction.py | 5 ++-- .../planner/_helpers/test_fixture_planning.py | 7 ++--- .../executor/build/concurrent/helpers.py | 3 +- .../concurrent/test_ordering_invariant.py | 3 +- .../sqlbuild/executor/build/custom/helpers.py | 3 +- .../src/sqlbuild/executor/build/helpers.py | 3 +- .../build/snowflake_metadata/helpers.py | 3 +- .../executor/build/test_snapshot_execution.py | 3 +- .../src/sqlbuild/lint/main/helpers.py | 5 ++-- .../rule_engine/host_payload/helpers.py | 5 ++-- .../warehouse/test_semantic_sources.py | 7 ++--- 22 files changed, 62 insertions(+), 93 deletions(-) delete mode 100644 src/sqlbuild/compiler/pipeline/main/project.py diff --git a/python-boundary.toml b/python-boundary.toml index ed6a047b38..3f5de15324 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 134615 +remaining_lines = 134588 [[stays]] path = "executor" @@ -231,3 +231,11 @@ note = "Best-effort cost refresh keeps its existing catch-all; discovery is now [[handled_native_errors]] location = "compiler/scopes/main/load_or_build_scope_index.py::load_or_build_scope_index" note = "Project input errors from discovery and compile-input stages select the tolerant scope index as before; a native compiler failure is a RuntimeError and is not caught." + +[[handled_native_errors]] +location = "lint/_helpers/fixes.py::_FixSession._relation_keys" +note = "A baseline that does not compile yields no relation keys, as before the compile entry became native; no Python implementation runs instead." + +[[handled_native_errors]] +location = "lint/_helpers/fixes.py::_FixSession._verdict" +note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py index 1868504ac8..b23675a53f 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py @@ -5,6 +5,7 @@ import sys import time +from sqlbuild._native import build_compiled_project from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -15,7 +16,6 @@ ) from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,7 +35,7 @@ def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileC ) with OperationLifecycle(operation_kind="project", operation_name="project_compile"): status.on_progress("Compiling project...") - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=invocation.discovered_inputs, adapter=adapter_context.adapter, selected_target=( diff --git a/src/sqlbuild/compiler/pipeline/main/project.py b/src/sqlbuild/compiler/pipeline/main/project.py deleted file mode 100644 index 2d4cb068fc..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/project.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Compile a project without building an execution plan.""" - -from __future__ import annotations - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def compile_project( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None = None, - resolved_connection: dict[str, object] | None = None, - no_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> CompiledProjectView: - """Compile discovered inputs into a target-defaulted project view.""" - - return build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - ) diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 477b9bce3d..2921cfd340 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -8,11 +8,11 @@ from dataclasses import dataclass, field, replace from pathlib import Path +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.lint._helpers.native_format import with_newline_style from sqlbuild.lint.constants import ( LINT_ENGINE_NATIVE, @@ -217,7 +217,9 @@ def _relation_keys(self) -> RelationKeys: if self.baseline is None: try: - self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + self.baseline = build_compiled_project( + discovered_inputs=self.inputs, adapter=self.adapter + ) except CompileInputError: return {} return compiled_relation_keys(self.baseline) @@ -228,12 +230,14 @@ def _verdict(self, *, files: dict[Path, str]) -> FixVerdict | None: verdict: FixVerdict | None try: if self.baseline is None: - self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + self.baseline = build_compiled_project( + discovered_inputs=self.inputs, adapter=self.adapter + ) if self.baseline_facts is None: self.baseline_facts = compile_facts( project=self.baseline, project_dir=self.project_dir ) - compiled: CompiledProjectView = compile_project( + compiled: CompiledProjectView = build_compiled_project( discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), adapter=self.adapter, ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index f07eda5c10..7096622f1d 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -6,11 +6,10 @@ from pathlib import Path from tempfile import TemporaryDirectory -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine.config import load_rules_config from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, @@ -110,4 +109,4 @@ def _toml_value(*, value: object) -> str: def _compile_rule_project(*, root: Path) -> CompiledProjectView: discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=root) adapter: DuckDbAdapter = DuckDbAdapter() - return compile_project(discovered_inputs=discovered, adapter=adapter) + return build_compiled_project(discovered_inputs=discovered, adapter=adapter) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 34c578cd0a..be816afaab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -13,7 +13,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import build_compile_inputs, build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds @@ -37,7 +37,6 @@ ) from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering @@ -431,7 +430,7 @@ def compile_source_project( write_files( project_dir=project_dir, files={**SOURCE_PROJECT_FILES, "sources/raw.yml": source_yml} ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, @@ -477,7 +476,7 @@ def compile_function_project( """Compile a project of `files`; return the project and its compile inputs.""" write_files(project_dir=project_dir, files=files) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, @@ -659,7 +658,7 @@ def native_analysed_project(*, project_dir: Path, models: dict[str, str]) -> Com }, }, ) - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), resolved_connection=_SOURCE_CONNECTION, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 411527b014..e45dc2ab34 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -8,13 +8,12 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, @@ -72,7 +71,7 @@ def test_given_repeated_output_schemas_when_serializing_then_columns_share_immut }, }, ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index f7e076a056..afbeadcaeb 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -8,7 +8,12 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_compile_inputs, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + build_compile_inputs, + build_compiled_project, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter @@ -20,7 +25,6 @@ from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.sql_values.types import CollectionRendering @@ -172,7 +176,7 @@ def compile_reshaped_star_model( f"MODEL (description 'Test model reshaped_orders.', materialized table);\n{query_sql}", encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ) @@ -237,7 +241,7 @@ def postgres_semantic_shapes( path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=PostgresAdapter(), resolved_connection=_OFFLINE_POSTGRES_CONNECTION, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 72833e0de9..3c250b11c2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -5,14 +5,13 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( GroupedDeclarationCompileTestCase, @@ -72,7 +71,7 @@ def test_given_active_adapter_when_macro_renders_constant_then_uses_adapter_sql( adapter: BaseAdapter = test_case.adapter_factory() discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 020b6eebeb..315866eb17 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import build_compiled_project, build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target @@ -15,7 +15,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError @@ -98,7 +97,7 @@ def test_given_typed_cte_chain_when_compiling_contract_then_type_remains_proven( write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=DuckDbAdapter(), ) @@ -566,7 +565,7 @@ def test_given_snowflake_local_override_without_target_namespace_when_compiling_ discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) with pytest.raises(ValueError, match=test_case.expected_error_fragment): - compile_project( + build_compiled_project( discovered_inputs=discovered_inputs, adapter=SnowflakeAdapter(), no_sql_validation=True, @@ -609,7 +608,7 @@ def test_given_snowflake_local_target_namespace_when_compiling_then_targets_reso write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=SnowflakeAdapter(), no_sql_validation=True, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index 40b7a2d4a5..a932dd2486 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -9,13 +9,12 @@ import pytest from duckdb import sqltypes -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SetOperationLineageIntegrationTestCase, SetOperationTypeIntegrationTestCase, @@ -60,7 +59,7 @@ def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) @@ -112,7 +111,7 @@ def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( encoding="utf-8", ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index ad62fbbfc4..d55fbccb50 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -5,14 +5,13 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ExpectedCountTestCase @@ -53,7 +52,7 @@ def test_given_expanded_mixed_mode_tests_when_compiling_project_then_complete_in }, ) - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 3f72f75288..eafda6ddd7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -5,11 +5,10 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import SqlTestPlanEntry, SqlTestSelection from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -66,7 +65,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) @@ -135,7 +134,7 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 4f67a6e85f..8c27d98cef 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,11 +7,10 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 3410f985c4..51185b7896 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,11 +9,10 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 4f26ab1359..abc6853e1d 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -425,10 +425,9 @@ def run_scheduler_build( ) -> tuple[Any, Any]: """Run a full build through the scheduler with custom materializations.""" - from sqlbuild._native import discover_project_inputs + from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index bd37c6a50a..4dd945f428 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,12 +10,11 @@ import duckdb -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 310eb93c04..059b3a7488 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,10 +13,9 @@ from pathlib import Path from typing import Any -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index 8d2b39277b..0a032be40f 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,10 +8,9 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, run_compile_pipeline from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild._native import run_compile_pipeline from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index 95b17b9c0c..7bbe835f06 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -5,12 +5,11 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.pipeline.main.project import compile_project _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -24,7 +23,7 @@ def compile_models( (project_dir / "models").mkdir(exist_ok=True) for name, sql in models: (project_dir / "models" / name).write_text(sql, encoding="utf-8") - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index e66cf8ef28..f31880b78a 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -5,11 +5,10 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine import custom_rules, fact_replay from sqlbuild.rule_engine._helpers.engine.fact_replay import ( fact_outcome_digest, @@ -63,7 +62,7 @@ def compile_files(*, root: Path, files: dict[str, str]) -> CompiledProjectView: path: Path = root / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(contents, encoding="utf-8") - return compile_project( + return build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=root), adapter=DuckDbAdapter(), external_sql_reference_resolver=OrdersReferenceResolver(), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 5b78667aeb..5ccf38ab2f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -10,12 +10,11 @@ import duckdb import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( _list_source_candidates, get_semantic_source_columns, @@ -59,7 +58,7 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) listings: Mock = Mock(wraps=adapter.list_relations) @@ -116,7 +115,7 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProjectView = compile_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) with duckdb.connect() as connection: From 467a060c03fa974dd025502e44fd26ccdd3addf7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:55:52 +0100 Subject: [PATCH 11/26] refactor: build discovery records natively Native discovery builds the Python discovery model objects directly, and the compile driver calls the native discoverers. The Python payload conversion layer (discovery/_helpers/native), the pure file-walk wrappers in filesystem/core.py, the YAML path and scenario helpers, and the payload-returning bindings are removed. Discoverers that import or run user code stay in Python. --- .../src/yaml_files/_helpers/mod.rs | 1 + .../src/yaml_files/_helpers/suffixes.rs | 16 + .../src/yaml_files/constants.rs | 12 + .../src/yaml_files/main/mod.rs | 2 + .../src/yaml_files/main/schema_yaml_paths.rs | 38 + .../src/yaml_files/main/source_yaml_paths.rs | 19 + .../sqlbuild-discovery/src/yaml_files/mod.rs | 1 + .../src/yaml_files/models.rs | 10 +- .../src/yaml_files/tests.rs | 2 + .../src/yaml_files/tests/helpers.rs | 18 + .../src/yaml_files/tests/test_types.rs | 8 +- .../src/yaml_files/tests/test_yaml_paths.rs | 75 ++ .../_helpers/compile_driver/discovery.rs | 135 ++-- .../_helpers/discovery/declaration_files.rs | 386 +--------- .../_helpers/discovery/declaration_records.rs | 700 ++++++++++++++++++ .../src/bindings/_helpers/discovery/faults.rs | 62 ++ .../_helpers/discovery/header_declarations.rs | 6 +- .../_helpers/discovery/layout_groups.rs | 84 +++ .../src/bindings/_helpers/discovery/mod.rs | 9 + .../_helpers/discovery/model_records.rs | 144 ++++ .../_helpers/discovery/project_files.rs | 568 +------------- .../_helpers/discovery/project_validation.rs | 87 ++- .../src/bindings/_helpers/discovery/raised.rs | 163 ++++ .../_helpers/discovery/registration.rs | 12 +- .../bindings/_helpers/discovery/snapshot.rs | 154 ++++ .../_helpers/discovery/sorted_keys.rs | 19 + .../_helpers/discovery/sql_test_blocks.rs | 23 +- .../_helpers/discovery/sql_test_records.rs | 317 ++++++++ .../_helpers/discovery/yaml_entries.rs | 6 +- .../_helpers/discovery/yaml_records.rs | 382 ++++++++++ crates/sqlbuild-python/src/bindings/models.rs | 42 -- python-boundary.toml | 16 + src/sqlbuild/_native.pyi | 124 ++-- .../_helpers/filesystem/aggregation.py | 24 +- .../discovery/_helpers/filesystem/core.py | 111 --- .../_helpers/filesystem/named_declarations.py | 4 +- .../_helpers/filesystem/scoped_paths.py | 50 -- .../_helpers/native/declaration_files.py | 495 ------------- .../discovery/_helpers/native/declarations.py | 57 -- .../discovery/_helpers/native/model_files.py | 81 -- .../discovery/_helpers/native/payloads.py | 247 ------ .../_helpers/native/sql_test_files.py | 190 ----- .../discovery/_helpers/native/yaml_files.py | 169 ----- .../discovery/_helpers/sql/scenarios.py | 50 -- .../_helpers/validation/discovery.py | 25 +- .../discovery/_helpers/yml/file_paths.py | 48 -- .../compiler/discovery/_helpers/yml/schema.py | 17 - .../discovery/_helpers/yml/sources.py | 15 - .../selected_contract_input_discoverer.py | 20 +- src/sqlbuild/compiler/discovery/constants.py | 9 - .../main/_model_description_inputs.py | 5 +- .../discovery/main/_parse_sql_test_file.py | 16 - src/sqlbuild/compiler/discovery/types.py | 16 +- .../_helpers/sql_tests/fixture_formatting.py | 4 +- .../src/sqlbuild/cli/commands/main/helpers.py | 29 +- .../cli/commands/main/test_discovery_work.py | 14 +- .../compiler/discovery/_test_types.py | 4 +- .../sqlbuild/compiler/discovery/helpers.py | 55 +- .../discovery/test_native_model_discovery.py | 8 +- .../discovery/test_native_read_errors.py | 12 +- .../discovery/_helpers/_test_helpers.py | 14 +- .../compiler/discovery/_helpers/helpers.py | 2 +- .../_helpers/test_filesystem_file_faults.py | 22 +- .../_helpers/test_filesystem_macros.py | 4 +- .../test_filesystem_scoped_declarations.py | 28 +- .../discovery/_helpers/test_sql_hooks.py | 4 +- .../discovery/_helpers/test_sql_models.py | 12 +- .../discovery/_helpers/test_sql_tests.py | 6 +- .../sqlbuild/compiler/discovery/helpers.py | 82 +- 69 files changed, 2687 insertions(+), 2903 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/constants.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/declarations.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/model_files.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/payloads.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/schema.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/sources.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs index e53d6ea07e..d28fbf2ed3 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod entry_names; pub(crate) mod failures; +pub(crate) mod suffixes; diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs new file mode 100644 index 0000000000..736e5419b4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/suffixes.rs @@ -0,0 +1,16 @@ +//! `Path.suffix` and `Path.glob` name matching as Python applies them to listed names. + +use sqlbuild_core::text::main::glob_name_match::glob_name_match; + +/// Python's `PurePath.suffix` of a file name. +pub(crate) fn python_suffix(name: &str) -> &str { + match name.rfind('.') { + Some(index) if index > 0 && index + 1 < name.len() => &name[index..], + _ => "", + } +} + +/// Whether `name` matches a single-name glob with the platform's default case sensitivity. +pub(crate) fn name_matches(name: &str, pattern: &str) -> bool { + glob_name_match(name, pattern, cfg!(windows)) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/constants.rs b/crates/sqlbuild-discovery/src/yaml_files/constants.rs new file mode 100644 index 0000000000..726dfc8b1c --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/constants.rs @@ -0,0 +1,12 @@ +//! Where authored source and schema YAML files live. + +/// The directory whose direct YAML children declare sources. +pub(crate) const SOURCES_DIRECTORY: &str = "sources"; +pub(crate) const MODELS_DIRECTORY: &str = "models"; +pub(crate) const SEEDS_DIRECTORY: &str = "seeds"; +/// Python's `YAML_FILE_SUFFIXES`. +pub(crate) const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; +/// Python's `SCHEMA_FILE_NAME`. +pub(crate) const SCHEMA_FILE_NAME: &str = "schema.yml"; +pub(crate) const SEED_DECLARATION_PATTERN: &str = "*.yml"; +pub(crate) const UNSUPPORTED_SEED_DECLARATION_PATTERN: &str = "*.yaml"; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs index b6ed6a999a..a41c241ae9 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -1,3 +1,5 @@ pub mod load_yaml_files; pub mod load_yaml_text; +pub mod schema_yaml_paths; +pub mod source_yaml_paths; pub mod yaml_entry_lines; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs new file mode 100644 index 0000000000..21f8fe6ef9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/schema_yaml_paths.rs @@ -0,0 +1,38 @@ +//! The unscoped model `schema.yml` files and seed declaration files, in Python's order. + +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use crate::yaml_files::_helpers::suffixes::name_matches; +use crate::yaml_files::constants::{ + MODELS_DIRECTORY, SCHEMA_FILE_NAME, SEED_DECLARATION_PATTERN, SEEDS_DIRECTORY, + UNSUPPORTED_SEED_DECLARATION_PATTERN, +}; +use crate::yaml_files::models::YamlPathFailure; + +fn matching(tree: &ProjectTree, root: &str, pattern: &str) -> Result, YamlPathFailure> { + rglob(tree, root, |entry| name_matches(&entry.name, pattern)).map_err(YamlPathFailure::Stage) +} + +/// Python's `schema_file_paths`; a `.yaml` seed declaration fails as a schema error. +pub fn schema_yaml_paths(tree: &ProjectTree) -> Result, YamlPathFailure> { + let mut paths: Vec = Vec::new(); + if tree.is_dir(MODELS_DIRECTORY) { + paths.extend( + matching(tree, MODELS_DIRECTORY, SCHEMA_FILE_NAME)? + .into_iter() + .filter(|path| !is_in_scoped_declaration_tree(path)), + ); + } + if tree.is_dir(SEEDS_DIRECTORY) { + if let Some(path) = + matching(tree, SEEDS_DIRECTORY, UNSUPPORTED_SEED_DECLARATION_PATTERN)?.first() + { + return Err(YamlPathFailure::YamlSeedDeclaration(path.clone())); + } + paths.extend(matching(tree, SEEDS_DIRECTORY, SEED_DECLARATION_PATTERN)?); + } + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + paths.retain(|path| seen.insert(path.clone())); + Ok(paths) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs new file mode 100644 index 0000000000..6c33d2c98a --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/source_yaml_paths.rs @@ -0,0 +1,19 @@ +//! The YAML files directly below `sources/`, as Python's source discovery lists them. + +use crate::models::StageFailure; +use crate::tree::main::children::children; +use crate::tree::models::ProjectTree; +use crate::yaml_files::_helpers::suffixes::python_suffix; +use crate::yaml_files::constants::{SOURCES_DIRECTORY, YAML_FILE_SUFFIXES}; + +/// `sorted(path for path in sources.iterdir() if path.suffix in YAML_FILE_SUFFIXES)`. +pub fn source_yaml_paths(tree: &ProjectTree) -> Result, StageFailure> { + if !tree.is_dir(SOURCES_DIRECTORY) { + return Ok(Vec::new()); + } + Ok(children(tree, SOURCES_DIRECTORY)? + .into_iter() + .filter(|(_, entry)| YAML_FILE_SUFFIXES.contains(&python_suffix(&entry.name))) + .map(|(path, _)| path) + .collect()) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/mod.rs index e6e5a52eb4..1e64f3d116 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-discovery/src/yaml_files/models.rs b/crates/sqlbuild-discovery/src/yaml_files/models.rs index 73d52e5c8a..f119786801 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/models.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/models.rs @@ -1,6 +1,6 @@ //! Native outcomes of reading and loading authored YAML files. -use crate::models::{DiscoveryFailure, ReadFailure}; +use crate::models::{DiscoveryFailure, ReadFailure, StageFailure}; use sqlbuild_config::models::ConfigValue; /// One YAML file read and loaded as YAML 1.2, or why it could not be. @@ -15,3 +15,11 @@ pub enum YamlFileOutcome { /// The file could not be read or decoded as Python's `read_text` would. Unreadable(ReadFailure), } + +/// Why the schema YAML files could not be listed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum YamlPathFailure { + Stage(StageFailure), + /// The first seed declaration file, by relative path, that uses `.yaml`. + YamlSeedDeclaration(String), +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests.rs b/crates/sqlbuild-discovery/src/yaml_files/tests.rs index 9026fd5196..cbe8b7097a 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests.rs @@ -6,3 +6,5 @@ mod load_yaml_files; mod test_types; #[path = "tests/test_yaml_entry_lines.rs"] mod yaml_entry_lines; +#[path = "tests/test_yaml_paths.rs"] +mod yaml_paths; diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs index f7330423e5..00de3a41db 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs @@ -37,3 +37,21 @@ pub(super) fn owned_failure( ) -> Option<(String, Option)> { expected.map(|(message, help)| (message.to_owned(), help.map(str::to_owned))) } + +/// The paths a listing of a project holding `files` (with empty contents) returns. +pub(super) fn listed_paths( + files: &[&str], + list: impl Fn(&ProjectTree) -> Result, crate::yaml_files::models::YamlPathFailure>, +) -> Result, crate::yaml_files::models::YamlPathFailure> { + let project = tempfile::tempdir().expect("temporary project"); + for file in files { + let path = project.path().join(file); + std::fs::create_dir_all(path.parent().expect("file parent")).expect("parent directory"); + std::fs::write(path, b"").expect("project file"); + } + list(&ProjectTree::new(project.path())) +} + +pub(super) fn owned_paths(paths: &[&str]) -> Vec { + paths.iter().map(|path| (*path).to_owned()).collect() +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs index 1d06fe9305..be46dd86c1 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::yaml_files::models::YamlFileOutcome; +use crate::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; pub(super) struct YamlFileTestCase { pub(super) description: &'static str, @@ -19,3 +19,9 @@ pub(super) struct EntryLinesTestCase { /// The line of each named entry the case checks; `None` where no entry has the name. pub(super) expected_lines: &'static [(&'static str, Option)], } + +pub(super) struct YamlPathsTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) expected_paths: Result, YamlPathFailure>, +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs new file mode 100644 index 0000000000..075258031b --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_paths.rs @@ -0,0 +1,75 @@ +use crate::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use crate::yaml_files::main::source_yaml_paths::source_yaml_paths; +use crate::yaml_files::models::YamlPathFailure; +use crate::yaml_files::tests::helpers::{listed_paths, owned_paths}; +use crate::yaml_files::tests::test_types::YamlPathsTestCase; + +#[test] +fn given_sources_directory_when_listing_source_yaml_then_direct_yaml_children_are_sorted() { + let test_cases = [ + YamlPathsTestCase { + description: "yml and yaml files directly below sources/ in path order", + files: &[ + "sources/b.yaml", + "sources/a.yml", + "sources/notes.txt", + "sources/.yml", + "sources/nested/c.yml", + ], + expected_paths: Ok(owned_paths(&["sources/a.yml", "sources/b.yaml"])), + }, + YamlPathsTestCase { + description: "no sources directory", + files: &["models/orders.sql"], + expected_paths: Ok(Vec::new()), + }, + ]; + for test_case in test_cases { + let listed = listed_paths(test_case.files, |tree| { + source_yaml_paths(tree).map_err(YamlPathFailure::Stage) + }); + assert_eq!( + listed, test_case.expected_paths, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_models_and_seeds_when_listing_schema_yaml_then_unscoped_schemas_and_seed_files_are_kept() { + let test_cases = [ + YamlPathsTestCase { + description: "unscoped schema files then seed declarations, each in path order", + files: &[ + "models/marts/schema.yml", + "models/schema.yml", + "models/marts/_sqlbuild/schemas/schema.yml", + "models/other.yml", + "seeds/products.yml", + "seeds/customers/customers.yml", + ], + expected_paths: Ok(owned_paths(&[ + "models/marts/schema.yml", + "models/schema.yml", + "seeds/customers/customers.yml", + "seeds/products.yml", + ])), + }, + YamlPathsTestCase { + description: "a seed declaration using .yaml fails naming the first one", + files: &["seeds/b.yaml", "seeds/a.yaml", "seeds/c.yml"], + expected_paths: Err(YamlPathFailure::YamlSeedDeclaration( + "seeds/a.yaml".to_owned(), + )), + }, + ]; + for test_case in test_cases { + let listed = listed_paths(test_case.files, schema_yaml_paths); + assert_eq!( + listed, test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 73dde619b3..94aa6db407 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -1,12 +1,26 @@ //! Project discovery: configuration, declaration files and imported project Python, validated. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::discovery::declaration_records::{ + discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, + discover_macro_files, discover_model_schema_files, discover_seed_files, + discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, +}; +use crate::bindings::_helpers::discovery::model_records::{ + ModelFileSelection, discovered_model_files, +}; +use crate::bindings::_helpers::discovery::sql_test_records::{ + discover_scenario_files, discover_test_files, +}; +use crate::bindings::_helpers::discovery::yaml_records::{ + discover_schema_files, discover_source_files, +}; const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; @@ -22,11 +36,6 @@ const EVENT_EXPORTERS_MODULE: &str = const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; const LOADERS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.integrations.loaders"; -const DECLARATION_FILES_MODULE: &str = - "sqlbuild.compiler.discovery._helpers.native.declaration_files"; -const MODEL_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.model_files"; -const SQL_TEST_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.sql_test_files"; -const YAML_FILES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.native.yaml_files"; const SNAPSHOT_MODULE: &str = "sqlbuild.compiler.discovery.classes.directory_snapshot"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; @@ -48,62 +57,58 @@ const DECLARATION_COLLECTIONS: [&str; 14] = [ "macro_files", ]; -/// Discoverers of the declaration files that take only `project_dir`, with their result fields. -const FILE_DISCOVERERS: [(&str, &str, &str); 8] = [ - ("enum_files", FILESYSTEM_MODULE, "discover_enum_files"), +/// A native discoverer that takes only `project_dir`. +type FileDiscoverer = for<'py> fn(&Bound<'py, PyAny>) -> PyResult>; + +/// Early discoverers by result field; `None` is the Python function-file discoverer (uses `ast`). +const FILE_DISCOVERERS: [(&str, Option); 8] = [ + ( + "enum_files", + Some(|dir| Ok(discover_enum_files(dir, None, false)?.into_any())), + ), ( "constant_files", - FILESYSTEM_MODULE, - "discover_constant_files", + Some(|dir| Ok(discover_constant_files(dir, None, false)?.into_any())), ), ( "model_schema_files", - FILESYSTEM_MODULE, - "discover_model_schema_files", + Some(|dir| Ok(discover_model_schema_files(dir, None)?.into_any())), ), ( "sql_function_files", - FILESYSTEM_MODULE, - "discover_sql_function_files", + Some(|dir| Ok(discover_sql_function_files(dir, None)?.into_any())), ), ( "sql_hook_files", - FILESYSTEM_MODULE, - "discover_sql_hook_files", + Some(|dir| Ok(discover_sql_hook_files(dir, None)?.into_any())), ), + ("python_function_files", None), ( - "python_function_files", - FILESYSTEM_MODULE, - "discover_python_function_files", + "schema_files", + Some(|dir| Ok(discover_schema_files(dir)?.into_any())), ), ( - "schema_files", - YAML_FILES_MODULE, - "discover_native_schema_files", + "seed_files", + Some(|dir| Ok(discover_seed_files(dir)?.into_any())), ), - ("seed_files", FILESYSTEM_MODULE, "discover_seed_files"), ]; /// Discoverers that run after the schema files, in their original order. -const LATE_FILE_DISCOVERERS: [(&str, &str, &str); 6] = [ - ( - "test_files", - SQL_TEST_FILES_MODULE, - "discover_native_test_files", - ), - ( - "scenario_files", - SQL_TEST_FILES_MODULE, - "discover_native_scenario_files", - ), - ("audit_files", FILESYSTEM_MODULE, "discover_audit_files"), - ("macro_files", FILESYSTEM_MODULE, "discover_macro_files"), - ("adapter_file", FILESYSTEM_MODULE, "discover_adapter_file"), - ( - "native_session", - DECLARATION_FILES_MODULE, - "retained_discovery_session", - ), +const LATE_FILE_DISCOVERERS: [(&str, FileDiscoverer); 6] = [ + ("test_files", |dir| { + Ok(discover_test_files(dir, None, None)?.into_any()) + }), + ("scenario_files", |dir| { + Ok(discover_scenario_files(dir, None)?.into_any()) + }), + ("audit_files", |dir| { + Ok(discover_audit_files(dir, None)?.into_any()) + }), + ("macro_files", |dir| { + Ok(discover_macro_files(dir, false)?.into_any()) + }), + ("adapter_file", discover_adapter_file), + ("native_session", retained_discovery_session), ]; /// Discoverers of imported project Python that take `project_dir` and `providers`. @@ -375,34 +380,30 @@ fn declaration_files<'py>( )?; within(&snapshot, |_| { let files: Bound<'py, PyDict> = PyDict::new(py); - files.set_item( - "source_files", - call_with( - &imported(py, YAML_FILES_MODULE, "discover_native_source_files")?, - &directory, - )?, - )?; + files.set_item("source_files", discover_source_files(project_dir, None)?)?; files.set_item( "model_files", - call_with( - &imported(py, MODEL_FILES_MODULE, "discover_native_model_files")?, - &[ - ("project_dir", project_dir.clone()), - ( - "extract_implicit_alias_columns", - PyBool::new(py, sql_analysis_enabled).to_owned().into_any(), - ), - ( - "extract_output_column_locations", - PyBool::new(py, extract_output_column_locations) - .to_owned() - .into_any(), - ), - ], + discovered_model_files( + project_dir, + (sql_analysis_enabled, extract_output_column_locations), + ModelFileSelection { + selected_model_names: None, + on_fault: None, + }, )?, )?; - for (field, module, name) in FILE_DISCOVERERS.iter().chain(LATE_FILE_DISCOVERERS.iter()) { - files.set_item(*field, call_with(&imported(py, module, name)?, &directory)?)?; + for (field, discover) in FILE_DISCOVERERS { + let discovered: Bound<'py, PyAny> = match discover { + Some(discover) => discover(project_dir)?, + None => call_with( + &imported(py, FILESYSTEM_MODULE, "discover_python_function_files")?, + &directory, + )?, + }; + files.set_item(field, discovered)?; + } + for (field, discover) in LATE_FILE_DISCOVERERS { + files.set_item(field, discover(project_dir)?)?; } Ok(files) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs index 62668f7c36..6317e4be5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs @@ -1,369 +1,38 @@ -//! Native declaration collections as the plain data the Python discovery facade materialises. +//! In-memory declaration file parsing, for the oracle comparing native and Python parsing. use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::parse_declaration_text::parse_declaration_text; -use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ - AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, - DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, - FunctionFile, HookFile, MacroFile, ParsedDeclarationText, SchemaFile, ScopedFile, SeedFile, + CollectionKind, DeclarationFileOptions, ParsedDeclarationText, }; -use sqlbuild_discovery::declaration_files::types::Collection; -use sqlbuild_discovery::models::{DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot}; -use sqlbuild_discovery::tree::models::ProjectTree; -use sqlbuild_sqltext::compiler::models::AuthoredValue; -use std::path::PathBuf; -use std::sync::Arc; +use sqlbuild_discovery::models::FileOutcome; -use crate::bindings::_helpers::discovery::project_files::{ - PyObject, failure_object, object, path_object, read_object, stage_failure_object, tuple_object, -}; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{ - DeclarationDiscoveryRequest, DeclarationTextRequest, NativeDiscoverySession, NativeProjectTree, -}; +use crate::bindings::models::{DeclarationTextRequest, NativeDiscoverySession}; use crate::bindings::types::CompilerDetach; -type Located = (String, usize, usize, usize, usize); - -fn collection_kind(kind: &str) -> PyResult { - CollectionKind::from_name(kind) - .ok_or_else(|| PyValueError::new_err(format!("unknown declaration collection {kind:?}"))) -} - -fn ok_object(py: Python<'_>, mut items: Vec) -> PyResult { - items.insert(0, object(py, "ok")?); - tuple_object(py, items) -} - -fn values_object(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult { - map_to_python(py, values.to_vec()) -} - -/// `{key: value}`, or `{}` when the value is absent, for Python to project and read back. -fn single_value_object( - py: Python<'_>, - key: &str, - value: Option<&AuthoredValue>, -) -> PyResult { - map_to_python( - py, - value - .map(|value| vec![(key.to_owned(), value.clone())]) - .unwrap_or_default(), - ) -} - -fn failure_or_none(py: Python<'_>, failure: Option<&DiscoveryFailure>) -> PyResult { - failure - .map(|failure| failure_object(py, failure.clone())) - .transpose()? - .into_py_any(py) -} - -fn enum_object(py: Python<'_>, file: &EnumFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - values_object(py, &declaration.members)?, - object(py, declaration.scalar_type)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![object(py, &file.contents)?, object(py, declarations)?], - ) -} - -fn constant_object(py: Python<'_>, file: &ConstantFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - single_value_object(py, "value", Some(&declaration.value))?, - object(py, &declaration.explicit_type)?, - object(py, &declaration.render_as)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![ - object(py, &file.contents)?, - object(py, declarations)?, - failure_or_none(py, file.failure.as_ref())?, - ], - ) -} - -fn schema_object(py: Python<'_>, file: &SchemaFile) -> PyResult { - let declarations: Vec = file - .declarations - .iter() - .map(|declaration| { - let locations: Vec = - declaration.column_locations.iter().map(located).collect(); - tuple_object( - py, - vec![ - object(py, &declaration.name)?, - object(py, &declaration.description)?, - object(py, &declaration.extends)?, - single_value_object(py, "columns", declaration.columns.as_ref())?, - object(py, locations)?, - ], - ) - }) - .collect::>()?; - ok_object( - py, - vec![ - object(py, &file.contents)?, - object(py, declarations)?, - failure_or_none(py, file.failure.as_ref())?, - ], - ) -} - -fn located((name, span): &(String, LineColumnSpan)) -> Located { - ( - name.clone(), - span.line, - span.column, - span.end_line, - span.end_column, - ) -} - -fn audit_object(py: Python<'_>, file: &AuditFile) -> PyResult { - let blocks: Vec = file - .blocks - .iter() - .map(|block| { - tuple_object( - py, - vec![ - object(py, block.audit_index)?, - values_object(py, &block.header_values)?, - object(py, &block.sql_body)?, - object(py, &block.name)?, - object(py, block.evaluation_mode)?, - object(py, &block.measure_sql)?, - object(py, &block.evidence_sql)?, - ], - ) - }) - .collect::>()?; - ok_object(py, vec![object(py, &file.contents)?, object(py, blocks)?]) -} - -fn hook_object(py: Python<'_>, file: &HookFile) -> PyResult { - ok_object( - py, - vec![ - object(py, &file.contents)?, - values_object(py, &file.header_values)?, - object(py, &file.sql_body)?, - object(py, &file.name)?, - object(py, &file.description)?, - ], - ) -} - -fn function_object(py: Python<'_>, file: &FunctionFile) -> PyResult { - ok_object( - py, - vec![ - object(py, &file.contents)?, - values_object(py, &file.header_values)?, - object(py, &file.body_sql)?, - ], - ) -} - -fn macro_object(py: Python<'_>, file: &MacroFile) -> PyResult { - ok_object(py, vec![object(py, &file.contents)?]) -} +const PARSED_TAG: &str = "ok"; +const FAILED_TAG: &str = "error"; +const UNREADABLE_TAG: &str = "read"; -fn seed_object(py: Python<'_>, _file: &SeedFile) -> PyResult { - ok_object(py, Vec::new()) -} - -fn outcome_object( - py: Python<'_>, - outcome: &FileOutcome, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - match outcome { - FileOutcome::Parsed(value) => parsed(py, value), - FileOutcome::Failed(failure) => failure_object(py, failure.clone()), - FileOutcome::Unreadable(failure) => read_object(py, failure.clone()), - } -} - -/// `(declaration kind, scope kind, ownership root, owning path, declaration root)`. -fn scope_object(py: Python<'_>, tree: &ProjectTree, scope: &FileScope) -> PyResult { - let path = |path: &Option| -> PyResult { - path.as_ref() - .map(|path| path_object(py, tree, path.clone())) - .transpose()? - .into_py_any(py) - }; - tuple_object( - py, - vec![ - object(py, scope.declaration_kind)?, - object(py, scope.scope_kind)?, - path(&scope.ownership_root)?, - path(&scope.owning_path)?, - path(&scope.declaration_root)?, - ], - ) -} - -/// Each file as `(relative_path, scope, payload)`, or the collection's failure payload. -fn collection_object( - py: Python<'_>, - tree: &ProjectTree, - collection: &Collection, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - let files = match collection { - Ok(files) => files, - Err(CollectionFailure::Stage(failure)) => { - return stage_failure_object(py, tree, failure.clone()); - } - Err(CollectionFailure::Layout(failure)) => return failure_object(py, failure.clone()), - }; - files - .iter() - .map(|scoped| scoped_file_object(py, tree, scoped, parsed)) - .collect::>>()? - .into_py_any(py) -} - -/// `(relative_path, scope, payload)` of one file. -fn scoped_file_object( - py: Python<'_>, - tree: &ProjectTree, - scoped: &ScopedFile, - parsed: fn(Python<'_>, &T) -> PyResult, -) -> PyResult { - let scope: PyObject = scoped - .scope - .as_ref() - .map(|scope| scope_object(py, tree, scope)) - .transpose()? - .into_py_any(py)?; - tuple_object( - py, - vec![ - path_object(py, tree, scoped.file.relative_path.clone())?, - scope, - outcome_object(py, &scoped.file.outcome, parsed)?, - ], - ) -} - -fn declaration_collection_object( - py: Python<'_>, - tree: &ProjectTree, - collection: &DeclarationCollection, -) -> PyResult { - match collection { - DeclarationCollection::Enums(files) => collection_object(py, tree, files, enum_object), - DeclarationCollection::Constants(files) => { - collection_object(py, tree, files, constant_object) - } - DeclarationCollection::ModelSchemas(files) => { - collection_object(py, tree, files, schema_object) - } - DeclarationCollection::SqlFunctions(files) => { - collection_object(py, tree, files, function_object) - } - DeclarationCollection::SqlHooks(files) => collection_object(py, tree, files, hook_object), - DeclarationCollection::Audits(files) => collection_object(py, tree, files, audit_object), - DeclarationCollection::Seeds(files) => collection_object(py, tree, files, seed_object), - DeclarationCollection::Macros(files) => collection_object(py, tree, files, macro_object), - } -} - -fn options( - (function_keys, audit_keys, hook_keys): (Vec, Vec, Vec), -) -> DeclarationFileOptions { - DeclarationFileOptions { - function_keys, - audit_keys, - hook_keys, - } -} - -#[pymethods] -impl NativeDiscoverySession { - #[new] - fn new(request: DeclarationDiscoveryRequest) -> PyResult { - Ok(Self { - inner: DiscoverySession::new( - ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }, - options((request.function_keys, request.audit_keys, request.hook_keys)), - ), - }) - } - - /// Discover the collection `kind` once per session: one payload per file, or its failure. - #[pyo3(signature = (kind, tree, isolate_kind=false))] - fn collection( - &self, - py: Python<'_>, - kind: &str, - tree: &NativeProjectTree, - isolate_kind: bool, - ) -> PyResult { - let request = CollectionRequest { - kind: collection_kind(kind)?, - isolate_kind, - }; - let collection: Arc = py - .compiler_detach(|| { - Ok(retained_collection(&self.inner, request, || { - discover_declaration_collection(&self.inner, &tree.inner, request) - })) - }) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - declaration_collection_object(py, &tree.inner, &collection) - } -} - -/// Parse in-memory contents as discovery parses a `kind` declaration file named `file_path`. +/// Whether a `kind` declaration file with these contents parses (`ok`), fails or is unreadable. #[pyfunction] fn parse_declaration_contents( py: Python<'_>, request: DeclarationTextRequest, names: (String, String), contents: String, -) -> PyResult { +) -> PyResult<&'static str> { let (file_path, hook_name): (String, String) = names; - let kind: CollectionKind = collection_kind(&request.kind)?; - let options: DeclarationFileOptions = - options((request.function_keys, request.audit_keys, request.hook_keys)); + let kind: CollectionKind = CollectionKind::from_name(&request.kind).ok_or_else(|| { + PyValueError::new_err(format!("unknown declaration collection {:?}", request.kind)) + })?; + let options = DeclarationFileOptions { + function_keys: request.function_keys, + audit_keys: request.audit_keys, + hook_keys: request.hook_keys, + }; let parsed: Option> = py .compiler_detach(|| { Ok(parse_declaration_text( @@ -374,20 +43,15 @@ fn parse_declaration_contents( )) }) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let Some(outcome) = parsed else { - return Err(PyValueError::new_err(format!( + match parsed { + Some(FileOutcome::Parsed(_)) => Ok(PARSED_TAG), + Some(FileOutcome::Failed(_)) => Ok(FAILED_TAG), + Some(FileOutcome::Unreadable(_)) => Ok(UNREADABLE_TAG), + None => Err(PyValueError::new_err(format!( "{} files have no authored text to parse", kind.as_str() - ))); - }; - outcome_object(py, &outcome, |py, parsed| match parsed { - ParsedDeclarationText::Enum(file) => enum_object(py, file), - ParsedDeclarationText::Constant(file) => constant_object(py, file), - ParsedDeclarationText::ModelSchema(file) => schema_object(py, file), - ParsedDeclarationText::SqlFunction(file) => function_object(py, file), - ParsedDeclarationText::SqlHook(file) => hook_object(py, file), - ParsedDeclarationText::Audit(file) => audit_object(py, file), - }) + ))), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs new file mode 100644 index 0000000000..0277039350 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -0,0 +1,700 @@ +//! Declaration file collections discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; +use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; +use sqlbuild_discovery::declaration_files::models::{ + AuditBlock, AuditFile, CollectionFailure, CollectionKind, CollectionRequest, ConstantFile, + DeclarationCollection, DeclarationFileOptions, DiscoverySession, EnumFile, FileScope, + FunctionFile, HookFile, MacroFile, SchemaFile, ScopedFile, +}; +use sqlbuild_discovery::declaration_files::types::Collection; +use sqlbuild_discovery::models::{FileOutcome, LineColumnSpan}; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::sync::Arc; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, snapshot_memo, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const SCHEMA_COLUMNS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.sql.schema_columns"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const PATHLIB_MODULE: &str = "pathlib"; +const SESSION_MEMO_KEY: &str = "native_discovery_session"; +const MEASUREMENT_MODE: &str = "MEASUREMENT"; + +/// One declaration file being built: its paths and the scope native discovery listed it in. +struct FileContext<'a, 'py> { + tree: &'a ProjectTree, + relative_path: Bound<'py, PyAny>, + file_path: Bound<'py, PyAny>, + scope: Option<&'a FileScope>, +} + +impl<'py> FileContext<'_, 'py> { + fn py(&self) -> Python<'py> { + self.relative_path.py() + } + + /// The fields every declaration file record starts with. + fn fields(&self, contents: &str) -> PyResult> { + keywords( + self.py(), + &[ + ("file_path", self.file_path.clone()), + ("relative_path", self.relative_path.clone()), + ("contents", contents.into_bound_py_any(self.py())?), + ], + ) + } + + /// Add the scope fields of a file native discovery listed in a role. + fn add_scope(&self, fields: &Bound<'py, PyDict>) -> PyResult<()> { + let Some(scope) = self.scope else { + return Ok(()); + }; + let py: Python<'py> = self.py(); + fields.set_item( + "scope_kind", + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.call1((scope.scope_kind,))?, + )?; + for (name, path) in [ + ("ownership_root", &scope.ownership_root), + ("owning_path", &scope.owning_path), + ("declaration_root", &scope.declaration_root), + ] { + let value: Bound<'py, PyAny> = match path { + Some(path) => { + path_class(py)?.call1((path_object(py, self.tree, path.clone())?,))? + } + None => py.None().into_bound(py), + }; + fields.set_item(name, value)?; + } + Ok(()) + } + + fn record(&self, class: &str, fields: &Bound<'py, PyDict>) -> PyResult> { + imported(self.py(), MODELS_MODULE, class)?.call((), Some(fields)) + } +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +fn authored<'py>( + py: Python<'py>, + values: &[(String, AuthoredValue)], +) -> PyResult> { + Ok(map_to_python(py, values.to_vec())? + .into_bound(py) + .downcast_into::()?) +} + +/// `{key: value}` or `{}`, read back as Python reads the projected value. +fn single_value<'py>( + py: Python<'py>, + key: &str, + value: Option<&AuthoredValue>, +) -> PyResult>> { + let values: Vec<(String, AuthoredValue)> = value + .map(|value| vec![(key.to_owned(), value.clone())]) + .unwrap_or_default(); + authored(py, &values)?.get_item(key) +} + +fn enum_file<'py>(context: &FileContext<'_, 'py>, file: &EnumFile) -> PyResult> { + let py: Python<'py> = context.py(); + let member_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumMember")?; + let declaration_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumDeclaration")?; + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let members: Vec> = authored(py, &declaration.members)? + .iter() + .map(|(name, value)| call_with(&member_class, &[("name", name), ("value", value)])) + .collect::>()?; + declarations.push(call_with( + &declaration_class, + &[ + ("name", declaration.name.as_str().into_bound_py_any(py)?), + ("members", PyTuple::new(py, members)?.into_any()), + ( + "scalar_type", + declaration.scalar_type.into_bound_py_any(py)?, + ), + ("relative_path", context.relative_path.clone()), + ], + )?); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredEnumFile", &fields) +} + +fn constant_file<'py>( + context: &FileContext<'_, 'py>, + file: &ConstantFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let value: Bound<'py, PyAny> = single_value(py, "value", Some(&declaration.value))? + .unwrap_or_else(|| py.None().into_bound(py)); + declarations.push(typed_constant_record( + py, + (declaration.name.clone(), value), + ( + declaration.explicit_type.clone(), + declaration.render_as.clone(), + ), + (context.file_path.clone(), context.relative_path.clone()), + )?); + } + if let Some(failure) = &file.failure { + return Err(failure_error(py, failure.clone())); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredConstantFile", &fields) +} + +/// Python's `native_locations`: a repeated name keeps its first position and last value. +pub(crate) fn source_locations<'py>( + py: Python<'py>, + relative_path: &Bound<'py, PyAny>, + rows: &[(String, LineColumnSpan)], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?; + let located: Bound<'py, PyDict> = PyDict::new(py); + for (name, span) in rows { + located.set_item( + name, + call_with( + &class, + &[ + ("path", relative_path.clone()), + ("line", span.line.into_bound_py_any(py)?), + ("column", span.column.into_bound_py_any(py)?), + ("end_line", span.end_line.into_bound_py_any(py)?), + ("end_column", span.end_column.into_bound_py_any(py)?), + ], + )?, + )?; + } + Ok(located) +} + +fn schema_file<'py>( + context: &FileContext<'_, 'py>, + file: &SchemaFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let parse_columns: Bound<'py, PyAny> = + imported(py, SCHEMA_COLUMNS_MODULE, "parse_schema_columns")?; + let declaration_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "ModelSchemaDeclaration")?; + let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); + for declaration in &file.declarations { + let raw_columns: Bound<'py, PyAny> = + single_value(py, "columns", declaration.columns.as_ref())? + .unwrap_or_else(|| py.None().into_bound(py)); + let columns: Bound<'py, PyAny> = call_with( + &parse_columns, + &[ + ("raw_columns", raw_columns), + ("file_path", context.file_path.clone()), + ( + "label", + format!("schema '{}'", declaration.name).into_bound_py_any(py)?, + ), + ( + "error_class", + imported(py, DISCOVERY_ERRORS_MODULE, "DeclarationParseError")?, + ), + ( + "column_locations", + source_locations(py, &context.relative_path, &declaration.column_locations)? + .into_any(), + ), + ("require_columns", true.into_bound_py_any(py)?), + ], + )?; + declarations.push(call_with( + &declaration_class, + &[ + ("name", declaration.name.as_str().into_bound_py_any(py)?), + ( + "description", + declaration.description.as_deref().into_bound_py_any(py)?, + ), + ( + "extends", + declaration.extends.as_deref().into_bound_py_any(py)?, + ), + ("columns", columns), + ("relative_path", context.relative_path.clone()), + ], + )?); + } + if let Some(failure) = &file.failure { + return Err(failure_error(py, failure.clone())); + } + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("declarations", PyTuple::new(py, declarations)?)?; + context.add_scope(&fields)?; + context.record("DiscoveredModelSchemaFile", &fields) +} + +fn function_file<'py>( + context: &FileContext<'_, 'py>, + file: &FunctionFile, +) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item( + "header_values", + authored(context.py(), &file.header_values)?, + )?; + fields.set_item("body_sql", &file.body_sql)?; + context.record("DiscoveredSqlFunctionFile", &fields) +} + +fn hook_file<'py>(context: &FileContext<'_, 'py>, file: &HookFile) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item( + "header_values", + authored(context.py(), &file.header_values)?, + )?; + fields.set_item("sql_body", &file.sql_body)?; + fields.set_item("name", &file.name)?; + fields.set_item("description", &file.description)?; + context.add_scope(&fields)?; + context.record("DiscoveredSqlHookFile", &fields) +} + +fn audit_block<'py>(py: Python<'py>, block: &AuditBlock) -> PyResult> { + let header_values: Bound<'py, PyDict> = authored(py, &block.header_values)?; + let mode: Bound<'py, PyAny> = imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? + .call1((block.evaluation_mode,))?; + let measurement: Bound<'py, PyAny> = + if mode.is(&imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? + .getattr(MEASUREMENT_MODE)?) + { + call_with( + &imported(py, AUDITING_MODELS_MODULE, "MeasurementContract")?, + &[ + ("value_column", header_values.as_any().get_item("value")?), + ( + "sample_count_column", + header_values + .as_any() + .call_method1("get", ("sample_count",))?, + ), + ( + "sample_unit", + header_values + .as_any() + .call_method1("get", ("sample_unit",))?, + ), + ], + )? + } else { + py.None().into_bound(py) + }; + call_with( + &imported(py, MODELS_MODULE, "DiscoveredAuditBlock")?, + &[ + ("audit_index", block.audit_index.into_bound_py_any(py)?), + ("header_values", header_values.into_any()), + ("sql_body", block.sql_body.as_str().into_bound_py_any(py)?), + ("name", block.name.as_deref().into_bound_py_any(py)?), + ("evaluation_mode", mode), + ("measurement_contract", measurement), + ( + "measure_sql", + block.measure_sql.as_deref().into_bound_py_any(py)?, + ), + ( + "evidence_sql", + block.evidence_sql.as_deref().into_bound_py_any(py)?, + ), + ], + ) +} + +fn audit_file<'py>( + context: &FileContext<'_, 'py>, + file: &AuditFile, +) -> PyResult> { + let py: Python<'py> = context.py(); + let blocks: Vec> = file + .blocks + .iter() + .map(|block| audit_block(py, block)) + .collect::>()?; + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + fields.set_item("blocks", PyTuple::new(py, blocks)?)?; + let declaration_kind: &str = context.scope.map_or("", |scope| scope.declaration_kind); + fields.set_item( + "declaration_kind", + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?.call1((declaration_kind,))?, + )?; + context.add_scope(&fields)?; + context.record("DiscoveredAuditFile", &fields) +} + +fn macro_file<'py>( + context: &FileContext<'_, 'py>, + file: &MacroFile, +) -> PyResult> { + let fields: Bound<'py, PyDict> = context.fields(&file.contents)?; + context.add_scope(&fields)?; + context.record("DiscoveredMacroFile", &fields) +} + +/// The session of this pass, created and memoised by the first declaration collection. +fn discovery_session<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + if let Some(session) = memo.get_item(SESSION_MEMO_KEY)? + && let Ok(session) = session.downcast_into::() + { + return Ok(session); + } + let created: Bound<'py, NativeDiscoverySession> = Bound::new( + py, + NativeDiscoverySession { + inner: DiscoverySession::new( + project_root(project_dir)?, + DeclarationFileOptions { + function_keys: sorted_keys(py, "SQL_FUNCTION_HEADER_KEYS")?, + audit_keys: sorted_keys(py, "SQL_AUDIT_HEADER_KEYS")?, + hook_keys: sorted_keys(py, "SQL_HOOK_HEADER_KEYS")?, + }, + ), + }, + )?; + memo.set_item(SESSION_MEMO_KEY, &created)?; + Ok(created) +} + +/// The session this discovery pass read its declaration files with, if any. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn retained_discovery_session<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + Ok(snapshot_memo(project_dir)? + .get_item(SESSION_MEMO_KEY)? + .filter(|session| session.is_instance_of::()) + .unwrap_or_else(|| py.None().into_bound(py))) +} + +/// One collection read in the pass's snapshot, with the tree it was read from. +fn collection<'py>( + project_dir: &Bound<'py, PyAny>, + request: CollectionRequest, +) -> PyResult<(Bound<'py, NativeProjectTree>, Arc)> { + in_snapshot(project_dir, || { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let session: Bound<'py, NativeDiscoverySession> = discovery_session(project_dir)?; + let collection: Arc = { + let session: &DiscoverySession = &session.get().inner; + let inner: &ProjectTree = &tree.get().inner; + py.detach(|| { + retained_collection(session, request, || { + discover_declaration_collection(session, inner, request) + }) + }) + }; + if let Some(failure) = collection_failure(&collection) { + return Err(match failure { + CollectionFailure::Stage(failure) => { + stage_error(&tree.get().inner, project_dir, failure.clone()) + } + CollectionFailure::Layout(failure) => failure_error(py, failure.clone()), + }); + } + seed_listings(project_dir, &tree)?; + Ok((tree, collection)) + }) +} + +/// The files of a collection whose failure was already raised. +fn files_of(collection: &Collection) -> &[ScopedFile] { + collection.as_ref().map(Vec::as_slice).unwrap_or_default() +} + +fn failure_of(collection: &Collection) -> Option<&CollectionFailure> { + collection.as_ref().err() +} + +fn collection_failure(collection: &DeclarationCollection) -> Option<&CollectionFailure> { + match collection { + DeclarationCollection::Enums(files) => failure_of(files), + DeclarationCollection::Constants(files) => failure_of(files), + DeclarationCollection::ModelSchemas(files) => failure_of(files), + DeclarationCollection::SqlFunctions(files) => failure_of(files), + DeclarationCollection::SqlHooks(files) => failure_of(files), + DeclarationCollection::Audits(files) => failure_of(files), + DeclarationCollection::Seeds(files) => failure_of(files), + DeclarationCollection::Macros(files) => failure_of(files), + } +} + +/// Build each parsed file of a collection, raising or reporting each failing one. +fn records<'py, T>( + faults: &FileFaults<'py>, + tree: &ProjectTree, + files: &[ScopedFile], + build: fn(&FileContext<'_, 'py>, &T) -> PyResult>, +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let mut built: Vec> = Vec::with_capacity(files.len()); + for scoped in files { + let relative_path: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, tree, scoped.file.relative_path.clone())?,))?; + let file_path: Bound<'py, PyAny> = faults.project_dir.div(&relative_path)?; + let record: PyResult> = match &scoped.file.outcome { + FileOutcome::Parsed(file) => build( + &FileContext { + tree, + relative_path: relative_path.clone(), + file_path, + scope: scoped.scope.as_ref(), + }, + file, + ), + FileOutcome::Failed(failure) => Err(failure_error(py, failure.clone())), + FileOutcome::Unreadable(failure) => Err(read_error(failure.clone(), &file_path)), + }; + built.extend(faults.kept(&relative_path, record)?); + } + record_tuple(py, built) +} + +fn discovered<'py>( + faults: &FileFaults<'py>, + kind: CollectionKind, + isolate_kind: bool, +) -> PyResult> { + let (tree, collection) = collection( + &faults.project_dir, + CollectionRequest { kind, isolate_kind }, + )?; + let tree: &ProjectTree = &tree.get().inner; + match collection.as_ref() { + DeclarationCollection::Enums(files) => records(faults, tree, files_of(files), enum_file), + DeclarationCollection::Constants(files) => { + records(faults, tree, files_of(files), constant_file) + } + DeclarationCollection::ModelSchemas(files) => { + records(faults, tree, files_of(files), schema_file) + } + DeclarationCollection::SqlFunctions(files) => { + records(faults, tree, files_of(files), function_file) + } + DeclarationCollection::SqlHooks(files) => records(faults, tree, files_of(files), hook_file), + DeclarationCollection::Audits(files) => records(faults, tree, files_of(files), audit_file), + DeclarationCollection::Macros(files) => records(faults, tree, files_of(files), macro_file), + DeclarationCollection::Seeds(files) => seed_records(faults, tree, files_of(files)), + } +} + +fn seed_records<'py, T>( + faults: &FileFaults<'py>, + tree: &ProjectTree, + files: &[ScopedFile], +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSeedFile")?; + let seeds: Vec> = files + .iter() + .map(|scoped| { + let relative: Py = path_object(py, tree, scoped.file.relative_path.clone())?; + call_with( + &class, + &[ + ("file_path", faults.project_dir.div(relative.bind(py))?), + ("relative_path", path_class(py)?.call1((relative,))?), + ], + ) + }) + .collect::>()?; + record_tuple(py, seeds) +} + +fn faults<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> FileFaults<'py> { + FileFaults { + project_dir: project_dir.clone(), + on_fault: on_fault.filter(|on_fault| !on_fault.is_none()), + } +} + +/// Discover global and scoped enum declaration files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] +pub(crate) fn discover_enum_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Enums, + isolate_declaration_kind, + ) +} + +/// Discover global and scoped constant declaration files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] +pub(crate) fn discover_constant_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Constants, + isolate_declaration_kind, + ) +} + +/// Discover project-wide and scoped reusable model schemas. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_model_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::ModelSchemas, + false, + ) +} + +/// Discover SQL function files under functions/sql/. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_sql_function_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlFunctions, + false, + ) +} + +/// Discover named SQL lifecycle hooks in their project-wide and scoped roles. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_sql_hook_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlHooks, + false, + ) +} + +/// Discover generic and singular audit SQL files in their project-wide and scoped roles. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_audit_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + discovered( + &faults(project_dir, on_fault), + CollectionKind::Audits, + false, + ) +} + +/// Discover global and scoped project macro Python files. +#[pyfunction] +#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] +pub(crate) fn discover_macro_files<'py>( + project_dir: &Bound<'py, PyAny>, + isolate_declaration_kind: bool, +) -> PyResult> { + discovered( + &faults(project_dir, None), + CollectionKind::Macros, + isolate_declaration_kind, + ) +} + +/// Discover seed CSV files under seeds/. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_seed_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + discovered(&faults(project_dir, None), CollectionKind::Seeds, false) +} + +/// Detect a project-level adapter.py without importing it. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_adapter_file<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; + if !file_path.call_method0("is_file")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + call_with( + &imported(py, MODELS_MODULE, "DiscoveredAdapterFile")?, + &[ + ("file_path", file_path), + ("relative_path", path_class(py)?.call1(("adapter.py",))?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; + module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs new file mode 100644 index 0000000000..8105fa7314 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs @@ -0,0 +1,62 @@ +//! Report and skip a failing file when the caller tolerates file faults. + +use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyTuple; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const FAULT_CLASS: &str = "DiscoveryFileFault"; +const PROJECT_DIR_MARKER: &str = "."; + +/// The files one discovery builds, with the caller's optional `on_fault` reporter. +pub(crate) struct FileFaults<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) on_fault: Option>, +} + +impl<'py> FileFaults<'py> { + /// Whether `on_fault` takes this error instead of the caller. + fn tolerated(&self, error: &PyErr) -> bool { + let py: Python<'py> = self.project_dir.py(); + self.on_fault.is_some() + && (error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py)) + } + + /// The built record; a tolerated failure for `relative_path` is reported and skipped. + pub(crate) fn kept( + &self, + relative_path: &Bound<'py, PyAny>, + built: PyResult>, + ) -> PyResult>> { + let error: PyErr = match built { + Ok(record) => return Ok(Some(record)), + Err(error) => error, + }; + let Some(on_fault) = self.on_fault.as_ref().filter(|_| self.tolerated(&error)) else { + return Err(error); + }; + let py: Python<'py> = self.project_dir.py(); + let message: Bound<'py, PyAny> = error + .value(py) + .str()? + .call_method1("replace", (self.project_dir.str()?, PROJECT_DIR_MARKER))?; + on_fault.call1((call_with( + &imported(py, DISCOVERY_MODELS_MODULE, FAULT_CLASS)?, + &[("path", relative_path.clone()), ("message", message)], + )?,))?; + Ok(None) + } +} + +/// The built records as the tuple a discoverer returns. +pub(crate) fn record_tuple<'py>( + py: Python<'py>, + records: Vec>, +) -> PyResult> { + PyTuple::new(py, records) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index f177fcbbee..93c2ccccc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -160,8 +160,7 @@ pub(crate) fn model_header_declarations<'py>( } /// `typed_constant_declaration` for a public constant file entry. -#[pyfunction] -fn typed_constant_record<'py>( +pub(crate) fn typed_constant_record<'py>( py: Python<'py>, constant: (String, Bound<'py, PyAny>), options: (Option, Option), @@ -193,6 +192,5 @@ fn typed_constant_record<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_header_declarations, module)?)?; - module.add_function(wrap_pyfunction!(typed_constant_record, module)?) + module.add_function(wrap_pyfunction!(model_header_declarations, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs new file mode 100644 index 0000000000..9b77e9553a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/layout_groups.rs @@ -0,0 +1,84 @@ +//! The named declaration groups of a valid declaration layout, memoised per snapshot. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; +use sqlbuild_discovery::declarations::models::{DeclarationGroup, DeclarationLayout}; +use sqlbuild_discovery::models::{DiscoveryFailure, StageFailure}; + +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{discovery_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{project_tree, seed_listings, snapshot_memo}; +use crate::bindings::models::NativeProjectTree; + +const GROUPS_MEMO_KEY: &str = "native_declaration_groups"; + +/// The rows to memoise: `(root, directory)` pairs, or the failure's `(kind, message, help)`. +fn memoised<'py>( + py: Python<'py>, + tree: &Bound<'py, NativeProjectTree>, + groups: Result, DiscoveryFailure>, +) -> PyResult> { + match groups { + Ok(groups) => { + let rows: Vec> = groups + .into_iter() + .map(|group| { + PyTuple::new( + py, + [ + path_object(py, &tree.get().inner, group.root)?, + path_object(py, &tree.get().inner, group.directory)?, + ], + ) + }) + .collect::>()?; + Ok(PyList::new(py, rows)?.into_any()) + } + Err(failure) => Ok(PyTuple::new( + py, + [ + failure.kind.as_str().into_bound_py_any(py)?, + failure.message.into_bound_py_any(py)?, + failure.help.into_bound_py_any(py)?, + ], + )? + .into_any()), + } +} + +/// The `(canonical root, group directory)` pairs of a valid named declaration layout. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn declaration_groups<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + let groups: Bound<'py, PyAny> = match memo.get_item(GROUPS_MEMO_KEY)? { + Some(groups) => groups, + None => { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let layout: Result = { + let inner = &tree.get().inner; + py.detach(|| declaration_layout(inner, None)) + }; + let layout: DeclarationLayout = + layout.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + seed_listings(project_dir, &tree)?; + let groups: Bound<'py, PyAny> = memoised(py, &tree, layout.named_groups)?; + memo.set_item(GROUPS_MEMO_KEY, &groups)?; + groups + } + }; + if let Ok(rows) = groups.downcast::() { + return Ok(rows.clone()); + } + let (kind, message, help): (String, String, Option) = groups.extract()?; + Err(discovery_error(py, &kind, message, help)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declaration_groups, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 99bd00951b..444fa2041c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,12 +1,21 @@ pub(crate) mod declaration_files; +pub(crate) mod declaration_records; +pub(crate) mod faults; pub(crate) mod globbing; pub(crate) mod header_declarations; pub(crate) mod key_checks; +pub(crate) mod layout_groups; pub(crate) mod locations; pub(crate) mod model_file_record; +pub(crate) mod model_records; pub(crate) mod project_facts; pub(crate) mod project_files; pub(crate) mod project_validation; +pub(crate) mod raised; pub(crate) mod registration; +pub(crate) mod snapshot; +pub(crate) mod sorted_keys; pub(crate) mod sql_test_blocks; +pub(crate) mod sql_test_records; pub(crate) mod yaml_entries; +pub(crate) mod yaml_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs new file mode 100644 index 0000000000..aff9e21786 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -0,0 +1,144 @@ +//! SQL model files discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::models::NativeProjectTree; + +const PATHLIB_MODULE: &str = "pathlib"; +const STEM_ATTRIBUTE: &str = "stem"; + +/// One discovered model file: its record, or the error building it raises. +type ModelItem<'py> = (Bound<'py, PyAny>, PyResult>); + +/// Discover every model file natively, each as its record or the error it raises when built. +fn model_items<'py>( + project_dir: &Bound<'py, PyAny>, + options: &ModelFileOptions, +) -> PyResult>> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root = project_root(project_dir)?; + let discovered: Result>, StageFailure> = { + let inner = &tree.get().inner; + py.detach(|| discover_models(&root, inner, options)) + }; + let files = + discovered.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let mut items: Vec> = Vec::with_capacity(files.len()); + for file in files { + let relative_path: Bound<'py, PyAny> = + path_class.call1((path_object(py, &tree.get().inner, file.relative_path)?,))?; + let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; + let built: PyResult> = match file.outcome { + FileOutcome::Parsed(model) => model_record( + py, + model, + (file_path, relative_path.clone()), + options.extract_implicit_alias_columns, + )? + .map(|record| record.into_bound(py)), + FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), + FileOutcome::Failed(failure) => Err(failure_error(py, failure)), + }; + items.push((relative_path, built)); + } + seed_listings(project_dir, &tree)?; + Ok(items) + }) +} + +/// Which discovered model files to build, and where failing files are reported. +pub(crate) struct ModelFileSelection<'py> { + pub(crate) selected_model_names: Option>, + pub(crate) on_fault: Option>, +} + +/// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. +pub(crate) fn discovered_model_files<'py>( + project_dir: &Bound<'py, PyAny>, + extraction: (bool, bool), + selection: ModelFileSelection<'py>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; + let options = ModelFileOptions { + supported_keys: sorted_keys(py, "SQL_MODEL_HEADER_KEYS")?, + removed_keys: sorted_keys(py, "REMOVED_SQL_MODEL_HEADER_KEYS")?, + extract_implicit_alias_columns, + extract_output_column_locations, + }; + let items: Vec> = model_items(project_dir, &options)?; + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), + }; + let selected: Option> = selection + .selected_model_names + .filter(|selected| !selected.is_none()); + let mut records: Vec> = Vec::with_capacity(items.len()); + for (relative_path, built) in items { + if let Some(selected) = &selected + && !selected.contains(relative_path.getattr(STEM_ATTRIBUTE)?)? + { + continue; + } + records.extend(faults.kept(&relative_path, built)?); + } + record_tuple(py, records) +} + +/// Discover the (selected) model files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn discover_model_files<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_model_files", + options, + ( + &[ + "project_dir", + "extract_implicit_alias_columns", + "extract_output_column_locations", + ], + &["selected_model_names", "on_fault"], + ), + )?; + discovered_model_files( + &arguments.value("project_dir")?, + ( + arguments + .value("extract_implicit_alias_columns")? + .extract()?, + arguments + .value("extract_output_column_locations")? + .extract()?, + ), + ModelFileSelection { + selected_model_names: Some(arguments.value("selected_model_names")?), + on_fault: Some(arguments.value("on_fault")?), + }, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_model_files, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index be9bc78a0d..5d823b4ecd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -1,193 +1,35 @@ //! Native discovery results as the plain data the Python discovery facade materialises. -use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; +use pyo3::exceptions::PyException; use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; -use sqlbuild_discovery::declarations::models::{ - DeclarationFileFact, DeclarationGroup, DeclarationKind, DeclarationLayout, -}; -use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; -use sqlbuild_discovery::models::{ - DiscoveredFile, DiscoveryFailure, FailureKind, FileOutcome, LineColumnSpan, ProjectRoot, - ReadFailure, StageFailure, -}; -use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; -use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; -use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; -use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; -use sqlbuild_discovery::sql_tests::models::{ - DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, -}; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; -use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; -use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; -use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use std::collections::HashSet; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Mutex; -use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::discovery::declaration_records::source_locations; use crate::bindings::_helpers::discovery::header_declarations::{ HeaderDeclarations, model_header_declarations, }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileParts, NativeModelFile}; +use crate::bindings::_helpers::discovery::raised::failure_error; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{ - ModelDiscoveryRequest, ModelTextRequest, NativeProjectTree, SqlTestDiscoveryRequest, - SqlTestTextRequest, YamlDiscoveryRequest, -}; +use crate::bindings::models::{ModelTextRequest, NativeProjectTree}; use crate::bindings::types::CompilerDetach; pub(crate) type PyObject = Py; -const PATHLIB_MODULE: &str = "pathlib"; -const PATH_CLASS: &str = "Path"; const STEM_ATTRIBUTE: &str = "stem"; const ENUMS_KEY: &str = "enums"; const CONSTANTS_KEY: &str = "constants"; -const RAISED_TAG: &str = "raised"; -type Located = (String, usize, usize, usize, usize); -type Listing = (PyObject, Vec<(PyObject, bool, bool)>); - -fn located(locations: Vec<(String, LineColumnSpan)>) -> Vec { - locations - .into_iter() - .map(|(name, span)| (name, span.line, span.column, span.end_line, span.end_column)) - .collect() -} - -pub(crate) fn tuple_object<'py>(py: Python<'py>, items: Vec) -> PyResult { - Ok(PyTuple::new(py, items)?.unbind().into_any()) -} - -pub(crate) fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { - value.into_py_any(py) -} - -pub(crate) fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { - tuple_object( - py, - vec![ - object(py, "error")?, - object(py, failure.kind.as_str())?, - object(py, failure.message)?, - object(py, failure.help)?, - ], - ) -} - -/// `("read", "os", errno, message, winerror)` or `("read", "decode", bytes, start, end, reason)`. -pub(crate) fn read_object(py: Python<'_>, failure: ReadFailure) -> PyResult { - match failure { - ReadFailure::Io { - errno, - winerror, - message, - } => tuple_object( - py, - vec![ - object(py, "read")?, - object(py, "os")?, - object(py, errno)?, - object(py, message)?, - object(py, winerror)?, - ], - ), - ReadFailure::Decode { - bytes, - start, - end, - reason, - } => tuple_object( - py, - vec![ - object(py, "read")?, - object(py, "decode")?, - PyBytes::new(py, &bytes).unbind().into_any(), - object(py, start)?, - object(py, end)?, - object(py, reason)?, - ], - ), - } -} - -/// A collection's failure payload; an unlistable directory is `("unlistable", path, read)`. -pub(crate) fn stage_failure_object( - py: Python<'_>, - tree: &ProjectTree, - failure: StageFailure, -) -> PyResult { - match failure { - StageFailure::Unlistable { - relative_path, - error, - } => tuple_object( - py, - vec![ - object(py, "unlistable")?, - path_object(py, tree, relative_path)?, - read_object(py, error)?, - ], - ), - StageFailure::Internal(reason) => Err(PyRuntimeError::new_err(reason)), - } -} - -fn outcome_object( - py: Python<'_>, - outcome: FileOutcome, - parsed: impl FnOnce(Python<'_>, T) -> PyResult, -) -> PyResult { - match outcome { - FileOutcome::Parsed(value) => parsed(py, value), - FileOutcome::Unreadable(failure) => read_object(py, failure), - FileOutcome::Failed(failure) => failure_object(py, failure), - } -} - -/// Each discovered file as `(relative_path, payload)`, or the collection's failure payload. -fn files_object( - py: Python<'_>, - tree: &ProjectTree, - discovered: Result>, StageFailure>, - parsed: impl Fn(Python<'_>, T) -> PyResult, -) -> PyResult { - match discovered { - Ok(files) => files - .into_iter() - .map(|file| { - Ok(( - path_object(py, tree, file.relative_path)?, - outcome_object(py, file.outcome, &parsed)?, - )) - }) - .collect::>>()? - .into_py_any(py), - Err(failure) => stage_failure_object(py, tree, failure), - } -} - -fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult { - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, model.contents)?, - map_to_python(py, model.header_values)?, - object(py, located(model.header_column_locations))?, - object(py, located(model.output_column_locations))?, - object(py, model.query_sql)?, - ], - ) -} +pub(crate) type Listing = (PyObject, Vec<(PyObject, bool, bool)>); /// A relative path as Python names it: the real, surrogate-escaped name where it is not UTF-8. pub(crate) fn path_object( @@ -196,8 +38,8 @@ pub(crate) fn path_object( relative_path: String, ) -> PyResult { match tree.raw_relative(&relative_path) { - Some(raw) => object(py, raw.into_os_string()), - None => object(py, relative_path), + Some(raw) => raw.into_os_string().into_py_any(py), + None => relative_path.into_py_any(py), } } @@ -206,83 +48,14 @@ fn entry_rows(py: Python<'_>, entries: &[TreeEntry]) -> PyResult object(py, raw)?, - None => object(py, &entry.name)?, + Some(raw) => raw.into_py_any(py)?, + None => entry.name.as_str().into_py_any(py)?, }; Ok((name, entry.is_dir, entry.is_walkable_dir)) }) .collect() } -fn fact_row(py: Python<'_>, tree: &ProjectTree, fact: DeclarationFileFact) -> PyResult { - let owning_path: Option = fact - .owning_path - .map(|path| path_object(py, tree, path)) - .transpose()?; - tuple_object( - py, - vec![ - path_object(py, tree, fact.relative_path)?, - object(py, fact.kind.as_str())?, - object(py, fact.scope_kind.as_str())?, - path_object(py, tree, fact.ownership_root)?, - object(py, owning_path)?, - path_object(py, tree, fact.declaration_root)?, - ], - ) -} - -fn group_row(py: Python<'_>, tree: &ProjectTree, group: DeclarationGroup) -> PyResult { - tuple_object( - py, - vec![ - path_object(py, tree, group.root)?, - path_object(py, tree, group.directory)?, - ], - ) -} - -/// `("ok", rows)` for a valid layout, or the failure Python raises for it. -fn layout_rows( - py: Python<'_>, - tree: &ProjectTree, - outcome: Result, DiscoveryFailure>, - row: fn(Python<'_>, &ProjectTree, T) -> PyResult, -) -> PyResult { - match outcome { - Ok(items) => { - let rows: Vec = items - .into_iter() - .map(|item| row(py, tree, item)) - .collect::>()?; - tuple_object(py, vec![object(py, "ok")?, object(py, rows)?]) - } - Err(failure) => failure_object(py, failure), - } -} - -fn declaration_kind(kind: Option<&str>) -> PyResult> { - match kind { - None => Ok(None), - Some("macro") => Ok(Some(DeclarationKind::Macro)), - Some("enum") => Ok(Some(DeclarationKind::Enum)), - Some("constant") => Ok(Some(DeclarationKind::Constant)), - Some(other) => Err(PyValueError::new_err(format!( - "unknown declaration kind {other:?}" - ))), - } -} - -fn yaml_failure_kind(kind: &str) -> PyResult { - match kind { - "source" => Ok(FailureKind::Source), - "schema" => Ok(FailureKind::Schema), - other => Err(PyValueError::new_err(format!( - "unknown YAML file kind {other:?}" - ))), - } -} - #[pymethods] impl NativeProjectTree { #[new] @@ -292,9 +65,11 @@ impl NativeProjectTree { exported: Mutex::new(HashSet::new()), } } +} - /// The listings read since the last call, for the pass's Python directory snapshot. - fn listings(&self, py: Python<'_>) -> PyResult> { +impl NativeProjectTree { + /// The listings read since the last export, each exported once. + pub(crate) fn exported_listings(&self, py: Python<'_>) -> PyResult> { let Ok(mut exported) = self.exported.lock() else { return Ok(Vec::new()); }; @@ -311,61 +86,13 @@ impl NativeProjectTree { } } -/// Each model file's `(relative path, record or failure payload)`, or the collection's failure. -#[pyfunction] -fn discover_model_files( - py: Python<'_>, - request: ModelDiscoveryRequest, - tree: &NativeProjectTree, - project_dir: Bound<'_, PyAny>, -) -> PyResult { - let root = ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }; - let options = ModelFileOptions { - supported_keys: request.supported_keys, - removed_keys: request.removed_keys, - extract_implicit_alias_columns: request.extract_implicit_alias_columns, - extract_output_column_locations: request.extract_output_column_locations, - }; - let extract_implicit_alias_columns: bool = options.extract_implicit_alias_columns; - let discovered = py - .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let files: Vec> = match discovered { - Ok(files) => files, - Err(failure) => return stage_failure_object(py, &tree.inner, failure), - }; - let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?; - files - .into_iter() - .map(|file| { - let relative_path: Bound<'_, PyAny> = - path_class.call1((path_object(py, &tree.inner, file.relative_path)?,))?; - let item: PyObject = match file.outcome { - FileOutcome::Parsed(model) => model_record( - py, - model, - (project_dir.div(&relative_path)?, relative_path.clone()), - extract_implicit_alias_columns, - )?, - FileOutcome::Unreadable(failure) => read_object(py, failure)?, - FileOutcome::Failed(failure) => failure_object(py, failure)?, - }; - Ok((relative_path.unbind(), item)) - }) - .collect::>>()? - .into_py_any(py) -} - -/// One parsed model file as its native record, or `("raised", error)` for invalid declarations. -fn model_record<'py>( +/// One parsed model file's record, or its header declaration error, raised only if it is built. +pub(crate) fn model_record<'py>( py: Python<'py>, model: DiscoveredModelFile, paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), extract_implicit_alias_columns: bool, -) -> PyResult { +) -> PyResult> { let (file_path, relative_path) = paths; let header_values: Bound<'py, PyDict> = map_to_python(py, model.header_values)? .into_bound(py) @@ -381,12 +108,7 @@ fn model_record<'py>( relative_path.clone(), ) { Ok(declarations) => declarations, - Err(error) if error.is_instance_of::(py) => { - return tuple_object( - py, - vec![object(py, RAISED_TAG)?, error.into_value(py).into_any()], - ); - } + Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), Err(error) => return Err(error), }; let (enum_declarations, constant_declarations) = declarations; @@ -401,16 +123,16 @@ fn model_record<'py>( declarations: (enum_declarations.unbind(), constant_declarations.unbind()), extract_implicit_alias_columns, }); - record.into_py_any(py) + Ok(Ok(record.into_py_any(py)?)) } -/// Parse in-memory SQL model contents into the payload model discovery returns for a file. +/// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. #[pyfunction] fn parse_model_contents( py: Python<'_>, request: ModelTextRequest, contents: String, -) -> PyResult { +) -> PyResult<(PyObject, String, PyObject)> { let options = ModelFileOptions { supported_keys: request.supported_keys, removed_keys: request.removed_keys, @@ -420,248 +142,24 @@ fn parse_model_contents( let parsed = py .compiler_detach(|| Ok(parse_model_text(&request.file_path, contents, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(model) => model_object(py, model), - Err(failure) => failure_object(py, failure), - } -} - -/// The declaration file facts (of one kind, if given) and named groups, or the stage failure. -#[pyfunction] -#[pyo3(signature = (tree, kind=None))] -fn discover_declaration_layout( - py: Python<'_>, - tree: &NativeProjectTree, - kind: Option<&str>, -) -> PyResult { - let kind: Option = declaration_kind(kind)?; - let layout: Result = py - .compiler_detach(|| Ok(declaration_layout(&tree.inner, kind))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match layout { - Ok(layout) => tuple_object( - py, - vec![ - layout_rows(py, &tree.inner, layout.file_facts, fact_row)?, - layout_rows(py, &tree.inner, layout.named_groups, group_row)?, - ], - ), - Err(failure) => stage_failure_object(py, &tree.inner, failure), - } -} - -fn test_block_object(py: Python<'_>, block: SqlTestBlock) -> PyResult { - tuple_object( - py, - vec![ - map_to_python(py, block.header_values)?, - object(py, block.sql_body)?, - object(py, block.body_span)?, - ], - ) -} - -fn test_file_object(py: Python<'_>, file: DiscoveredSqlTestFile) -> PyResult { - let blocks: Vec = file - .blocks - .into_iter() - .map(|block| test_block_object(py, block)) - .collect::>()?; - let failure: Option = file - .failure - .map(|failure| failure_object(py, failure)) - .transpose()?; - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, file.contents)?, - object(py, blocks)?, - object(py, failure)?, - ], - ) -} - -fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult { - tuple_object( - py, - vec![ - object(py, "ok")?, - object(py, file.contents)?, - map_to_python(py, file.header_values)?, - object(py, file.sql_body)?, - object(py, file.body_span)?, - ], - ) -} - -/// The root and options of one SQL test or scenario discovery request. -fn sql_test_inputs( - request: SqlTestDiscoveryRequest, -) -> PyResult<(ProjectRoot, SqlTestFileOptions)> { + let model: DiscoveredModelFile = parsed.map_err(|failure| failure_error(py, failure))?; + let path: Bound<'_, PyAny> = py + .import("pathlib")? + .getattr("Path")? + .call1((request.file_path.as_str(),))?; Ok(( - ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }, - SqlTestFileOptions { - test_keys: request.test_keys, - scenario_keys: request.scenario_keys, - }, + map_to_python(py, model.header_values)?, + model.query_sql, + source_locations(py, &path, &model.header_column_locations)? + .into_any() + .unbind(), )) } -/// Discover and split the SQL test files: one payload per file, or the failure payload. -#[pyfunction] -fn discover_sql_test_files( - py: Python<'_>, - request: SqlTestDiscoveryRequest, - tree: &NativeProjectTree, -) -> PyResult { - let (root, options) = sql_test_inputs(request)?; - let discovered = py - .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - files_object(py, &tree.inner, discovered, test_file_object) -} - -/// Discover and header-parse the scenario files: one payload per file, or the failure payload. -#[pyfunction] -fn discover_scenario_files( - py: Python<'_>, - request: SqlTestDiscoveryRequest, - tree: &NativeProjectTree, -) -> PyResult { - let (root, options) = sql_test_inputs(request)?; - let discovered = py - .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - files_object(py, &tree.inner, discovered, scenario_object) -} - -/// The options of one in-memory SQL test or scenario request. -fn text_options(request: &SqlTestTextRequest) -> PyResult { - Ok(SqlTestFileOptions { - test_keys: request.test_keys.clone(), - scenario_keys: request.scenario_keys.clone(), - }) -} - -/// Split in-memory SQL test contents into the payload test discovery returns for a file. -#[pyfunction] -fn parse_sql_test_contents( - py: Python<'_>, - request: SqlTestTextRequest, - contents: String, -) -> PyResult { - let options: SqlTestFileOptions = text_options(&request)?; - let parsed = py - .compiler_detach(|| Ok(parse_sql_test_text(&request.file_path, contents, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(file) => test_file_object(py, file), - Err(failure) => failure_object(py, failure), - } -} - -/// Header-parse in-memory SQL scenario contents into the payload scenario discovery returns. -#[pyfunction] -fn parse_scenario_contents( - py: Python<'_>, - request: SqlTestTextRequest, - contents: String, -) -> PyResult { - let options: SqlTestFileOptions = text_options(&request)?; - let parsed = py - .compiler_detach(|| Ok(parse_scenario_text(&request.file_path, contents, &options))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match parsed { - Ok(file) => scenario_object(py, file), - Err(failure) => failure_object(py, failure), - } -} - -/// `("ok", contents, value)`; a value Python cannot build fails like invalid YAML. -fn loaded_yaml_object( - py: Python<'_>, - (file_path, kind): (&str, FailureKind), - (contents, value): (String, sqlbuild_config::models::ConfigValue), -) -> PyResult { - match config_value_to_python(py, value) { - Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), - Err(error) => failure_object( - py, - DiscoveryFailure::new( - kind, - format!("{file_path} contains invalid YAML: {}", error.value(py)), - ), - ), - } -} - -/// Read and load YAML files: one payload per path, or the collection's failure payload. -#[pyfunction] -fn load_yaml_files( - py: Python<'_>, - request: YamlDiscoveryRequest, - relative_paths: Vec, - tree: &NativeProjectTree, -) -> PyResult { - let kind: FailureKind = yaml_failure_kind(&request.kind)?; - let root = ProjectRoot { - directory: PathBuf::from(request.project_dir), - display_prefix: request.display_prefix, - }; - let loaded: Result, StageFailure> = py - .compiler_detach(|| Ok(load_yaml(&root, &tree.inner, &relative_paths, kind))) - .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - let outcomes: Vec = match loaded { - Ok(outcomes) => outcomes, - Err(failure) => return stage_failure_object(py, &tree.inner, failure), - }; - outcomes - .into_iter() - .zip(&relative_paths) - .map(|(outcome, relative_path)| match outcome { - YamlFileOutcome::Loaded { contents, value } => loaded_yaml_object( - py, - (&root.display_path(relative_path), kind), - (contents, value), - ), - YamlFileOutcome::Failed(failure) => failure_object(py, failure), - YamlFileOutcome::Unreadable(failure) => read_object(py, failure), - }) - .collect::>>()? - .into_py_any(py) -} - -/// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. -#[pyfunction] -fn load_yaml_document( - py: Python<'_>, - file_path: &str, - text: String, - kind: &str, -) -> PyResult { - let kind: FailureKind = yaml_failure_kind(kind)?; - match load_yaml_text(file_path, &text, kind) { - Ok(value) => loaded_yaml_object(py, (file_path, kind), (text, value)), - Err(failure) => failure_object(py, failure), - } -} - /// Whether native discovery reproduces the string semantics of this Python and Unicode version. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_document, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_test_contents, module)?)?; - module.add_function(wrap_pyfunction!(parse_scenario_contents, module)?)?; module.add_class::()?; module.add_class::()?; - module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; - module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs index 7960161131..82bb91c3cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -1,59 +1,74 @@ //! Cross-file validation of one discovery's inputs. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; use crate::bindings::_helpers::discovery::project_facts::{NodeRow, project_facts}; -use crate::bindings::_helpers::discovery::project_files::{ - PyObject, failure_object, object, read_object, tuple_object, -}; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error}; -const CSV_ERROR_TAG: &str = "csv"; -const CSV_DIALECT_TAG: &str = "csv_dialect"; -const UNREADABLE_TAG: &str = "unreadable"; +const CSV_MODULE: &str = "csv"; +const BUILTINS_MODULE: &str = "builtins"; +const PATHLIB_MODULE: &str = "pathlib"; +/// The exceptions a CSV dialect may raise, by class name. +const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; -/// `None` when the inputs are consistent, else the payload of the first error Python raises. +fn raised(result: PyResult>) -> PyErr { + match result { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Raise the first cross-file conflict across the discovered inputs, as Python raises it. #[pyfunction] -fn discovered_inputs_failure( +fn validate_discovered_project_inputs( py: Python<'_>, inputs: Bound<'_, PyAny>, nodes: Vec, -) -> PyResult> { +) -> Result<(), PyErr> { let facts: ProjectFacts = project_facts(py, &inputs, nodes)?; - let failure: ValidationFailure = match validate_project_inputs(&facts) { - Ok(()) => return Ok(None), - Err(failure) => failure, + let Err(failure) = validate_project_inputs(&facts) else { + return Ok(()); }; - let payload: PyObject = match failure { - ValidationFailure::Discovery(failure) => failure_object(py, failure)?, + Err(match failure { + ValidationFailure::Discovery(failure) => failure_error(py, failure), ValidationFailure::Unreadable { absolute_path, failure, - } => tuple_object( - py, - vec![ - object(py, UNREADABLE_TAG)?, - object(py, absolute_path)?, - read_object(py, failure)?, - ], - )?, - ValidationFailure::Csv(message) => { - tuple_object(py, vec![object(py, CSV_ERROR_TAG)?, object(py, message)?])? + } => match py + .import(PATHLIB_MODULE) + .and_then(|module| module.getattr("Path")) + .and_then(|path| path.call1((absolute_path,))) + { + Ok(file_path) => read_error(failure, &file_path), + Err(error) => error, + }, + ValidationFailure::Csv(message) => raised( + py.import(CSV_MODULE) + .and_then(|module| module.getattr("Error")) + .and_then(|class| class.call1((message,))), + ), + ValidationFailure::CsvDialect(error) => { + if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { + return Err(PyKeyError::new_err(error.class.to_string())); + } + raised( + py.import(BUILTINS_MODULE) + .and_then(|module| module.getattr(error.class.as_str())) + .and_then(|class| class.call1((error.message,))), + ) } - ValidationFailure::CsvDialect(error) => tuple_object( - py, - vec![ - object(py, CSV_DIALECT_TAG)?, - object(py, error.class)?, - object(py, error.message)?, - ], - )?, - }; - Ok(Some(payload)) + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discovered_inputs_failure, module)?) + module.add_function(wrap_pyfunction!( + validate_discovered_project_inputs, + module + )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs new file mode 100644 index 0000000000..06b454c4da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -0,0 +1,163 @@ +//! Native discovery failures as the Python exceptions discovery raises for them. + +use pyo3::exceptions::{PyKeyError, PyRuntimeError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::models::{DiscoveryFailure, ReadFailure, StageFailure}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::discovery::project_files::path_object; + +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const BUILTINS_MODULE: &str = "builtins"; +const OS_MODULE: &str = "os"; +const DECODE_ENCODING: &str = "utf-8"; + +/// The exception class of each native failure kind. +const FAILURE_CLASSES: [(&str, &str, &str); 12] = [ + ("model_sql", DISCOVERY_ERRORS_MODULE, "ModelSqlParseError"), + ( + "declaration", + DISCOVERY_ERRORS_MODULE, + "DeclarationParseError", + ), + ("sql_test", DISCOVERY_ERRORS_MODULE, "SqlTestParseError"), + ( + "sql_scenario", + DISCOVERY_ERRORS_MODULE, + "SqlScenarioParseError", + ), + ("schema", DISCOVERY_ERRORS_MODULE, "SchemaParseError"), + ("source", DISCOVERY_ERRORS_MODULE, "SourceParseError"), + ("project_path", DISCOVERY_ERRORS_MODULE, "ProjectPathError"), + ("sql_hook", DISCOVERY_ERRORS_MODULE, "SqlHookParseError"), + ("sql_audit", DISCOVERY_ERRORS_MODULE, "SqlAuditParseError"), + ( + "resource_identity", + IDENTITY_ERRORS_MODULE, + "ResourceIdentityError", + ), + ( + "conflict", + DISCOVERY_ERRORS_MODULE, + "DiscoveryConflictError", + ), + ("seed", DISCOVERY_ERRORS_MODULE, "SeedDiscoveryError"), +]; + +fn raised(result: PyResult>) -> PyErr { + match result { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `error_class(message, help=help)` for the failure kind `kind`. +pub(crate) fn discovery_error( + py: Python<'_>, + kind: &str, + message: String, + help: Option, +) -> PyErr { + let Some((_, module, class)) = FAILURE_CLASSES.iter().find(|(name, _, _)| *name == kind) else { + return PyKeyError::new_err(kind.to_owned()); + }; + raised((|| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + py.import(*module)? + .getattr(*class)? + .call((message,), Some(&keywords)) + })()) +} + +/// Python's `native_failure`: the discovery error a native failure describes. +pub(crate) fn failure_error(py: Python<'_>, failure: DiscoveryFailure) -> PyErr { + discovery_error(py, failure.kind.as_str(), failure.message, failure.help) +} + +/// Python's `native_read_error`: what `read_text` raises for a file native reading could not read. +pub(crate) fn read_error(failure: ReadFailure, file_path: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = file_path.py(); + raised((|| { + let builtins = py.import(BUILTINS_MODULE)?; + match failure { + ReadFailure::Decode { + bytes, + start, + end, + reason, + } => builtins.getattr("UnicodeDecodeError")?.call1(( + DECODE_ENCODING, + PyBytes::new(py, &bytes), + start, + end, + reason, + )), + ReadFailure::Io { + errno, + winerror, + message, + } => { + let os_error = builtins.getattr("OSError")?; + let path: String = file_path.str()?.to_string(); + match (winerror, errno) { + (Some(winerror), _) => os_error.call1((py.None(), message, path, winerror)), + (None, None) => os_error.call1((message,)), + (None, Some(errno)) => { + let reason = py.import(OS_MODULE)?.getattr("strerror")?.call1((errno,))?; + os_error.call1((errno, reason, path)) + } + } + } + } + })()) +} + +/// Python's `native_collection` failure: an unlistable directory's read error, else the bug. +pub(crate) fn stage_error( + tree: &ProjectTree, + project_dir: &Bound<'_, PyAny>, + failure: StageFailure, +) -> PyErr { + match failure { + StageFailure::Unlistable { + relative_path, + error, + } => match path_object(project_dir.py(), tree, relative_path) + .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) + { + Ok(directory) => read_error(error, &directory), + Err(error) => error, + }, + StageFailure::Internal(reason) => PyRuntimeError::new_err(reason), + } +} + +/// The error `read_error` builds for an operating-system read failure, for the platform oracle. +#[pyfunction] +#[pyo3(signature = (errno, winerror, message, *, file_path))] +fn _oracle_read_error<'py>( + errno: Option, + winerror: Option, + message: String, + file_path: &Bound<'py, PyAny>, +) -> Bound<'py, PyAny> { + let failure = ReadFailure::Io { + errno, + winerror, + message, + }; + read_error(failure, file_path) + .into_value(file_path.py()) + .into_bound(file_path.py()) + .into_any() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 3cc8e6b882..248a1c5234 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -3,15 +3,21 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ - declaration_files, globbing, header_declarations, key_checks, locations, project_files, - project_validation, sql_test_blocks, yaml_entries, + declaration_files, declaration_records, globbing, header_declarations, key_checks, + layout_groups, locations, model_records, project_files, project_validation, raised, + sql_test_records, yaml_entries, yaml_records, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project_files::register(module)?; declaration_files::register(module)?; + declaration_records::register(module)?; + layout_groups::register(module)?; + model_records::register(module)?; + sql_test_records::register(module)?; + yaml_records::register(module)?; + raised::register(module)?; locations::register(module)?; - sql_test_blocks::register(module)?; header_declarations::register(module)?; key_checks::register(module)?; globbing::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs new file mode 100644 index 0000000000..77019fa064 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs @@ -0,0 +1,154 @@ +//! The Python directory snapshot one discovery pass shares, and the native tree it memoises. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_discovery::models::ProjectRoot; +use sqlbuild_discovery::tree::models::ProjectTree; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::raised::discovery_error; +use crate::bindings::models::NativeProjectTree; + +const SNAPSHOT_MODULE: &str = "sqlbuild.compiler.discovery.classes.directory_snapshot"; +const SNAPSHOT_CLASS: &str = "DirectorySnapshot"; +const DISCOVERY_TYPES_MODULE: &str = "sqlbuild.compiler.discovery.types"; +const SNAPSHOT_ENTRY_CLASS: &str = "DirectorySnapshotEntry"; +const TREE_MEMO_KEY: &str = "native_project_tree"; +const DISPLAY_PROBE: &str = "_"; +const PROJECT_PATH_KIND: &str = "project_path"; +const SURROGATE_ESCAPE_BASE: u32 = 0xDC00; +const ESCAPED_BYTE_FIRST: u16 = 0xDC80; +const ESCAPED_BYTE_LAST: u16 = 0xDCFF; +const UTF16_ENCODING: &str = "utf-16-le"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// Run `body` inside `DirectorySnapshot.scope(project_dir=project_dir)`. +pub(crate) fn in_snapshot<'py, T>( + project_dir: &Bound<'py, PyAny>, + body: impl FnOnce() -> PyResult, +) -> PyResult { + let py: Python<'py> = project_dir.py(); + let scope: Bound<'py, PyAny> = call_with( + &imported(py, SNAPSHOT_MODULE, SNAPSHOT_CLASS)?.getattr("scope")?, + &[("project_dir", project_dir.clone())], + )?; + within(&scope, |_| body()) +} + +/// `DirectorySnapshot.current(project_dir=project_dir).memo`. +pub(crate) fn snapshot_memo<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + current_snapshot(project_dir)? + .getattr("memo")? + .downcast_into::() + .map_err(Into::into) +} + +fn current_snapshot<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(project_dir.py(), SNAPSHOT_MODULE, SNAPSHOT_CLASS)?.getattr("current")?, + &[("project_dir", project_dir.clone())], + ) +} + +/// `str(project_dir / "_")` without its probe: the text printed before a relative path. +pub(crate) fn display_prefix(project_dir: &Bound<'_, PyAny>) -> PyResult { + let probed: String = project_dir.div(DISPLAY_PROBE)?.str()?.to_str()?.to_owned(); + Ok(probed[..probed.len() - DISPLAY_PROBE.len()].to_owned()) +} + +/// The project root native discovery reads, named as Python prints it. +pub(crate) fn project_root(project_dir: &Bound<'_, PyAny>) -> PyResult { + Ok(ProjectRoot { + directory: PathBuf::from(project_dir.str()?.to_str()?), + display_prefix: display_prefix(project_dir)?, + }) +} + +/// The native listings shared by every native call of the pass, created once per snapshot. +pub(crate) fn project_tree<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let memo: Bound<'py, PyDict> = snapshot_memo(project_dir)?; + if let Some(tree) = memo.get_item(TREE_MEMO_KEY)? + && let Ok(tree) = tree.downcast_into::() + { + return Ok(tree); + } + let directory: Bound<'py, PyString> = project_dir.str()?; + let prefixed: Bound<'py, PyString> = project_dir.div(DISPLAY_PROBE)?.str()?; + for text in [&directory, &prefixed] { + if text.to_str().is_err() { + let mut shown: String = escaped(text)?; + if text.is(&prefixed) { + shown.truncate(shown.len() - DISPLAY_PROBE.len()); + } + return Err(discovery_error( + py, + PROJECT_PATH_KIND, + format!( + "Project directory {shown} is not valid UTF-8; move the project to a \ + directory whose path is valid UTF-8" + ), + None, + )); + } + } + let created: Bound<'py, NativeProjectTree> = Bound::new( + py, + NativeProjectTree { + inner: ProjectTree::new(std::path::Path::new(directory.to_str()?)), + exported: Mutex::new(HashSet::new()), + }, + )?; + memo.set_item(TREE_MEMO_KEY, &created)?; + Ok(created) +} + +/// Share the native walk's new listings with the pass's Python directory snapshot. +pub(crate) fn seed_listings( + project_dir: &Bound<'_, PyAny>, + tree: &Bound<'_, NativeProjectTree>, +) -> PyResult<()> { + let py: Python<'_> = project_dir.py(); + let entry_class: Bound<'_, PyAny> = imported(py, DISCOVERY_TYPES_MODULE, SNAPSHOT_ENTRY_CLASS)?; + let listings: Bound<'_, PyDict> = PyDict::new(py); + for (directory, entries) in tree.get().exported_listings(py)? { + let entries: Vec> = entries + .into_iter() + .map(|(name, is_dir, walkable)| entry_class.call1((name, is_dir, walkable))) + .collect::>()?; + listings.set_item( + project_dir.div(directory.bind(py))?, + PyTuple::new(py, entries)?, + )?; + } + current_snapshot(project_dir)?.call_method1("seed_listings", (listings,))?; + Ok(()) +} + +/// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. +pub(crate) fn escaped(text: &Bound<'_, PyString>) -> PyResult { + let encoded: Vec = text + .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? + .extract()?; + let units = encoded + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])); + Ok(char::decode_utf16(units) + .map(|decoded| match decoded { + Ok(character) => character.to_string(), + Err(error) => escaped_unit(error.unpaired_surrogate()), + }) + .collect()) +} + +fn escaped_unit(unit: u16) -> String { + if cfg!(unix) && (ESCAPED_BYTE_FIRST..=ESCAPED_BYTE_LAST).contains(&unit) { + return format!("\\x{:02x}", u32::from(unit) - SURROGATE_ESCAPE_BASE); + } + format!("\\u{unit:04x}") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs new file mode 100644 index 0000000000..a32dc24efb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs @@ -0,0 +1,19 @@ +//! Header key sets as the sorted lists native discovery options take. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyFrozenSetMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; + +/// `sorted(constants.)`: a header key set, sorted by code point as Python sorts it. +pub(crate) fn sorted_keys(py: Python<'_>, name: &str) -> PyResult> { + let keys: Bound<'_, PyFrozenSet> = imported(py, CONSTANTS_MODULE, name)?.downcast_into()?; + let mut keys: Vec = keys + .iter() + .map(|key| key.extract()) + .collect::>()?; + keys.sort(); + Ok(keys) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 6a128ec24b..5ef774ed56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -1,10 +1,8 @@ //! SQL `TEST(...)` blocks validated natively and built as discovery records. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::sql_tests::main::sql_test_block::sql_test_block; use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_names_failure; use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; @@ -17,7 +15,7 @@ const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; -type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); +pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); fn parse_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( @@ -101,8 +99,7 @@ fn block_record<'py>( } /// Each `(header, sql_body, span)` block of a SQL test file as a `DiscoveredSqlTestBlock`. -#[pyfunction] -fn sql_test_blocks<'py>( +pub(crate) fn sql_test_block_records<'py>( py: Python<'py>, file_path: &str, blocks: Vec>, @@ -120,19 +117,13 @@ fn sql_test_blocks<'py>( } /// `validate_sql_test_names` over each block's `(test_index, name)`. -#[pyfunction] -fn validate_sql_test_block_names( +pub(crate) fn validate_block_names( py: Python<'_>, file_path: &str, - blocks: Vec<(usize, Option)>, + blocks: &[(usize, Option)], ) -> Result<(), PyErr> { - match sql_test_names_failure(file_path, &blocks) { + match sql_test_names_failure(file_path, blocks) { None => Ok(()), Some(message) => Err(parse_error(py, message)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_blocks, module)?)?; - module.add_function(wrap_pyfunction!(validate_sql_test_block_names, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs new file mode 100644 index 0000000000..16002c4b6e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -0,0 +1,317 @@ +//! SQL unit-test and scenario files discovered natively and built as Python discovery records. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, StageFailure}; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; +use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; +use sqlbuild_discovery::sql_tests::models::{ + DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, +}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; +use crate::bindings::_helpers::discovery::snapshot::{ + in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; +use crate::bindings::_helpers::discovery::sql_test_blocks::{ + sql_test_block_records, validate_block_names, +}; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::models::NativeProjectTree; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const PATHLIB_MODULE: &str = "pathlib"; +const SCENARIO_KIND: &str = "sql_scenario"; +const DESCRIPTION_KEY: &str = "description"; +const TAGS_KEY: &str = "tags"; + +/// A native discoverer of one kind of SQL test file. +type Discoverer = fn( + &sqlbuild_discovery::models::ProjectRoot, + &ProjectTree, + &SqlTestFileOptions, +) -> Result>, StageFailure>; + +/// Build one parsed file's record from its `(file_path, relative_path)`. +type RecordBuilder<'py, T> = + fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; + +fn options(py: Python<'_>) -> PyResult { + Ok(SqlTestFileOptions { + test_keys: sorted_keys(py, "SQL_TEST_HEADER_KEYS")?, + scenario_keys: sorted_keys(py, "SQL_SCENARIO_HEADER_KEYS")?, + }) +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +/// `Path(constants.)`. +fn ownership_root<'py>(py: Python<'py>, name: &str) -> PyResult> { + path_class(py)?.call1((imported(py, CONSTANTS_MODULE, name)?,)) +} + +/// Discover the files natively in the pass's snapshot: each relative path text and outcome. +fn discovered_files<'py, T: Send>( + project_dir: &Bound<'py, PyAny>, + discover: Discoverer, +) -> PyResult<(Bound<'py, NativeProjectTree>, Vec>)> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root = project_root(project_dir)?; + let options: SqlTestFileOptions = options(py)?; + let discovered: Result>, StageFailure> = { + let inner = &tree.get().inner; + py.detach(|| discover(&root, inner, &options)) + }; + let files = + discovered.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + seed_listings(project_dir, &tree)?; + Ok((tree, files)) + }) +} + +/// Each block as a `DiscoveredSqlTestBlock`, then the split failure, then the name check. +fn test_blocks<'py>( + file_path: &Bound<'py, PyAny>, + blocks: Vec, + failure: Option, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let mut native = Vec::with_capacity(blocks.len()); + for block in blocks { + native.push(( + map_to_python(py, block.header_values)? + .into_bound(py) + .downcast_into::()?, + block.sql_body, + block.body_span.into_bound_py_any(py)?, + )); + } + let records: Vec> = sql_test_block_records(py, &path, native)?; + if let Some(failure) = failure { + return Err(failure_error(py, failure)); + } + let names: Vec<(usize, Option)> = records + .iter() + .map(|block| { + Ok(( + block.getattr("test_index")?.extract()?, + block.getattr("name")?.extract()?, + )) + }) + .collect::>()?; + validate_block_names(py, &path, &names)?; + PyTuple::new(py, records) +} + +fn test_file<'py>( + paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + file: DiscoveredSqlTestFile, +) -> PyResult> { + let (file_path, relative_path) = paths; + let py: Python<'py> = file_path.py(); + call_with( + &imported(py, MODELS_MODULE, "DiscoveredSqlTestFile")?, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", file.contents.into_bound_py_any(py)?), + ( + "blocks", + test_blocks(file_path, file.blocks, file.failure)?.into_any(), + ), + ( + "ownership_root", + ownership_root(py, "SQL_TESTS_OWNERSHIP_ROOT")?, + ), + ], + ) +} + +/// Python's `build_sql_scenario_file`: validate the header and build the record. +fn scenario_file<'py>( + paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + file: DiscoveredScenarioFile, +) -> PyResult> { + let (file_path, relative_path) = paths; + let py: Python<'py> = file_path.py(); + let header_values: Bound<'py, PyDict> = map_to_python(py, file.header_values)? + .into_bound(py) + .downcast_into::()?; + let shown: Bound<'py, PyString> = file_path.str()?; + let failure = |message: String| Err(discovery_error(py, SCENARIO_KIND, message, None)); + if let Some(description) = header_values.get_item(DESCRIPTION_KEY)? + && !description.is_instance_of::() + { + return failure(format!( + "SCENARIO() description in '{shown}' must be a string" + )); + } + if let Some(tags) = header_values.get_item(TAGS_KEY)? { + let tags_are_strings: bool = match tags.downcast::() { + Ok(tags) => tags.iter().all(|tag| tag.is_instance_of::()), + Err(_) => false, + }; + if !tags_are_strings { + return failure(format!( + "SCENARIO() tags in '{shown}' must be a list of strings" + )); + } + } + if file.sql_body.is_empty() { + return failure(format!( + "SQL scenario '{shown}' must define SQL after SCENARIO(...)" + )); + } + call_with( + &imported(py, MODELS_MODULE, "DiscoveredSqlScenarioFile")?, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", file.contents.into_bound_py_any(py)?), + ("header_values", header_values.into_any()), + ("sql_body", file.sql_body.into_bound_py_any(py)?), + ("sql_body_span", file.body_span.into_bound_py_any(py)?), + ("name", file_path.getattr("stem")?), + ( + "ownership_root", + ownership_root(py, "SQL_SCENARIOS_OWNERSHIP_ROOT")?, + ), + ], + ) +} + +/// Build each file, raising or reporting each failing one. +fn records<'py, T>( + faults: &FileFaults<'py>, + files: Vec<(Bound<'py, PyAny>, FileOutcome)>, + build: RecordBuilder<'py, T>, +) -> PyResult> { + let py: Python<'py> = faults.project_dir.py(); + let mut built: Vec> = Vec::with_capacity(files.len()); + for (relative_path, outcome) in files { + let file_path: Bound<'py, PyAny> = faults.project_dir.div(&relative_path)?; + let record: PyResult> = match outcome { + FileOutcome::Parsed(file) => build((&file_path, &relative_path), file), + FileOutcome::Failed(failure) => Err(failure_error(py, failure)), + FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), + }; + built.extend(faults.kept(&relative_path, record)?); + } + record_tuple(py, built) +} + +/// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] +pub(crate) fn discover_test_files<'py>( + project_dir: &Bound<'py, PyAny>, + selected_paths: Option>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_tests)?; + let tree: &ProjectTree = &tree.get().inner; + let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); + for file in files { + let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); + if let Some(selected) = &selected_paths + && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? + { + continue; + } + kept.push((path_class(py)?.call1((relative,))?, file.outcome)); + } + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + test_file, + ) +} + +/// Discover scenario files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_scenario_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_scenarios)?; + let tree: &ProjectTree = &tree.get().inner; + let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files + .into_iter() + .map(|file| { + Ok(( + path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, + file.outcome, + )) + }) + .collect::>()?; + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + scenario_file, + ) +} + +/// Parse in-memory SQL test contents into ordered blocks, as test discovery does. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path))] +pub(crate) fn parse_sql_test_file<'py>( + contents: String, + file_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options(py)?; + let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); + let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; + test_blocks(file_path, file.blocks, file.failure) +} + +/// Parse in-memory scenario contents into the record scenario discovery builds for a file. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path, relative_path))] +pub(crate) fn parse_sql_scenario_file<'py>( + contents: String, + file_path: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options(py)?; + let parsed = py.detach(|| parse_scenario_text(&path, contents, &options)); + let file: DiscoveredScenarioFile = parsed.map_err(|failure| failure_error(py, failure))?; + scenario_file((file_path, relative_path), file) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 8418ddb2e6..3d3638e69d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -27,7 +27,7 @@ const SOURCE_PARSE_ERROR: &str = "SourceParseError"; /// The Python classes parsed YAML entries become. #[derive(FromPyObject)] #[pyo3(from_item_all)] -struct YamlClasses<'py> { +pub(crate) struct YamlClasses<'py> { contracts: ContractClasses<'py>, schema_seed_entry: Bound<'py, PyAny>, seed_csv_settings: Bound<'py, PyAny>, @@ -157,7 +157,7 @@ fn seed_entry<'py>( /// The seed entries of one loaded schema YAML document, or the parse error Python raises. #[pyfunction] -fn parse_schema_document<'py>( +pub(crate) fn parse_schema_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, @@ -339,7 +339,7 @@ fn source_entry<'py>( /// The source entries of one loaded sources YAML document, or the parse error Python raises. #[pyfunction] -fn parse_sources_document<'py>( +pub(crate) fn parse_sources_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs new file mode 100644 index 0000000000..cc729fb70c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -0,0 +1,382 @@ +//! Source and schema YAML files discovered natively and built as Python discovery records. + +use pyo3::exceptions::PyUnicodeEncodeError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyList, PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind, ProjectRoot, StageFailure}; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; +use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; +use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; +use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; +use std::path::PathBuf; + +use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; +use crate::bindings::_helpers::discovery::snapshot::{ + escaped, in_snapshot, project_root, project_tree, seed_listings, +}; +use crate::bindings::_helpers::discovery::yaml_entries::{ + YamlClasses, parse_schema_document, parse_sources_document, +}; +use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +/// The most YAML bytes one native load reads before its files are built and released. +const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; +const YAML_CLASSES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.yaml_classes"; +const PATHLIB_MODULE: &str = "pathlib"; +const PROJECT_PATH_KIND: &str = "project_path"; +const SCHEMA_KIND: &str = "schema"; + +/// The YAML files one discovery reads, and the kind their failures raise as. +#[derive(Clone, Copy)] +enum YamlKind { + Source, + Schema, +} + +impl YamlKind { + fn failure_kind(self) -> FailureKind { + match self { + Self::Source => FailureKind::Source, + Self::Schema => FailureKind::Schema, + } + } +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, PATHLIB_MODULE, "Path") +} + +/// `(contents, loaded)` of a loaded file; a value Python cannot build fails like invalid YAML. +fn loaded<'py>( + py: Python<'py>, + (display_path, kind): (&str, FailureKind), + (contents, value): (String, ConfigValue), +) -> PyResult<(String, Bound<'py, PyAny>)> { + match config_value_to_python(py, value) { + Ok(loaded) => Ok((contents, loaded.into_bound(py))), + Err(error) => Err(failure_error( + py, + DiscoveryFailure::new( + kind, + format!("{display_path} contains invalid YAML: {}", error.value(py)), + ), + )), + } +} + +/// One file's `(contents, loaded)`, or the error Python raises for it. +fn outcome_loaded<'py>( + root: &ProjectRoot, + (relative_path, kind): (&str, FailureKind), + outcome: YamlFileOutcome, + file_path: &Bound<'py, PyAny>, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let py: Python<'py> = file_path.py(); + match outcome { + YamlFileOutcome::Loaded { contents, value } => loaded( + py, + (&root.display_path(relative_path), kind), + (contents, value), + ), + YamlFileOutcome::Failed(failure) => Err(failure_error(py, failure)), + YamlFileOutcome::Unreadable(failure) => Err(read_error(failure, file_path)), + } +} + +/// The pass's tree, or a private one for an undecodable project directory (fails on first read). +fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + match project_dir.str()?.to_str() { + Ok(_) => Ok(ListingTree::Shared(project_tree(project_dir)?)), + Err(error) if error.is_instance_of::(project_dir.py()) => { + let directory: PathBuf = project_dir.extract()?; + Ok(ListingTree::Private(ProjectTree::new(&directory))) + } + Err(error) => Err(error), + } +} + +enum ListingTree<'py> { + Shared(Bound<'py, NativeProjectTree>), + Private(ProjectTree), +} + +impl ListingTree<'_> { + fn tree(&self) -> &ProjectTree { + match self { + Self::Shared(tree) => &tree.get().inner, + Self::Private(tree) => tree, + } + } +} + +/// Split the files into batches bounded by size, as Python's `_batches` does. +fn batches(tree: &ProjectTree, paths: Vec, limit: u64) -> Vec> { + let mut batches: Vec> = Vec::new(); + let mut batch: Vec = Vec::new(); + let mut batch_bytes: u64 = 0; + for path in paths { + let size: u64 = + std::fs::metadata(tree.absolute(&path)).map_or(0, |metadata| metadata.len()); + if !batch.is_empty() && batch_bytes + size > limit { + batches.push(std::mem::take(&mut batch)); + batch_bytes = 0; + } + batch.push(path); + batch_bytes += size; + } + if !batch.is_empty() { + batches.push(batch); + } + batches +} + +/// Pass each file's paths and `(contents, loaded)` or error to `on_file`, in size-bounded batches. +fn load_files<'py>( + project_dir: &Bound<'py, PyAny>, + (paths, kind): (Vec, YamlKind), + mut on_file: impl FnMut( + (Bound<'py, PyAny>, Bound<'py, PyAny>), + PyResult<(String, Bound<'py, PyAny>)>, + ) -> PyResult<()>, +) -> PyResult<()> { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root: ProjectRoot = project_root(project_dir)?; + for batch in batches(&tree.get().inner, paths, YAML_BATCH_BYTES) { + let named: Vec = batch + .iter() + .filter(|path| !tree.get().inner.is_undecodable(path)) + .cloned() + .collect(); + let outcomes: Result, StageFailure> = { + let inner = &tree.get().inner; + let failure_kind: FailureKind = kind.failure_kind(); + py.detach(|| load_yaml(&root, inner, &named, failure_kind)) + }; + let mut outcomes = outcomes + .map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))? + .into_iter(); + seed_listings(project_dir, &tree)?; + for path in batch { + let relative: Bound<'py, PyAny> = + path_object(py, &tree.get().inner, path.clone())?.into_bound(py); + let relative_path: Bound<'py, PyAny> = path_class(py)?.call1((&relative,))?; + let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; + let built: PyResult<(String, Bound<'py, PyAny>)> = + if tree.get().inner.is_undecodable(&path) { + Err(unnamed_error(project_dir, &relative)?) + } else { + match outcomes.next() { + Some(outcome) => { + outcome_loaded(&root, (&path, kind.failure_kind()), outcome, &file_path) + } + None => Err(pyo3::exceptions::PyRuntimeError::new_err( + "native YAML loading returned too few files", + )), + } + }; + on_file((relative_path, file_path), built)?; + } + } + Ok(()) +} + +/// The `ProjectPathError` of a file whose name native discovery cannot read. +fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> PyResult { + let shown: String = escaped(&project_dir.div(relative)?.str()?)?; + Ok(discovery_error( + project_dir.py(), + PROJECT_PATH_KIND, + format!("Project path {shown} is not valid UTF-8; rename it so SQLBuild can read it"), + None, + )) +} + +fn yaml_classes(py: Python<'_>) -> PyResult> { + imported(py, YAML_CLASSES_MODULE, "yaml_entry_classes")? + .call0()? + .extract() +} + +/// Discover source YAML files natively, reporting failing files to `on_fault`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] +pub(crate) fn discover_source_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = source_yaml_paths(listing.tree()) + .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault, + }; + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSourceFile")?; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Source), + |(relative_path, file_path), built| { + let record = built.and_then(|(contents, loaded)| { + let shown: String = file_path.str()?.to_str()?.to_owned(); + call_with( + &class, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", contents.into_bound_py_any(py)?), + ( + "source_entries", + parse_sources_document(py, loaded, &shown, yaml_classes(py)?)? + .into_any(), + ), + ], + ) + }); + records.extend(faults.kept(&relative_path, record)?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) +} + +/// Discover model schema.yml and seed declaration files natively. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = match schema_yaml_paths(listing.tree()) { + Ok(paths) => paths, + Err(YamlPathFailure::Stage(failure)) => { + return Err(stage_error(listing.tree(), project_dir, failure)); + } + Err(YamlPathFailure::YamlSeedDeclaration(path)) => { + let relative: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; + return Err(discovery_error( + py, + SCHEMA_KIND, + format!( + "Seed declaration file {} must use .yml; .yaml is not supported", + relative.str()? + ), + None, + )); + } + }; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSchemaFile")?; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Schema), + |(relative_path, file_path), built| { + let (contents, loaded) = built?; + let shown: String = file_path.str()?.to_str()?.to_owned(); + records.push(call_with( + &class, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("contents", contents.into_bound_py_any(py)?), + ("model_entries", PyTuple::empty(py).into_any()), + ( + "seed_entries", + parse_schema_document(py, loaded, &shown, yaml_classes(py)?)? + .into_any(), + ), + ], + )?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) +} + +fn yaml_kind(kind: &str) -> PyResult { + match kind { + "source" => Ok(FailureKind::Source), + "schema" => Ok(FailureKind::Schema), + other => Err(pyo3::exceptions::PyValueError::new_err(format!( + "unknown YAML file kind {other:?}" + ))), + } +} + +/// Read and load YAML files: each file's `(contents, loaded)`, or the error it raises. +#[pyfunction] +fn load_yaml_files<'py>( + py: Python<'py>, + request: YamlDiscoveryRequest, + relative_paths: Vec, + tree: &NativeProjectTree, +) -> PyResult> { + let kind: FailureKind = yaml_kind(&request.kind)?; + let root = ProjectRoot { + directory: PathBuf::from(&request.project_dir), + display_prefix: request.display_prefix, + }; + let project_dir: Bound<'py, PyAny> = path_class(py)?.call1((request.project_dir.as_str(),))?; + let outcomes: Vec = py + .detach(|| load_yaml(&root, &tree.inner, &relative_paths, kind)) + .map_err(|failure| stage_error(&tree.inner, &project_dir, failure))?; + let mut rows: Vec> = Vec::with_capacity(outcomes.len()); + for (outcome, relative_path) in outcomes.into_iter().zip(&relative_paths) { + let file_path: Bound<'py, PyAny> = project_dir.div(relative_path)?; + rows.push( + match outcome_loaded(&root, (relative_path, kind), outcome, &file_path) { + Ok(loaded) => loaded.into_bound_py_any(py)?, + Err(error) => error.into_value(py).into_any().into_bound(py), + }, + ); + } + PyList::new(py, rows) +} + +/// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. +#[pyfunction] +fn load_yaml_document<'py>( + py: Python<'py>, + file_path: &str, + text: String, + kind: &str, +) -> PyResult> { + let kind: FailureKind = yaml_kind(kind)?; + let value: ConfigValue = + load_yaml_text(file_path, &text, kind).map_err(|failure| failure_error(py, failure))?; + Ok(loaded(py, (file_path, kind), (text, value))?.1) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_document, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index d0b0304e8d..4a3925a674 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -98,37 +98,6 @@ pub(crate) struct NativeProjectTree { pub(crate) exported: Mutex>, } -/// One native model discovery request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct ModelDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) supported_keys: Vec, - pub(crate) removed_keys: Vec, - pub(crate) extract_implicit_alias_columns: bool, - pub(crate) extract_output_column_locations: bool, -} - -/// One native SQL test or scenario discovery request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) test_keys: Vec, - pub(crate) scenario_keys: Vec, -} - -/// One in-memory SQL test or scenario parse request; `file_path` is the path messages name. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestTextRequest { - pub(crate) file_path: String, - pub(crate) test_keys: Vec, - pub(crate) scenario_keys: Vec, -} - /// One native YAML file discovery request; `kind` is `source` or `schema`. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] @@ -156,17 +125,6 @@ pub(crate) struct NativeDiscoverySession { pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, } -/// One native declaration discovery session request from the Python discovery facade. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct DeclarationDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) function_keys: Vec, - pub(crate) audit_keys: Vec, - pub(crate) hook_keys: Vec, -} - /// One in-memory declaration file parse request; `kind` names the collection it belongs to. #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] diff --git a/python-boundary.toml b/python-boundary.toml index 3f5de15324..fc6b5173d8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -204,6 +204,14 @@ path = "tasks.py" reason = "public-api" note = "Public facade." +[[handled_native_errors]] +location = "compiler/discovery/_helpers/filesystem/aggregation.py::_discover_declarations" +note = "Tolerant scope discovery records a failing declaration category as a fault; no Python implementation runs." + +[[handled_native_errors]] +location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" +note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." + [[handled_native_errors]] location = "compiler/macro_bridge/_helpers/store_keys.py::macro_store_token" note = "A macro without a stable key is not stored; no Python implementation runs." @@ -239,3 +247,11 @@ note = "A baseline that does not compile yields no relation keys, as before the [[handled_native_errors]] location = "lint/_helpers/fixes.py::_FixSession._verdict" note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." + +[[handled_native_errors]] +location = "compiler/discovery/_helpers/filesystem/aggregation.py::_discover_declarations" +note = "Tolerant scope discovery records a failing declaration category as a fault; no Python implementation runs." + +[[handled_native_errors]] +location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" +note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d216eca47c..8f034a0efd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,6 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence, Set from pathlib import Path, PurePath from typing import Any, Literal, TypedDict, overload @@ -53,11 +53,16 @@ from sqlbuild.compiler.compile.types import ( ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, + DiscoveredAdapterFile, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredConstantFile, + DiscoveredEnumFile, DiscoveredHookFunction, DiscoveredLoaderFunction, + DiscoveredMacroFile, DiscoveredMaterializationFile, + DiscoveredModelSchemaFile, DiscoveredProjectInputs, DiscoveredPythonFunctionFile, DiscoveredSchemaFile, @@ -66,7 +71,9 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, DiscoveredSqlTestFile, + DiscoveryFileFault, EnumDeclaration, ModelSchemaDeclaration, TolerantScopeDiscovery, @@ -899,7 +906,6 @@ def model_header_failure( class NativeProjectTree: def __init__(self, project_dir: str) -> None: ... - def listings(self) -> list[tuple[str, list[tuple[str, bool, bool]]]]: ... class DiscoveredSqlModelFile: """A discovered SQL model file and its raw contents.""" @@ -1076,45 +1082,83 @@ def public_declaration_indexes( def public_model_schema_index( inputs: DiscoveredProjectInputs, ) -> dict[str, ModelSchemaDeclaration]: ... -def discovered_inputs_failure( - inputs: DiscoveredProjectInputs, nodes: list[dict[str, object]] -) -> tuple[object, ...] | None: ... +def validate_discovered_project_inputs( + inputs: DiscoveredProjectInputs, nodes: list[dict[str, object]], / +) -> None: ... def discover_model_files( - request: dict[str, object], tree: NativeProjectTree, project_dir: Path -) -> list[tuple[Path, object]] | tuple[object, ...]: ... -def discover_sql_test_files( - request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... + *, + project_dir: Path, + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, + selected_model_names: Set[str] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, +) -> tuple[DiscoveredSqlModelFile, ...]: ... +def discover_test_files( + *, + project_dir: Path, + selected_paths: Set[Path] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, +) -> tuple[DiscoveredSqlTestFile, ...]: ... def discover_scenario_files( - request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlScenarioFile, ...]: ... +def parse_sql_test_file( + *, contents: str, file_path: Path +) -> tuple[DiscoveredSqlTestBlock, ...]: ... +def parse_sql_scenario_file( + *, contents: str, file_path: Path, relative_path: Path +) -> DiscoveredSqlScenarioFile: ... +def discover_source_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSourceFile, ...]: ... +def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: ... def load_yaml_files( request: dict[str, object], relative_paths: list[str], tree: NativeProjectTree -) -> list[tuple[object, ...]] | tuple[object, ...]: ... -def load_yaml_document(file_path: str, text: str, kind: str) -> tuple[object, ...]: ... - -class NativeDiscoverySession: - def __init__(self, request: dict[str, object]) -> None: ... - def collection( - self, kind: str, tree: NativeProjectTree, isolate_kind: bool = False - ) -> ( - list[ - tuple[ - str, tuple[str, str, str | None, str | None, str | None] | None, tuple[object, ...] - ] - ] - | tuple[object, ...] - ): ... +) -> list[tuple[str, object] | Exception]: ... +def load_yaml_document(file_path: str, text: str, kind: str) -> object: ... +def _oracle_read_error( + errno: int | None, winerror: int | None, message: str, *, file_path: Path +) -> Exception: ... +def discover_enum_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, + isolate_declaration_kind: bool = False, +) -> tuple[DiscoveredEnumFile, ...]: ... +def discover_constant_files( + *, + project_dir: Path, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, + isolate_declaration_kind: bool = False, +) -> tuple[DiscoveredConstantFile, ...]: ... +def discover_model_schema_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredModelSchemaFile, ...]: ... +def discover_sql_function_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlFunctionFile, ...]: ... +def discover_sql_hook_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlHookFile, ...]: ... +def discover_audit_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredAuditFile, ...]: ... +def discover_macro_files( + *, project_dir: Path, isolate_declaration_kind: bool = False +) -> tuple[DiscoveredMacroFile, ...]: ... +def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: ... +def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: ... +def retained_discovery_session(*, project_dir: Path) -> NativeDiscoverySession | None: ... +def declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: ... + +class NativeDiscoverySession: ... def parse_declaration_contents( request: dict[str, object], names: tuple[str, str], contents: str -) -> tuple[object, ...]: ... -def discover_declaration_layout( - tree: NativeProjectTree, kind: str | None = None -) -> tuple[tuple[object, ...], tuple[object, ...]] | tuple[object, ...]: ... -def parse_model_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... -def parse_sql_test_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... -def parse_scenario_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... +) -> str: ... +def parse_model_contents( + request: dict[str, object], contents: str +) -> tuple[dict[str, object], str, dict[str, SourceLocation]]: ... class NativeDeclarationContexts: def __init__( @@ -1235,21 +1279,9 @@ def unresolved_reference_marker_kind( def normalize_typed_sql_value( raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / ) -> object: ... -def sql_test_blocks( - file_path: str, blocks: list[tuple[dict[str, object], str, tuple[int, int]]], / -) -> list[object]: ... -def validate_sql_test_block_names( - file_path: str, blocks: list[tuple[int, str | None]], / -) -> None: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... -def typed_constant_record( - constant: tuple[str, object], - options: tuple[str | None, str | None], - paths: tuple[Path, Path], - /, -) -> object: ... def deferred_target_segments( template: str, logical: str, context_keys: list[str], variables: list[str], / ) -> list[tuple[bool, str]]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index e3bd07712c..72cc814c11 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -5,24 +5,18 @@ from collections.abc import Callable from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, + discover_model_files, discover_model_schema_files, + discover_scenario_files, + discover_source_files, discover_sql_function_files, discover_sql_hook_files, -) -from sqlbuild.compiler.discovery._helpers.native.model_files import ( - discover_native_model_files, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, - discover_native_test_files, -) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_source_files, + discover_test_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot @@ -59,7 +53,7 @@ def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscov function=discover_sql_hook_files, project_dir=project_dir ) sources, source_faults = _discover_category( - function=discover_native_source_files, project_dir=project_dir + function=discover_source_files, project_dir=project_dir ) audits, audit_faults = _discover_category( function=discover_audit_files, project_dir=project_dir @@ -129,7 +123,7 @@ def _discover_models( *, project_dir: Path ) -> tuple[tuple[DiscoveredSqlModelFile, ...], tuple[DiscoveryFileFault, ...]]: faults: list[DiscoveryFileFault] = [] - models: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + models: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, @@ -191,10 +185,10 @@ def _discover_relationships( tuple[DiscoveryFileFault, ...], ]: faults: list[DiscoveryFileFault] = [] - tests: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( + tests: tuple[DiscoveredSqlTestFile, ...] = discover_test_files( project_dir=project_dir, on_fault=faults.append ) - scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_native_scenario_files( + scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_scenario_files( project_dir=project_dir, on_fault=faults.append ) return tests, scenarios, tuple(faults) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 90c3091091..d40bb0f531 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -21,16 +21,6 @@ from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( project_relative_path, ) -from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( - native_audit_files, - native_constant_files, - native_enum_files, - native_macro_files, - native_model_schema_files, - native_seed_files, - native_sql_function_files, - native_sql_hook_files, -) from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( @@ -43,30 +33,20 @@ PythonNodeDiscoveryError, ) from sqlbuild.compiler.discovery.models import ( - DiscoveredAdapterFile, DiscoveredAssetFunction, DiscoveredAuditFactory, - DiscoveredAuditFile, DiscoveredCheckFunction, DiscoveredCommandOutputSink, - DiscoveredConstantFile, - DiscoveredEnumFile, DiscoveredEventExporter, DiscoveredEventExporterDeclaration, DiscoveredHookFunction, DiscoveredLoaderFunction, - DiscoveredMacroFile, DiscoveredMaterializationFile, - DiscoveredModelSchemaFile, DiscoveredProvider, DiscoveredProviderUsage, DiscoveredPythonFunctionFile, DiscoveredPythonNodeFunctions, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, DiscoveredTaskFunction, - DiscoveryFileFault, NamedDeclarationRoot, ) from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( @@ -132,52 +112,6 @@ def add_audit_factory(self, item: DiscoveredAuditFactory) -> None: self.audit_factories.append(item) -def discover_enum_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - isolate_declaration_kind: bool = False, -) -> tuple[DiscoveredEnumFile, ...]: - """Discover global and scoped enum declaration files.""" - - return native_enum_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - on_fault=on_fault, - ) - - -def discover_constant_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - isolate_declaration_kind: bool = False, -) -> tuple[DiscoveredConstantFile, ...]: - """Discover global and scoped constant declaration files.""" - - return native_constant_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - on_fault=on_fault, - ) - - -def discover_model_schema_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredModelSchemaFile, ...]: - """Discover project-wide and scoped reusable model schemas.""" - - return native_model_schema_files(project_dir=project_dir, on_fault=on_fault) - - -def discover_sql_function_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlFunctionFile, ...]: - """Discover SQL function files under functions/sql/.""" - - return native_sql_function_files(project_dir=project_dir, on_fault=on_fault) - - def discover_python_function_files( *, project_dir: Path ) -> tuple[DiscoveredPythonFunctionFile, ...]: @@ -210,30 +144,6 @@ def discover_python_function_files( return tuple(discovered_function_files) -def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: - """Discover seed CSV files under seeds/.""" - - return native_seed_files(project_dir=project_dir) - - -def discover_audit_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredAuditFile, ...]: - """Discover generic and singular audit SQL files in their project-wide and scoped roles.""" - - return native_audit_files(project_dir=project_dir, on_fault=on_fault) - - -def discover_macro_files( - *, project_dir: Path, isolate_declaration_kind: bool = False -) -> tuple[DiscoveredMacroFile, ...]: - """Discover global and scoped project macro Python files.""" - - return native_macro_files( - project_dir=project_dir, isolate_declaration_kind=isolate_declaration_kind - ) - - def discover_materialization_files( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> tuple[DiscoveredMaterializationFile, ...]: @@ -344,14 +254,6 @@ def discover_hook_functions( return tuple(discovered_hooks) -def discover_sql_hook_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlHookFile, ...]: - """Discover named SQL lifecycle hooks in their project-wide and scoped roles.""" - - return native_sql_hook_files(project_dir=project_dir, on_fault=on_fault) - - def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: """Discover provider classes under providers/.""" @@ -1313,19 +1215,6 @@ def _load_materialization_module(*, file_path: Path, project_dir: Path) -> Modul return module -def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: - """Detect a project-level adapter.py without importing it.""" - - file_path: Path = project_dir / "adapter.py" - if not file_path.is_file(): - return None - - return DiscoveredAdapterFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - ) - - def _sorted_glob(*, project_dir: Path, root: Path, pattern: str) -> list[Path]: """Return ``sorted(root.rglob(pattern))`` from the active discovery snapshot.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index d833e2b626..4608352f51 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from pathlib import Path -from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_groups +import sqlbuild._native as _native from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import NamedDeclarationRoot from sqlbuild.compiler.scopes.constants import ( @@ -70,5 +70,5 @@ def named_declaration_files( def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Path], ...]: return tuple( (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) - for root, directory in native_declaration_groups(project_dir=project_dir) + for root, directory in _native.declaration_groups(project_dir=project_dir) ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py index 0db80edd8a..0757686bc3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py @@ -2,23 +2,8 @@ from __future__ import annotations -from functools import lru_cache from pathlib import Path -from sqlbuild.compiler.discovery.constants import ( - CANONICAL_AUTHORED_ROOTS, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - SCOPED_DECLARATION_DIRECTORIES, -) -from sqlbuild.compiler.scopes.types import DeclarationKind - -_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" -_SQL_FILE_SUFFIX: str = ".sql" -_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts - def project_relative_path(*, path: Path, project_dir: Path) -> Path: """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" @@ -31,38 +16,3 @@ def project_relative_path(*, path: Path, project_dir: Path) -> Path: ): return path.relative_to(project_dir) return path.with_segments(*path_parts[len(root_parts) :]) - - -def is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: - """Return whether a file lives inside a scoped declaration tree rather than a resource tree.""" - - return _is_scoped_declaration_directory( - directory=file_path.parent, - project_dir=project_dir, - sql_file=file_path.suffix == _SQL_FILE_SUFFIX, - ) - - -@lru_cache(maxsize=8192) -def _is_scoped_declaration_directory(*, directory: Path, project_dir: Path, sql_file: bool) -> bool: - """Classify a file's directory once; every file in it shares the same scoped-tree answer.""" - - directory_parts: tuple[str, ...] = project_relative_path( - path=directory, project_dir=project_dir - ).parts - root_components: tuple[str, ...] - for root_components in CANONICAL_AUTHORED_ROOTS: - if directory_parts[: len(root_components)] != root_components: - continue - scoped_components: tuple[str, ...] = directory_parts[len(root_components) :] - if ( - root_components == _SQL_TEST_ROOT_COMPONENTS - and sql_file - and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) - ): - return False - return any( - component == DECLARATION_GROUP_DIRECTORY or component in SCOPED_DECLARATION_DIRECTORIES - for component in scoped_components - ) - return False diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py deleted file mode 100644 index 2b80061b1d..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Declaration files discovered by the native engine, materialised as Python discovery records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.auditing.models import MeasurementContract -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_failure, - native_locations, - native_project_tree, - native_read_error, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_PARSED_TAG, - SQL_AUDIT_HEADER_KEYS, - SQL_FUNCTION_HEADER_KEYS, - SQL_HOOK_HEADER_KEYS, -) -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredMacroFile, - DiscoveredModelSchemaFile, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveryFileFault, - EnumDeclaration, - EnumMember, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.discovery.types import NativeFileScope, NativeLocation, NativeScopeFields -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind - -type _NativeDeclarationFile = tuple[str, NativeFileScope | None, tuple[object, ...]] -type _Build[RecordT] = Callable[..., RecordT] -type _EnumPayload = tuple[str, dict[str, object], str] -type _ConstantPayload = tuple[str, dict[str, object], str | None, str | None] -type _SchemaPayload = tuple[str, str | None, str | None, dict[str, object], list[NativeLocation]] - -_SESSION_MEMO_KEY: str = "native_discovery_session" - - -def native_enum_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredEnumFile, ...]: - """Discover the enum files natively; Python parses the files native parsing defers.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.ENUM, - isolate_kind=isolate_declaration_kind, - build=_enum_file, - on_fault=on_fault, - ) - - -def native_constant_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredConstantFile, ...]: - """Discover the constant files natively; Python normalises each constant's value.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.CONSTANT, - isolate_kind=isolate_declaration_kind, - build=_constant_file, - on_fault=on_fault, - ) - - -def native_model_schema_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredModelSchemaFile, ...]: - """Discover the reusable model schema files natively; Python parses each schema's columns.""" - - return _native_records( - project_dir=project_dir, - kind="model_schema", - build=_schema_file, - on_fault=on_fault, - ) - - -def native_sql_function_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredSqlFunctionFile, ...]: - """Discover the SQL function files natively.""" - - return _native_records( - project_dir=project_dir, - kind="sql_function", - build=_function_file, - on_fault=on_fault, - ) - - -def native_sql_hook_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredSqlHookFile, ...]: - """Discover the named SQL hook files natively.""" - - return _native_records( - project_dir=project_dir, - kind="sql_hook", - build=_hook_file, - on_fault=on_fault, - ) - - -def native_audit_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredAuditFile, ...]: - """Discover the generic and singular audit files natively.""" - - return _native_records( - project_dir=project_dir, - kind="audit", - build=_audit_file, - on_fault=on_fault, - ) - - -def native_macro_files( - *, - project_dir: Path, - isolate_declaration_kind: bool, -) -> tuple[DiscoveredMacroFile, ...]: - """Discover and read the macro files natively; Python reads the files native reading defers.""" - - return _native_records( - project_dir=project_dir, - kind=DeclarationKind.MACRO, - isolate_kind=isolate_declaration_kind, - build=_macro_file, - on_fault=None, - ) - - -def native_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: - """Discover the seed files natively.""" - - seeds: tuple[DiscoveredSeedFile, ...] = tuple( - DiscoveredSeedFile(file_path=project_dir / relative_path, relative_path=Path(relative_path)) - for relative_path, _scope, _payload in _native_files(project_dir=project_dir, kind="seed") - ) - return seeds - - -def retained_discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession | None: - """Return the native session this discovery pass read its declaration files with, if any.""" - - session: object = DirectorySnapshot.current(project_dir=project_dir).memo.get(_SESSION_MEMO_KEY) - return session if isinstance(session, _native.NativeDiscoverySession) else None - - -def _native_records[RecordT]( - *, - project_dir: Path, - kind: str, - build: _Build[RecordT], - on_fault: Callable[[DiscoveryFileFault], None] | None, - isolate_kind: bool = False, -) -> tuple[RecordT, ...]: - files: list[_NativeDeclarationFile] = _native_files( - project_dir=project_dir, kind=kind, isolate_kind=isolate_kind - ) - return materialise_native_files( - project_dir=project_dir, - files=((Path(relative_path), (scope, payload)) for relative_path, scope, payload in files), - build=lambda relative_path, item: _record( - project_dir=project_dir, - relative_path=relative_path, - item=item, - build=build, - ), - on_fault=on_fault, - ) - - -def _native_files( - *, project_dir: Path, kind: str, isolate_kind: bool = False -) -> list[_NativeDeclarationFile]: - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: list[_NativeDeclarationFile] = native_collection( - result=_discovery_session(project_dir=project_dir).collection(kind, tree, isolate_kind), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return files - - -def _discovery_session(*, project_dir: Path) -> _native.NativeDiscoverySession: - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - session: _native.NativeDiscoverySession | None = retained_discovery_session( - project_dir=project_dir - ) - if session is not None: - return session - created: _native.NativeDiscoverySession = _native.NativeDiscoverySession( - native_request( - project_dir=project_dir, - fields={ - "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), - "audit_keys": sorted(SQL_AUDIT_HEADER_KEYS), - "hook_keys": sorted(SQL_HOOK_HEADER_KEYS), - }, - ) - ) - snapshot.memo[_SESSION_MEMO_KEY] = created - return created - - -def _record[RecordT]( - *, - project_dir: Path, - relative_path: Path, - item: tuple[NativeFileScope | None, tuple[object, ...]], - build: _Build[RecordT], -) -> RecordT: - """Build a parsed file or raise Python's error for it.""" - - scope, payload = item - tag: object = payload[0] - if tag == NATIVE_PARSED_TAG: - return build( - project_dir=project_dir, relative_path=relative_path, scope=scope, payload=payload - ) - if tag == NATIVE_FAILED_TAG: - raise native_failure(payload) - raise native_read_error(payload=payload, file_path=project_dir / relative_path) - - -def _raise_partial_failure(failure: object) -> None: - if failure is not None: - raise native_failure(cast(tuple[object, ...], failure)) - - -def _scope_fields(scope: NativeFileScope | None) -> NativeScopeFields: - if scope is None: - return NativeScopeFields() - _kind, scope_kind, ownership_root, owning_path, declaration_root = scope - return NativeScopeFields( - scope_kind=ScopeKind(scope_kind), - ownership_root=_optional_path(ownership_root), - owning_path=_optional_path(owning_path), - declaration_root=_optional_path(declaration_root), - ) - - -def _optional_path(text: str | None) -> Path | None: - return None if text is None else Path(text) - - -def _audit_block(block: tuple[object, ...]) -> DiscoveredAuditBlock: - audit_index, values, sql_body, name, mode, measure_sql, evidence_sql = block - header_values: dict[str, object] = cast(dict[str, object], values) - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode(str(mode)) - return DiscoveredAuditBlock( - audit_index=cast(int, audit_index), - header_values=header_values, - sql_body=str(sql_body), - name=cast(str | None, name), - evaluation_mode=evaluation_mode, - measurement_contract=( - MeasurementContract( - value_column=cast(str, header_values["value"]), - sample_count_column=cast(str | None, header_values.get("sample_count")), - sample_unit=cast(str | None, header_values.get("sample_unit")), - ) - if evaluation_mode is AuditEvaluationMode.MEASUREMENT - else None - ), - measure_sql=cast(str | None, measure_sql), - evidence_sql=cast(str | None, evidence_sql), - ) - - -def _enum_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredEnumFile: - _tag, contents, declarations = payload - return DiscoveredEnumFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - declarations=tuple( - _enum_declaration(relative_path=relative_path, declaration=declaration) - for declaration in cast(list[_EnumPayload], declarations) - ), - **_scope_fields(scope), - ) - - -def _enum_declaration(*, relative_path: Path, declaration: _EnumPayload) -> EnumDeclaration: - name, members, scalar_type = declaration - projected: dict[str, object] = members - return EnumDeclaration( - name=name, - members=tuple( - EnumMember(name=member, value=cast(str | int, value)) - for member, value in projected.items() - ), - scalar_type=scalar_type, - relative_path=relative_path, - ) - - -def _constant_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredConstantFile: - _tag, contents, declarations, failure = payload - file_path: Path = project_dir / relative_path - constants: tuple[ConstantDeclaration, ...] = tuple( - _constant_declaration( - file_path=file_path, relative_path=relative_path, declaration=declaration - ) - for declaration in cast(list[_ConstantPayload], declarations) - ) - _raise_partial_failure(failure) - return DiscoveredConstantFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - declarations=constants, - **_scope_fields(scope), - ) - - -def _constant_declaration( - *, file_path: Path, relative_path: Path, declaration: _ConstantPayload -) -> ConstantDeclaration: - name, value, explicit_type, render_as = declaration - return cast( - ConstantDeclaration, - _native.typed_constant_record( - (name, value["value"]), (explicit_type, render_as), (file_path, relative_path) - ), - ) - - -def _schema_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredModelSchemaFile: - _tag, contents, declarations, failure = payload - file_path: Path = project_dir / relative_path - schemas: tuple[ModelSchemaDeclaration, ...] = tuple( - _schema_declaration( - file_path=file_path, relative_path=relative_path, declaration=declaration - ) - for declaration in cast(list[_SchemaPayload], declarations) - ) - _raise_partial_failure(failure) - return DiscoveredModelSchemaFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - declarations=schemas, - **_scope_fields(scope), - ) - - -def _schema_declaration( - *, file_path: Path, relative_path: Path, declaration: _SchemaPayload -) -> ModelSchemaDeclaration: - name, description, extends, columns, locations = declaration - return ModelSchemaDeclaration( - name=name, - description=description, - extends=extends, - columns=parse_schema_columns( - raw_columns=columns.get("columns"), - file_path=file_path, - label=f"schema '{name}'", - error_class=DeclarationParseError, - column_locations=native_locations(locations=locations, relative_path=relative_path), - require_columns=True, - ), - relative_path=relative_path, - ) - - -def _function_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredSqlFunctionFile: - _ = scope - _tag, contents, header_values, body_sql = payload - return DiscoveredSqlFunctionFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - header_values=cast(dict[str, object], header_values), - body_sql=str(body_sql), - ) - - -def _hook_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredSqlHookFile: - _tag, contents, header_values, sql_body, name, description = payload - return DiscoveredSqlHookFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - header_values=cast(dict[str, object], header_values), - sql_body=str(sql_body), - name=str(name), - description=cast(str | None, description), - **_scope_fields(scope), - ) - - -def _audit_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredAuditFile: - _tag, contents, blocks = payload - return DiscoveredAuditFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - blocks=tuple(_audit_block(block) for block in cast(list[tuple[object, ...]], blocks)), - declaration_kind=DeclarationKind(cast(NativeFileScope, scope)[0]), - **_scope_fields(scope), - ) - - -def _macro_file( - *, - project_dir: Path, - relative_path: Path, - scope: NativeFileScope | None, - payload: tuple[object, ...], -) -> DiscoveredMacroFile: - _tag, contents = payload - return DiscoveredMacroFile( - file_path=project_dir / relative_path, - relative_path=relative_path, - contents=str(contents), - **_scope_fields(scope), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py deleted file mode 100644 index f8f83add05..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py +++ /dev/null @@ -1,57 +0,0 @@ -"""The declaration layout walked and validated natively for the declaration collections.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_collection, - native_failure, - native_project_tree, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import NATIVE_FAILED_TAG -from sqlbuild.compiler.scopes.types import DeclarationKind - -_LAYOUT_MEMO_KEY: str = "native_declaration_layout" -_GROUPS_MEMO_KEY: str = "native_declaration_groups" - - -def native_declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: - """Return the `(canonical root, group directory)` pairs of a valid named layout.""" - - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - groups: object = snapshot.memo.get(_GROUPS_MEMO_KEY) - if groups is None: - _facts, groups = _layout(project_dir=project_dir, declaration_kind=None) - return cast(list[tuple[str, str]], _rows(cast(tuple[object, ...], groups))) - - -def _layout( - *, project_dir: Path, declaration_kind: DeclarationKind | None -) -> tuple[tuple[object, ...], tuple[object, ...]]: - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - memo_key: tuple[str, DeclarationKind | None] = (_LAYOUT_MEMO_KEY, declaration_kind) - cached: object = snapshot.memo.get(memo_key) - if cached is not None: - return cast(tuple[tuple[object, ...], tuple[object, ...]], cached) - tree: _native.NativeProjectTree = native_project_tree(project_dir) - layout: tuple[tuple[object, ...], tuple[object, ...]] = native_collection( - result=_native.discover_declaration_layout( - tree, None if declaration_kind is None else declaration_kind.value - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - snapshot.memo[memo_key] = layout - _ = snapshot.memo.setdefault(_GROUPS_MEMO_KEY, layout[1]) - return layout - - -def _rows(payload: tuple[object, ...]) -> object: - if payload[0] == NATIVE_FAILED_TAG: - raise native_failure(payload) - return payload[1] diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py deleted file mode 100644 index bb40b05506..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ /dev/null @@ -1,81 +0,0 @@ -"""SQL model files discovered by the native engine as native records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_payload_error, - native_project_tree, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - REMOVED_SQL_MODEL_HEADER_KEYS, - SQL_MODEL_HEADER_KEYS, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveryFileFault -from sqlbuild.compiler.discovery.types import NativeModelFiles - -_RAISED_TAG: str = "raised" - - -def discover_native_model_files( - *, - project_dir: Path, - extract_implicit_alias_columns: bool, - extract_output_column_locations: bool, - selected_model_names: frozenset[str] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, -) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover the (selected) model files natively, reporting failing files to `on_fault`.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: NativeModelFiles = native_collection( - result=_native.discover_model_files( - native_request( - project_dir=project_dir, - fields={ - "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), - "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), - "extract_implicit_alias_columns": extract_implicit_alias_columns, - "extract_output_column_locations": extract_output_column_locations, - }, - ), - tree, - project_dir, - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return materialise_native_files( - project_dir=project_dir, - files=( - (relative_path, item) - for relative_path, item in files - if selected_model_names is None or relative_path.stem in selected_model_names - ), - build=lambda relative_path, item: _model_file( - project_dir=project_dir, relative_path=relative_path, item=item - ), - on_fault=on_fault, - ) - - -def _model_file(*, project_dir: Path, relative_path: Path, item: object) -> DiscoveredSqlModelFile: - if isinstance(item, DiscoveredSqlModelFile): - return item - payload: tuple[object, ...] = cast(tuple[object, ...], item) - if payload[0] == _RAISED_TAG: - raise cast(Exception, payload[1]) - error: Exception | None = native_payload_error( - payload=payload, file_path=project_dir / relative_path - ) - raise error if error is not None else ValueError(payload) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py deleted file mode 100644 index b6255bcc15..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Shared decoding of the plain-data payloads native discovery returns.""" - -from __future__ import annotations - -import os -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_UNLISTABLE_TAG, - NATIVE_UNREADABLE_TAG, -) -from sqlbuild.compiler.discovery.exceptions import ( - DeclarationParseError, - DiscoveryConflictError, - DiscoveryError, - ModelSqlParseError, - ProjectPathError, - SchemaParseError, - SeedDiscoveryError, - SourceParseError, - SqlAuditParseError, - SqlHookParseError, - SqlScenarioParseError, - SqlTestParseError, -) -from sqlbuild.compiler.discovery.models import DiscoveryFileFault -from sqlbuild.compiler.discovery.types import ( - DirectorySnapshotEntry, - NativeListing, - NativeLocation, -) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.spec.contracts.models import SourceLocation - -_DISPLAY_PROBE: str = "_" -_PATH_ENCODING: str = "utf-8" -_DECODE_ENCODING: str = "utf-8" -_DECODE_READ_KIND: str = "decode" -_NATIVE_TREE_MEMO_KEY: str = "native_project_tree" -_PROJECT_PATH_KIND: str = "project_path" -_NAMES_ARE_BYTES: bool = os.supports_bytes_environ -_SURROGATE_ESCAPE_BASE: int = 0xDC00 -_ESCAPED_BYTE_FIRST: int = 0xDC80 -_ESCAPED_BYTE_LAST: int = 0xDCFF -_SURROGATE_FIRST: int = 0xD800 -_SURROGATE_LAST: int = 0xDFFF -_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = { - "model_sql": ModelSqlParseError, - "declaration": DeclarationParseError, - "sql_test": SqlTestParseError, - "sql_scenario": SqlScenarioParseError, - "schema": SchemaParseError, - "source": SourceParseError, - _PROJECT_PATH_KIND: ProjectPathError, - "sql_hook": SqlHookParseError, - "sql_audit": SqlAuditParseError, - "resource_identity": ResourceIdentityError, - "conflict": DiscoveryConflictError, - "seed": SeedDiscoveryError, -} - - -def native_display_prefix(project_dir: Path) -> str: - """Return the text `str(project_dir / name)` prints before `name`.""" - - return str(project_dir / _DISPLAY_PROBE)[: -len(_DISPLAY_PROBE)] - - -def native_failure(payload: tuple[object, ...]) -> DiscoveryError: - """Build the discovery error a native failure payload describes.""" - - _tag, kind, message, help_text = payload - error_class: type[DiscoveryError] = _FAILURE_CLASSES[str(kind)] - return error_class(str(message), help=None if help_text is None else str(help_text)) - - -def native_read_error(*, payload: tuple[object, ...], file_path: Path) -> Exception: - """Build the error Python's `read_text` raises for a file native discovery could not read.""" - - if payload[1] == _DECODE_READ_KIND: - _tag, _kind, data, start, end, reason = payload - return UnicodeDecodeError( - _DECODE_ENCODING, cast(bytes, data), cast(int, start), cast(int, end), str(reason) - ) - _tag, _kind, errno, message, winerror = payload - if winerror is not None: - return OSError(None, str(message), str(file_path), cast(int, winerror)) - if errno is None: - return OSError(str(message)) - return OSError(errno, os.strerror(cast(int, errno)), str(file_path)) - - -def native_payload_error(*, payload: tuple[object, ...], file_path: Path) -> Exception | None: - """Return the error a failed or unreadable file payload raises, or `None` for a parsed one.""" - - tag: object = payload[0] - if tag == NATIVE_FAILED_TAG: - return native_failure(payload) - if tag == NATIVE_UNREADABLE_TAG: - return native_read_error(payload=payload, file_path=file_path) - return None - - -def native_collection[ResultT]( - *, result: ResultT | tuple[object, ...], project_dir: Path -) -> ResultT: - """Return a native collection, raising the error of a collection that could not be read.""" - - if not isinstance(result, tuple) or not isinstance(result[0], str): - return cast(ResultT, result) - payload: tuple[object, ...] = result - if payload[0] == NATIVE_UNLISTABLE_TAG: - _tag, relative_path, read_payload = payload - raise native_read_error( - payload=cast(tuple[object, ...], read_payload), - file_path=project_dir / str(relative_path), - ) - raise native_failure(payload) - - -def materialise_native_files[ItemT, RecordT]( - *, - project_dir: Path, - files: Iterable[tuple[Path, ItemT]], - build: Callable[[Path, ItemT], RecordT], - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[RecordT, ...]: - """Build each native file payload, reporting and skipping a failing file when asked.""" - - records: list[RecordT] = [] - for relative_path, payload in files: - try: - records.append(build(relative_path, payload)) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - if on_fault is None: - raise - on_fault( - DiscoveryFileFault( - path=relative_path, message=str(error).replace(str(project_dir), ".") - ) - ) - return tuple(records) - - -def native_locations( - *, locations: list[NativeLocation], relative_path: Path -) -> dict[str, SourceLocation]: - """Build authored locations; a repeated name keeps its first position and last value.""" - - return { - name: SourceLocation( - path=relative_path, - line=line, - column=column, - end_line=end_line, - end_column=end_column, - ) - for name, line, column, end_line, end_column in locations - } - - -def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: - """Return the native listings shared by every native call of this discovery pass.""" - - snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - tree: object = snapshot.memo.get(_NATIVE_TREE_MEMO_KEY) - if isinstance(tree, _native.NativeProjectTree): - return tree - for text in (str(project_dir), native_display_prefix(project_dir)): - if not native_path_text_supported(text): - raise ProjectPathError( - f"Project directory {_escaped(text)} is not valid UTF-8; move the project to a " - "directory whose path is valid UTF-8" - ) - created: _native.NativeProjectTree = _native.NativeProjectTree(str(project_dir)) - snapshot.memo[_NATIVE_TREE_MEMO_KEY] = created - return created - - -def native_request(*, project_dir: Path, fields: dict[str, object]) -> dict[str, object]: - """Return a native discovery request for the project.""" - - return { - "project_dir": str(project_dir), - "display_prefix": native_display_prefix(project_dir), - **fields, - } - - -def seed_snapshot_listings(*, project_dir: Path, tree: _native.NativeProjectTree) -> None: - """Share the native walk's listings with the pass's Python directory snapshot.""" - - listings: list[NativeListing] = tree.listings() - DirectorySnapshot.current(project_dir=project_dir).seed_listings( - {project_dir / directory: _snapshot_entries(entries) for directory, entries in listings} - ) - - -def _snapshot_entries( - entries: list[tuple[str, bool, bool]], -) -> tuple[DirectorySnapshotEntry, ...]: - return tuple( - DirectorySnapshotEntry(name=name, is_dir=is_dir, is_walkable_dir=walkable) - for name, is_dir, walkable in entries - ) - - -def native_path_text_supported(text: str) -> bool: - """Whether a path's text crosses to the native engine unchanged (no surrogate escapes).""" - - try: - _ = text.encode(_PATH_ENCODING) - except UnicodeEncodeError: - return False - return True - - -def native_unreadable_path_payload(*, project_dir: Path, relative_text: str) -> tuple[object, ...]: - """Return the per-file failure payload for a path native discovery cannot name, if any.""" - - if native_path_text_supported(relative_text): - return () - return ( - NATIVE_FAILED_TAG, - _PROJECT_PATH_KIND, - f"Project path {_escaped(str(project_dir / relative_text))} is not valid UTF-8; rename it " - "so SQLBuild can read it", - None, - ) - - -def _escaped(text: str) -> str: - return "".join(_escaped_character(character) for character in text) - - -def _escaped_character(character: str) -> str: - code_point: int = ord(character) - if _NAMES_ARE_BYTES and _ESCAPED_BYTE_FIRST <= code_point <= _ESCAPED_BYTE_LAST: - return f"\\x{code_point - _SURROGATE_ESCAPE_BASE:02x}" - if _SURROGATE_FIRST <= code_point <= _SURROGATE_LAST: - return f"\\u{code_point:04x}" - return character diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py deleted file mode 100644 index 432fbc7113..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py +++ /dev/null @@ -1,190 +0,0 @@ -"""SQL unit-test and scenario files discovered by the native engine, as Python records.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_failure, - native_payload_error, - native_project_tree, - native_request, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIO_HEADER_KEYS, - SQL_TEST_HEADER_KEYS, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, - DiscoveryFileFault, -) -from sqlbuild.compiler.discovery.types import NativeFiles - -type _NativeDiscovery = Callable[[dict[str, object], _native.NativeProjectTree], object] - - -def discover_native_test_files( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, -) -> tuple[DiscoveredSqlTestFile, ...]: - """Discover the (selected) SQL test files natively, reporting failing files to `on_fault`.""" - - files: NativeFiles = _native_files( - project_dir=project_dir, discover=_native.discover_sql_test_files - ) - return materialise_native_files( - project_dir=project_dir, - files=( - (Path(relative_path), payload) - for relative_path, payload in files - if selected_paths is None or (project_dir / relative_path).resolve() in selected_paths - ), - build=lambda relative_path, payload: _test_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def discover_native_scenario_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlScenarioFile, ...]: - """Discover scenario files natively, reporting failing files to `on_fault`.""" - - files: NativeFiles = _native_files( - project_dir=project_dir, discover=_native.discover_scenario_files - ) - return materialise_native_files( - project_dir=project_dir, - files=((Path(relative_path), payload) for relative_path, payload in files), - build=lambda relative_path, payload: _scenario_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def parse_native_sql_test_contents( - *, contents: str, file_path: Path -) -> tuple[DiscoveredSqlTestBlock, ...]: - """Parse in-memory SQL test contents into ordered blocks, as test discovery does.""" - - payload: tuple[object, ...] = _native.parse_sql_test_contents( - _text_request(file_path), contents - ) - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, _contents, native_blocks, block_failure = payload - return _test_blocks( - file_path=file_path, - native_blocks=cast(list[tuple[dict[str, object], str, tuple[int, int]]], native_blocks), - block_failure=cast(tuple[object, ...] | None, block_failure), - ) - - -def _text_request(file_path: Path) -> dict[str, object]: - return { - "file_path": str(file_path), - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - } - - -def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> NativeFiles: - with DirectorySnapshot.scope(project_dir=project_dir): - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: NativeFiles = native_collection( - result=cast( - NativeFiles | tuple[object, ...], - discover( - native_request( - project_dir=project_dir, - fields={ - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - }, - ), - tree, - ), - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return files - - -def _test_file( - *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] -) -> DiscoveredSqlTestFile: - file_path: Path = project_dir / relative_path - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, native_contents, native_blocks, block_failure = payload - return DiscoveredSqlTestFile( - file_path=file_path, - relative_path=relative_path, - contents=str(native_contents), - blocks=_test_blocks( - file_path=file_path, - native_blocks=cast(list[tuple[dict[str, object], str, tuple[int, int]]], native_blocks), - block_failure=cast(tuple[object, ...] | None, block_failure), - ), - ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), - ) - - -def _test_blocks( - *, - file_path: Path, - native_blocks: list[tuple[dict[str, object], str, tuple[int, int]]], - block_failure: tuple[object, ...] | None, -) -> tuple[DiscoveredSqlTestBlock, ...]: - blocks: tuple[DiscoveredSqlTestBlock, ...] = tuple( - cast(list[DiscoveredSqlTestBlock], _native.sql_test_blocks(str(file_path), native_blocks)) - ) - if block_failure is not None: - raise native_failure(block_failure) - _native.validate_sql_test_block_names( - str(file_path), [(block.test_index, block.name) for block in blocks] - ) - return blocks - - -def _scenario_file( - *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] -) -> DiscoveredSqlScenarioFile: - return _scenario_file_from_payload( - payload=payload, file_path=project_dir / relative_path, relative_path=relative_path - ) - - -def _scenario_file_from_payload( - *, payload: tuple[object, ...], file_path: Path, relative_path: Path -) -> DiscoveredSqlScenarioFile: - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, contents, values, sql_body, sql_body_span = payload - return build_sql_scenario_file( - header_values=cast(dict[str, object], values), - sql_body=str(sql_body), - sql_body_span=cast(tuple[int, int], sql_body_span), - contents=str(contents), - file_path=file_path, - relative_path=relative_path, - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py deleted file mode 100644 index 0760756762..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Source and schema YAML files read and loaded by the native engine.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import project_relative_path -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - materialise_native_files, - native_collection, - native_display_prefix, - native_payload_error, - native_project_tree, - native_unreadable_path_payload, - seed_snapshot_listings, -) -from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( - schema_file_paths, - source_file_paths, -) -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import ( - NATIVE_SCHEMA_YAML_KIND, - NATIVE_SOURCE_YAML_KIND, - NATIVE_YAML_BATCH_BYTES, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSourceFile, - DiscoveryFileFault, -) - -type _Payload = tuple[object, ...] - - -def discover_native_source_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSourceFile, ...]: - """Discover source YAML files natively, reporting failing files to `on_fault`.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) - return materialise_native_files( - project_dir=project_dir, - files=( - (project_relative_path(path=file_path, project_dir=project_dir), payload) - for file_path, payload in _native_payloads( - project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SOURCE_YAML_KIND - ) - ), - build=lambda relative_path, payload: _source_file( - project_dir=project_dir, relative_path=relative_path, payload=payload - ), - on_fault=on_fault, - ) - - -def discover_native_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: - """Discover model schema.yml and seed declaration files natively.""" - - with DirectorySnapshot.scope(project_dir=project_dir): - file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) - return tuple( - _schema_file(project_dir=project_dir, file_path=file_path, payload=payload) - for file_path, payload in _native_payloads( - project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SCHEMA_YAML_KIND - ) - ) - - -def _source_file( - *, project_dir: Path, relative_path: Path, payload: _Payload -) -> DiscoveredSourceFile: - file_path: Path = project_dir / relative_path - contents, loaded = _loaded(payload=payload, file_path=file_path) - return DiscoveredSourceFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), - ) - - -def _schema_file(*, project_dir: Path, file_path: Path, payload: _Payload) -> DiscoveredSchemaFile: - contents, loaded = _loaded(payload=payload, file_path=file_path) - model_entries, seed_entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) - return DiscoveredSchemaFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - model_entries=model_entries, - seed_entries=seed_entries, - ) - - -def _loaded(*, payload: _Payload, file_path: Path) -> tuple[str, object]: - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, contents, loaded = payload - return str(contents), loaded - - -def _relative_text(*, project_dir: Path, file_path: Path) -> str: - return project_relative_path(path=file_path, project_dir=project_dir).as_posix() - - -def _native_payloads( - *, project_dir: Path, file_paths: tuple[Path, ...], kind: str -) -> Iterator[tuple[Path, _Payload]]: - """Load the files natively in batches bounded by size, releasing each file once used.""" - - if not file_paths: - return - tree: _native.NativeProjectTree = native_project_tree(project_dir) - request: dict[str, object] = { - "project_dir": str(project_dir), - "display_prefix": native_display_prefix(project_dir), - "kind": kind, - } - for batch in _batches(file_paths=file_paths): - relative_texts: list[str] = [ - _relative_text(project_dir=project_dir, file_path=path) for path in batch - ] - unnamed: list[_Payload] = [ - native_unreadable_path_payload(project_dir=project_dir, relative_text=text) - for text in relative_texts - ] - payloads: list[_Payload] = native_collection( - result=_native.load_yaml_files( - request, - [ - text - for text, failure in zip(relative_texts, unnamed, strict=True) - if not failure - ], - tree, - ), - project_dir=project_dir, - ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - payloads.reverse() - for file_path, failure in zip(batch, unnamed, strict=True): - yield file_path, failure or payloads.pop() - - -def _batches(*, file_paths: tuple[Path, ...]) -> Iterator[tuple[Path, ...]]: - batch: list[Path] = [] - batch_bytes: int = 0 - for file_path in file_paths: - size: int = _file_size(file_path) - if batch and batch_bytes + size > NATIVE_YAML_BATCH_BYTES: - yield tuple(batch) - batch, batch_bytes = [], 0 - batch.append(file_path) - batch_bytes += size - if batch: - yield tuple(batch) - - -def _file_size(file_path: Path) -> int: - try: - return file_path.stat().st_size - except OSError: - return 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py deleted file mode 100644 index 9ddb5b669c..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Parsing helpers for authored SQL scenario files.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIOS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile - -_SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" -_SCENARIO_TAGS_HEADER_KEY: str = "tags" - - -def build_sql_scenario_file( - *, - header_values: dict[str, object], - sql_body: str, - contents: str, - file_path: Path, - relative_path: Path, - sql_body_span: tuple[int, int] | None = None, -) -> DiscoveredSqlScenarioFile: - """Validate a header-parsed SCENARIO file with only supported keys and build its record.""" - - description_value: object | None = header_values.get(_SCENARIO_DESCRIPTION_HEADER_KEY) - if _SCENARIO_DESCRIPTION_HEADER_KEY in header_values and not isinstance(description_value, str): - raise SqlScenarioParseError(f"SCENARIO() description in '{file_path}' must be a string") - tags_value: object | None = header_values.get(_SCENARIO_TAGS_HEADER_KEY) - if _SCENARIO_TAGS_HEADER_KEY in header_values and ( - not isinstance(tags_value, list) or not all(isinstance(tag, str) for tag in tags_value) - ): - raise SqlScenarioParseError(f"SCENARIO() tags in '{file_path}' must be a list of strings") - if not sql_body: - raise SqlScenarioParseError( - f"SQL scenario '{file_path}' must define SQL after SCENARIO(...)" - ) - - return DiscoveredSqlScenarioFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - header_values=header_values, - sql_body=sql_body, - sql_body_span=sql_body_span, - name=file_path.stem, - ownership_root=Path(SQL_SCENARIOS_OWNERSHIP_ROOT), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 7301212521..46d967b628 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -2,13 +2,10 @@ from __future__ import annotations -import csv from collections.abc import Callable -from pathlib import Path -from typing import Protocol, cast +from typing import Protocol import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.payloads import native_failure, native_read_error from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -35,10 +32,6 @@ class _NodeDefinition(Protocol): def name(self) -> str: ... -_UNREADABLE_TAG: str = "unreadable" -_CSV_ERROR_TAG: str = "csv" -_CSV_DIALECT_TAG: str = "csv_dialect" -_CSV_DIALECT_ERRORS: dict[str, type[Exception]] = {"TypeError": TypeError, "ValueError": ValueError} _DEFINITION_READERS: tuple[Callable[[Callable[..., object]], _NodeDefinition | None], ...] = ( read_loader_definition, read_task_definition, @@ -50,21 +43,7 @@ def name(self) -> str: ... def validate_discovered_inputs(discovered_inputs: DiscoveredProjectInputs) -> None: """Validate cross-file conflicts across discovered project inputs.""" - payload: tuple[object, ...] | None = _native.discovered_inputs_failure( - discovered_inputs, _nodes(discovered_inputs) - ) - if payload is None: - return - tag: object = payload[0] - if tag == _UNREADABLE_TAG: - raise native_read_error( - payload=cast(tuple[object, ...], payload[2]), file_path=Path(str(payload[1])) - ) - if tag == _CSV_ERROR_TAG: - raise csv.Error(str(payload[1])) - if tag == _CSV_DIALECT_TAG: - raise _CSV_DIALECT_ERRORS[str(payload[1])](str(payload[2])) - raise native_failure(payload) + _native.validate_discovered_project_inputs(discovered_inputs, _nodes(discovered_inputs)) def _nodes(inputs: DiscoveredProjectInputs) -> list[dict[str, object]]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py b/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py deleted file mode 100644 index 39c9289c7a..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/file_paths.py +++ /dev/null @@ -1,48 +0,0 @@ -"""The source and schema YAML files discovery reads, in Python's discovery order.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( - is_in_scoped_declaration_tree, -) -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import SCHEMA_FILE_NAME, YAML_FILE_SUFFIXES -from sqlbuild.compiler.discovery.exceptions import SchemaParseError - - -def source_file_paths(*, project_dir: Path) -> tuple[Path, ...]: - """Return the YAML files directly below sources/, in path order.""" - - sources_root: Path = project_dir / "sources" - if not sources_root.is_dir(): - return () - return tuple( - sorted(path for path in sources_root.iterdir() if path.suffix in YAML_FILE_SUFFIXES) - ) - - -def schema_file_paths(*, project_dir: Path) -> tuple[Path, ...]: - """Return the unscoped model schema.yml files and seed declaration files, in Python's order.""" - - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - schema_paths: list[Path] = [] - models_root: Path = project_dir / "models" - seeds_root: Path = project_dir / "seeds" - - if models_root.is_dir(): - schema_paths.extend( - path - for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) - if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) - ) - if seeds_root.is_dir(): - yaml_path: Path - for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): - raise SchemaParseError( - f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " - ".yaml is not supported" - ) - schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) - return tuple(dict.fromkeys(schema_paths)) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py deleted file mode 100644 index c2d1cd2033..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Parsing of authored schema.yml files, run natively.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes -from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry - - -def parse_loaded_schema_yml( - *, loaded: object, file_path: Path -) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: - """Parse the document one schema.yml file loads to into raw model and seed metadata.""" - - return (), _native.parse_schema_document(loaded, str(file_path), yaml_entry_classes()) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py deleted file mode 100644 index c92d543598..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Parsing of authored sources/*.yml files, run natively.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes -from sqlbuild.spec.contracts.models import SourceEntry - - -def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[SourceEntry, ...]: - """Parse the document one sources/*.yml file loads to into raw source declarations.""" - - return _native.parse_sources_document(loaded, str(file_path), yaml_entry_classes()) diff --git a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py index 63b49da16b..0effa19175 100644 --- a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py +++ b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py @@ -4,12 +4,12 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files -from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_schema_files, - discover_native_source_files, +from sqlbuild._native import ( + discover_model_files, + discover_model_schema_files, + discover_schema_files, + discover_source_files, + discover_test_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, @@ -39,16 +39,16 @@ def discover( project_config=project_config, local_config=local_config, project_dir=project_dir, - model_files=discover_native_model_files( + model_files=discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, selected_model_names=referenced_model_names, ), model_schema_files=discover_model_schema_files(project_dir=project_dir), - schema_files=discover_native_schema_files(project_dir=project_dir), - source_files=discover_native_source_files(project_dir=project_dir), - test_files=discover_native_test_files( + schema_files=discover_schema_files(project_dir=project_dir), + source_files=discover_source_files(project_dir=project_dir), + test_files=discover_test_files( project_dir=project_dir, selected_paths=selected_test_paths, ), diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index cf4d52c5ac..8d4c1a6f0d 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -7,7 +7,6 @@ LEGACY_PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.yml" LEGACY_LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.yml" TOML_FILE_SUFFIX: str = ".toml" -SCHEMA_FILE_NAME: str = "schema.yml" YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) PYTHON_INIT_MODULE_STEM: str = "__init__" @@ -173,11 +172,3 @@ "description", ) SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") - -NATIVE_FAILED_TAG: str = "error" -NATIVE_UNREADABLE_TAG: str = "read" -NATIVE_UNLISTABLE_TAG: str = "unlistable" -NATIVE_PARSED_TAG: str = "ok" -NATIVE_SOURCE_YAML_KIND: str = "source" -NATIVE_SCHEMA_YAML_KIND: str = "schema" -NATIVE_YAML_BATCH_BYTES: int = 4 * 1024 * 1024 diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py index 4784e2421d..775e3be5b5 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -4,8 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild._native import discover_model_files, discover_model_schema_files from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault @@ -20,7 +19,7 @@ def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProject return DiscoveredProjectInputs( project_config=project_config, local_config=LocalConfig(), - model_files=discover_native_model_files( + model_files=discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, diff --git a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py b/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py deleted file mode 100644 index a91f696df2..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public discovery entry point for parsing in-memory SQL test contents.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - parse_native_sql_test_contents, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock - - -def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: - """Parse current SQL test contents into ordered test blocks.""" - - return parse_native_sql_test_contents(contents=contents, file_path=file_path) diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index a054a52dca..916cf3e92a 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -5,7 +5,7 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, TypedDict +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol from sqlbuild.compiler.scopes.types import ScopeKind @@ -13,11 +13,6 @@ from sqlbuild.provider.classes.provider import Provider type ProjectProvider = Provider -type NativeLocation = tuple[str, int, int, int, int] -type NativeListing = tuple[str, list[tuple[str, bool, bool]]] -type NativeFiles = list[tuple[str, tuple[object, ...]]] -type NativeModelFiles = list[tuple[Path, object]] -type NativeFileScope = tuple[str, str, str | None, str | None, str | None] class LoaderConnectionMode(StrEnum): @@ -37,15 +32,6 @@ class ScopedDeclarationFile(Protocol): declaration_root: Path | None -class NativeScopeFields(TypedDict, total=False): - """The scope fields of a declaration file record that native discovery listed in a role.""" - - scope_kind: ScopeKind - ownership_root: Path | None - owning_path: Path | None - declaration_root: Path | None - - class DirectorySnapshotEntry(NamedTuple): """One directory entry in a discovery snapshot.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index 64dacfb4f4..011ea774ab 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -5,6 +5,7 @@ import re from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.sql_test_cte_extractor import SqlTestCteExtractor from sqlbuild.compiler.compile.constants import ( REF_TEST_CTE_PREFIX, @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.main._parse_sql_test_file import parse_sql_test_file from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, @@ -90,7 +90,7 @@ def format_redundant_fixture_nulls( if contents is None: continue try: - current_blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file( + current_blocks: tuple[DiscoveredSqlTestBlock, ...] = _native.parse_sql_test_file( contents=contents, file_path=file_path, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 9b8f37cf44..678f20d622 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -13,7 +13,6 @@ import duckdb import pytest -from sqlbuild._native import NativeProjectTree from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -29,9 +28,8 @@ from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.analysis_session.models import NativeQueryColumns from sqlbuild.compiler.compile._helpers.assembly import source_bindings as source_bindings_module -from sqlbuild.compiler.discovery._helpers.native import ( - model_files as native_model_files_module, -) +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( @@ -649,18 +647,23 @@ def recording(**kwargs: Any) -> tuple[NativeQueryColumns, ...]: return analysed -def record_eager_output_column_scans(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: - """Record every native model discovery request that locates output columns eagerly.""" +def eager_output_column_scans(*, capture_dir: Path) -> tuple[int, int]: + """Count the captured model files, and those discovery located output columns for eagerly.""" + + located: list[bool] = [] + + def record(item: dict[str, object]) -> dict[str, object]: + located.extend(map(bool, map(item.get, filter(item.__contains__, _OUTPUT_LOCATIONS)))) + return item - eager: list[dict[str, object]] = [] - original: Callable[..., object] = native_model_files_module._native.discover_model_files + for capture in sorted( + capture_dir.glob(f"*-{CompilerStage.DISCOVERED_PROJECT_INPUTS.value}{STAGE_CAPTURE_SUFFIX}") + ): + _ = json.loads(capture.read_text(encoding="utf-8"), object_hook=record) + return len(located), sum(located) - def recording(request: dict[str, object], tree: NativeProjectTree, project_dir: Path) -> object: - eager.extend(filter(lambda item: item["extract_output_column_locations"], (request,))) - return original(request, tree, project_dir) - monkeypatch.setattr(native_model_files_module._native, "discover_model_files", recording) - return eager +_OUTPUT_LOCATIONS: tuple[str, ...] = ("output_column_locations",) def write_repeated_json_parse_project( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 53c3a4c471..c359a31d5c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -7,10 +7,11 @@ import pytest from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import DiscoveryWorkTestCase from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + eager_output_column_scans, execute_duckdb_sql, - record_eager_output_column_scans, write_expression_source_project, ) @@ -57,14 +58,19 @@ def test_given_project_when_running_command_then_discovery_skips_eager_column_lo 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), ) - scanned: list[dict[str, object]] = record_eager_output_column_scans(monkeypatch) + capture_dir: Path = tmp_path / "captures" + monkeypatch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(capture_dir)) _ = capsys.readouterr() exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command]) output: str = "".join(capsys.readouterr()) - assert exit_code == test_case.expected_exit_code, output - assert len(scanned) == test_case.expected_eager_output_column_scans + model_files, eager_scans = eager_output_column_scans(capture_dir=capture_dir) + assert (exit_code, model_files > 0, eager_scans) == ( + test_case.expected_exit_code, + True, + test_case.expected_eager_output_column_scans, + ), output if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index a8cabf8e17..27b6a18d25 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -126,10 +126,10 @@ class UndecodablePathToleranceTestCase: @dataclass(frozen=True) class NativeOsErrorTestCase: - """An operating-system read payload and the Python error it becomes.""" + """An OS read failure `(errno, winerror, message)` and the Python error it becomes.""" description: str - payload: tuple[object, ...] + payload: tuple[int | None, int | None, str] expected_type: str expected_fields: tuple[object, ...] diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 1917af97ec..19f32584fb 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -20,20 +20,17 @@ from typing import Any, cast from sqlbuild import _native -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( - build_tolerant_scope_discovery, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_model_schema_files, + discover_project_inputs, discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_payload_error, +from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( + build_tolerant_scope_discovery, ) from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, @@ -464,24 +461,25 @@ def random_undecodable_bytes(*, rng: random.Random) -> bytes: def native_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: """Return the type and text of the read error native reading reports for each file, if any.""" - payloads: list[tuple[object, ...]] = cast( - list[tuple[object, ...]], + return [ + {True: (type(row).__name__, str(row)), False: ("NoneType", "None")}[ + isinstance(row, UnicodeError | OSError) + ] + for row in _native_yaml_rows(project_dir=project_dir, relative_paths=relative_paths) + ] + + +def _native_yaml_rows(*, project_dir: Path, relative_paths: list[str]) -> list[object]: + """Each file's `(contents, loaded)`, or the error native source YAML loading raises for it.""" + + return cast( + list[object], _native.load_yaml_files( {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, relative_paths, _native.NativeProjectTree(str(project_dir)), ), ) - errors: list[Exception | None] = [ - native_payload_error(payload=payload, file_path=project_dir / path) - for payload, path in zip(payloads, relative_paths, strict=True) - ] - return [ - {True: (type(error).__name__, str(error)), False: ("NoneType", "None")}[ - isinstance(error, UnicodeError | OSError) - ] - for error in errors - ] def python_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: @@ -581,7 +579,7 @@ def declaration_files_outcome(*, project_dir: Path, kind: str) -> tuple[object, def native_declaration_tag(*, file_path: Path, kind: str) -> str: """Return whether native parsing parses, fails or defers one file's contents.""" - payload: tuple[object, ...] = _native.parse_declaration_contents( + return _native.parse_declaration_contents( { "kind": kind, "function_keys": sorted(SQL_FUNCTION_HEADER_KEYS), @@ -591,7 +589,6 @@ def native_declaration_tag(*, file_path: Path, kind: str) -> str: (str(file_path), file_path.stem), file_path.read_text(encoding="utf-8"), ) - return str(payload[0]) def tolerant_declaration_outcome( @@ -1035,18 +1032,14 @@ def deep_model_header_values(*, project_dir: Path, frames: int) -> dict[str, obj def native_yaml_tags_and_values( *, project_dir: Path, relative_paths: list[str] ) -> tuple[list[object], list[object]]: - """Return each file's native tag and the values of the files that loaded.""" + """Return whether each file loaded (`ok`) or failed (`error`), and the loaded values.""" - rows: list[tuple[object, ...]] = cast( - list[tuple[object, ...]], - _native.load_yaml_files( - {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, - relative_paths, - _native.NativeProjectTree(str(project_dir)), - ), + rows: list[object] = _native_yaml_rows(project_dir=project_dir, relative_paths=relative_paths) + failed: list[bool] = [isinstance(row, Exception) for row in rows] + loaded: list[tuple[str, object]] = cast( + list[tuple[str, object]], list(compress(rows, [not failure for failure in failed])) ) - loaded: list[tuple[object, ...]] = list(compress(rows, [row[0] == "ok" for row in rows])) - return [row[0] for row in rows], [row[2] for row in loaded] + return [{True: "error", False: "ok"}[failure] for failure in failed], [row[1] for row in loaded] VALIDATION_MODEL: tuple[str, bytes] = ("models/orders.sql", b"MODEL ();\nSELECT 1 AS order_id") diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index 92108dc62d..4e5e6736bb 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -7,8 +7,8 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.model_files import ( - discover_native_model_files, +from sqlbuild._native import ( + discover_model_files, ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile @@ -52,7 +52,7 @@ def test_given_runtime_and_root_when_discovering_natively_then_supported_runs_an with capture: models.extend( - discover_native_model_files( + discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -82,7 +82,7 @@ def test_given_native_model_walk_when_globbing_models_later_in_the_pass_then_the ) -> None: write_project(project_dir=tmp_path, files=(("models/marts/orders.sql", _VALID_MODEL),)) with DirectorySnapshot.scope(project_dir=tmp_path) as snapshot: - _ = discover_native_model_files( + _ = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=True, extract_output_column_locations=True, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py index af15c466c9..cbc207d2f9 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.payloads import native_read_error +import sqlbuild._native as _native from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( NativeOsErrorTestCase, ReadErrorOracleTestCase, @@ -128,29 +128,29 @@ def test_given_unreadable_file_when_discovering_then_python_read_error_is_raised [ NativeOsErrorTestCase( description="a refused read is Python's errno error naming the file", - payload=("read", "os", 13, "Permission denied (os error 13)", None), + payload=(13, None, "Permission denied (os error 13)"), expected_type="PermissionError", expected_fields=(13, "Permission denied", "orders.sql"), ), NativeOsErrorTestCase( description="a Windows listing error carries its Win32 code for Python to map", - payload=("read", "os", None, "Access is denied", 5), + payload=(None, 5, "Access is denied"), expected_type="OSError", expected_fields=(None, "Access is denied", "orders.sql"), ), NativeOsErrorTestCase( description="an error without a code keeps the operating system message", - payload=("read", "os", None, "stream did not contain valid UTF-8", None), + payload=(None, None, "stream did not contain valid UTF-8"), expected_type="OSError", expected_fields=(None, None, None), ), ], ids=lambda case: case.description, ) -def test_given_os_read_payload_when_building_error_then_python_error_matches_platform( +def test_given_os_read_failure_when_building_error_then_python_error_matches_platform( test_case: NativeOsErrorTestCase, ) -> None: - error: Exception = native_read_error(payload=test_case.payload, file_path=Path("orders.sql")) + error: Exception = _native._oracle_read_error(*test_case.payload, file_path=Path("orders.sql")) assert ( type(error).__name__, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index fa2086eb26..d06ac4c3f9 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -11,19 +11,19 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, + discover_model_files, discover_model_schema_files, + discover_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, + discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_schema_files from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, @@ -63,7 +63,7 @@ _PATTERNS: tuple[str, ...] = ("*", "*.sql", "*.py", "*.yml", "schema.yml", "_sqlbuild", "enums") _ISOLATED_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { "model_files": partial( - discover_native_model_files, + discover_model_files, extract_implicit_alias_columns=True, extract_output_column_locations=True, ), @@ -72,9 +72,9 @@ "model_schema_files": discover_model_schema_files, "sql_function_files": discover_sql_function_files, "sql_hook_files": discover_sql_hook_files, - "schema_files": discover_native_schema_files, + "schema_files": discover_schema_files, "seed_files": discover_seed_files, - "test_files": discover_native_test_files, + "test_files": discover_test_files, "audit_files": discover_audit_files, "macro_files": discover_macro_files, } diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index dad7c3bfbc..03612eb17c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, discover_constant_files, discover_enum_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py index 743ac9f319..c53b3b7445 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py @@ -7,23 +7,23 @@ import pytest +from sqlbuild._native import ( + discover_audit_files, + discover_constant_files, + discover_enum_files, + discover_scenario_files, + discover_source_files, + discover_sql_function_files, + discover_sql_hook_files, +) from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_declarations, ) from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_audit_files, - discover_constant_files, - discover_enum_files, discover_event_exporter_declarations, discover_materialization_files, discover_provider_classes, - discover_sql_function_files, - discover_sql_hook_files, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, ) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import ( EventExporterDiscoveryError, ProviderDiscoveryError, @@ -40,8 +40,8 @@ "audits": discover_audit_files, "constants": discover_constant_files, "enums": discover_enum_files, - "scenarios": discover_native_scenario_files, - "sources": discover_native_source_files, + "scenarios": discover_scenario_files, + "sources": discover_source_files, "sql_functions": discover_sql_function_files, "sql_hooks": discover_sql_hook_files, } diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py index f57ad57df5..b73f54fb5d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_macro_files +from sqlbuild._native import ( + discover_macro_files, +) from sqlbuild.compiler.discovery.models import DiscoveredMacroFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMacroFilesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index de12cf40e1..703405a9ba 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -8,20 +8,20 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_audit_files, - discover_hook_functions, discover_macro_files, - discover_python_function_files, + discover_project_inputs, + discover_scenario_files, + discover_source_files, discover_sql_function_files, discover_sql_hook_files, + discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, - discover_native_test_files, +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_hook_functions, + discover_python_function_files, ) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, @@ -43,7 +43,7 @@ declaration_contents, discover_declarations, ) -from tests.unit.src.sqlbuild.compiler.discovery.helpers import discover_model_files +from tests.unit.src.sqlbuild.compiler.discovery.helpers import discover_all_model_files class _DiscoveredFile(Protocol): @@ -385,13 +385,13 @@ def test_given_file_in_declaration_role_when_discovering_resources_then_returns_ file_path.parent.mkdir(parents=True) file_path.write_text(test_case.contents, encoding="utf-8") discoverers: dict[str, Callable[..., tuple[_DiscoveredFile, ...]]] = { - "model": discover_model_files, - "test": discover_native_test_files, - "scenario": discover_native_scenario_files, + "model": discover_all_model_files, + "test": discover_test_files, + "scenario": discover_scenario_files, "hook": discover_sql_hook_files, "function": discover_sql_function_files, "audit": discover_audit_files, - "source": discover_native_source_files, + "source": discover_source_files, "python_function": discover_python_function_files, "python_hook": discover_hook_functions, } @@ -424,7 +424,7 @@ def test_given_similarly_prefixed_directory_when_discovering_then_treats_it_as_o file_path.parent.mkdir(parents=True) file_path.write_text("MODEL ();\nSELECT 1\n", encoding="utf-8") - result: tuple[_DiscoveredFile, ...] = discover_model_files(project_dir=tmp_path) + result: tuple[_DiscoveredFile, ...] = discover_all_model_files(project_dir=tmp_path) assert tuple(file.relative_path.as_posix() for file in result) == ( test_case.expected_relative_paths diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py index a90d183b86..3ee4442bd7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_sql_hook_files +from sqlbuild._native import ( + discover_sql_hook_files, +) from sqlbuild.compiler.discovery.exceptions import SqlHookParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 173822217a..460ef2ab2f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -4,7 +4,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild._native import ( + discover_model_files, +) from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers from sqlbuild.compiler.discovery._helpers.sql.model_files import model_output_column_locations from sqlbuild.compiler.discovery.models import ( @@ -27,7 +29,7 @@ ParseModelSqlHeaderTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( - discover_model_files, + discover_all_model_files, model_header_column_locations, parse_model_sql, ) @@ -53,7 +55,7 @@ def test_given_unique_model_headers_when_discovering_then_header_values_and_loca f"MODEL ({second_header});\nSELECT 2 AS total\n", encoding="utf-8" ) - discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files(project_dir=tmp_path) + discovered: tuple[DiscoveredSqlModelFile, ...] = discover_all_model_files(project_dir=tmp_path) assert len(discovered) == test_case.expected_count assert [model.header_values for model in discovered] == [ @@ -126,7 +128,7 @@ def test_given_multiple_invalid_model_headers_when_discovering_then_fault_order_ (models_dir / "second.sql").write_text("MODEL (schema ${MISSING); SELECT 2", encoding="utf-8") faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -164,7 +166,7 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection encoding="utf-8", ) - model_file: DiscoveredSqlModelFile = discover_native_model_files( + model_file: DiscoveredSqlModelFile = discover_model_files( project_dir=tmp_path, extract_implicit_alias_columns=(test_case.expected_extract_implicit_alias_columns), extract_output_column_locations=False, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 23b5edbcfc..3f28cc8e09 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -6,7 +6,9 @@ import pytest -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files +from sqlbuild._native import ( + discover_test_files, +) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( omitted_ceremonial_select_offset, @@ -96,7 +98,7 @@ def test_given_invalid_test_file_when_discovering_tolerantly_then_reports_file_f (tests_dir / "b_valid.sql").write_text('TEST (name "valid_order");\nSELECT 1', encoding="utf-8") faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( + discovered: tuple[DiscoveredSqlTestFile, ...] = discover_test_files( project_dir=tmp_path, on_fault=faults.append, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index bf0435e6b5..f6b19d46e7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,30 +1,16 @@ from pathlib import Path -from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files -from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_locations, - native_payload_error, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - parse_native_sql_test_contents, -) -from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes from sqlbuild.compiler.discovery.constants import ( REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, - SQL_SCENARIO_HEADER_KEYS, - SQL_TEST_HEADER_KEYS, ) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, ) -from sqlbuild.compiler.discovery.types import NativeLocation from sqlbuild.spec.contracts.models import ( SchemaModelEntry, SchemaSeedEntry, @@ -47,10 +33,10 @@ def write_lifecycle_event_sink_project(*, project_dir: Path, exporter_config: st ) -def discover_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, ...]: +def discover_all_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, ...]: """Discover every model file with the column extraction full compilation uses.""" - return discover_native_model_files( + return _native.discover_model_files( project_dir=project_dir, extract_implicit_alias_columns=True, extract_output_column_locations=True, @@ -60,8 +46,8 @@ def discover_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: """Parse model contents as model discovery does, returning header values and query SQL.""" - payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=file_path) - return cast(dict[str, object], payload[2]), str(payload[5]) + header_values, query_sql, _ = _parsed_model(contents=contents, file_path=file_path) + return header_values, query_sql def model_header_column_locations( @@ -69,14 +55,13 @@ def model_header_column_locations( ) -> dict[str, SourceLocation]: """Return the authored MODEL(columns) locations model discovery records.""" - payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=relative_path) - return native_locations( - locations=cast(list[NativeLocation], payload[3]), relative_path=relative_path - ) + return _parsed_model(contents=contents, file_path=relative_path)[2] -def _parsed_model(*, contents: str, file_path: Path) -> tuple[object, ...]: - payload: tuple[object, ...] = _native.parse_model_contents( +def _parsed_model( + *, contents: str, file_path: Path +) -> tuple[dict[str, object], str, dict[str, SourceLocation]]: + return _native.parse_model_contents( { "file_path": str(file_path), "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), @@ -86,14 +71,12 @@ def _parsed_model(*, contents: str, file_path: Path) -> tuple[object, ...]: }, contents, ) - _raise_payload_error(payload=payload, file_path=file_path) - return payload def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: """Parse SQL test contents as test discovery does.""" - return parse_native_sql_test_contents(contents=contents, file_path=file_path) + return _native.parse_sql_test_file(contents=contents, file_path=file_path) def parse_sql_scenario_file( @@ -101,31 +84,18 @@ def parse_sql_scenario_file( ) -> DiscoveredSqlScenarioFile: """Parse scenario contents as scenario discovery does.""" - payload: tuple[object, ...] = _native.parse_scenario_contents( - { - "file_path": str(file_path), - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - }, - contents, - ) - _raise_payload_error(payload=payload, file_path=file_path) - return build_sql_scenario_file( - header_values=cast(dict[str, object], payload[2]), - sql_body=str(payload[3]), - contents=str(payload[1]), - file_path=file_path, - relative_path=relative_path, - sql_body_span=cast(tuple[int, int], payload[4]), + return _native.parse_sql_scenario_file( + contents=contents, file_path=file_path, relative_path=relative_path ) def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: """Load and parse sources YAML contents as source discovery does.""" - return parse_loaded_sources_yml( - loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="source"), - file_path=file_path, + return _native.parse_sources_document( + _native.load_yaml_document(str(file_path), contents, "source"), + str(file_path), + yaml_entry_classes(), ) @@ -134,18 +104,8 @@ def parse_schema_yml( ) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: """Load and parse schema or seed YAML contents as schema discovery does.""" - return parse_loaded_schema_yml( - loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="schema"), - file_path=file_path, + return (), _native.parse_schema_document( + _native.load_yaml_document(str(file_path), contents, "schema"), + str(file_path), + yaml_entry_classes(), ) - - -def _loaded_yaml(*, contents: str, file_path: Path, kind: str) -> object: - payload: tuple[object, ...] = _native.load_yaml_document(str(file_path), contents, kind) - _raise_payload_error(payload=payload, file_path=file_path) - return payload[2] - - -def _raise_payload_error(*, payload: tuple[object, ...], file_path: Path) -> None: - for error in filter(None, (native_payload_error(payload=payload, file_path=file_path),)): - raise error From 4fb13b02580d8ba8803bffb92fd394565d174874 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:16:30 +0100 Subject: [PATCH 12/26] refactor: add native dialect and configured namespace helpers --- .../src/assembly/analysis_session/main/mod.rs | 1 + .../analysis_session/main/native_dialect.rs | 8 + .../src/assembly/analysis_session/tests.rs | 2 + .../tests/test_native_dialect.rs | 38 + .../analysis_session/tests/test_types.rs | 6 + .../project/main/configured_namespace.rs | 9 + .../src/assembly/project/main/mod.rs | 1 + .../src/assembly/project/tests.rs | 2 + .../tests/test_configured_namespace.rs | 39 + .../src/assembly/project/tests/test_types.rs | 6 + .../compile/_helpers/assembly/project.py | 687 ------------------ .../compile/main/_assemble_project.py | 80 -- .../project_assembly/main/_native_resource.py | 16 - 13 files changed, 112 insertions(+), 783 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/assembly/project.py delete mode 100644 src/sqlbuild/compiler/compile/main/_assemble_project.py delete mode 100644 src/sqlbuild/compiler/project_assembly/main/_native_resource.py diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index 6fb1e98127..d4e0b4a64d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -5,6 +5,7 @@ pub mod finished_fact_models; pub mod finished_model_facts; pub mod model_reference_facts; pub mod names_set_operation; +pub mod native_dialect; pub mod projection_facts; pub mod prove_dynamic_contracts; pub mod prove_finished_dynamic_contracts; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs new file mode 100644 index 0000000000..29f8ac170e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/native_dialect.rs @@ -0,0 +1,8 @@ +//! The polyglot dialect name a project's SQL analysis dialect selects. + +use crate::assembly::analysis_session::_helpers::mappings; + +/// Python's `NATIVE_DIALECT_ALIASES.get(dialect or "", dialect)`. +pub fn native_dialect(dialect: Option<&str>) -> Option<&str> { + dialect.map(mappings::native_dialect) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index 0cc551662e..d3db69bdad 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod mappings; #[path = "tests/test_model_columns.rs"] mod model_columns; +#[path = "tests/test_native_dialect.rs"] +mod native_dialect; #[path = "tests/test_reference_facts.rs"] mod reference_facts; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs new file mode 100644 index 0000000000..aae3097136 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_native_dialect.rs @@ -0,0 +1,38 @@ +use crate::assembly::analysis_session::main::native_dialect::native_dialect; +use crate::assembly::analysis_session::tests::test_types::NativeDialectTestCase; + +#[test] +fn given_analysis_dialect_when_resolving_native_dialect_then_matches_python_aliases() { + let test_cases = [ + NativeDialectTestCase { + description: "an aliased dialect maps to its polyglot name", + dialect: Some("postgres"), + expected_dialect: Some("postgresql"), + }, + NativeDialectTestCase { + description: "an unaliased dialect is kept", + dialect: Some("snowflake"), + expected_dialect: Some("snowflake"), + }, + NativeDialectTestCase { + description: "an empty dialect is kept", + dialect: Some(""), + expected_dialect: Some(""), + }, + NativeDialectTestCase { + description: "no dialect stays absent", + dialect: None, + expected_dialect: None, + }, + ]; + + for test_case in test_cases { + let dialect = native_dialect(test_case.dialect); + + assert_eq!( + dialect, test_case.expected_dialect, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 8a81d47677..a25f7ee25e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -177,3 +177,9 @@ pub(crate) struct ReferenceFactsTestCase { pub(crate) expected_lineage: &'static [(&'static str, &'static str, &'static str)], pub(crate) expected_required: &'static [&'static str], } + +pub(crate) struct NativeDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) expected_dialect: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs b/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs new file mode 100644 index 0000000000..ef7ad6a4b2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/configured_namespace.rs @@ -0,0 +1,9 @@ +//! `_str_or_none`: a configured database or schema name, stripped, when it names one. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Python's `value.strip() if value.strip() else None` for a string `value`. +pub fn configured_namespace(value: &str) -> Option { + let stripped: &str = python_strip(value); + (!stripped.is_empty()).then(|| stripped.to_owned()) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index 9cbfd9734e..9ac75e428e 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -1,5 +1,6 @@ pub mod assemble_project_resources; pub mod check_sql_syntax; +pub mod configured_namespace; pub mod environment_template_names; pub mod preserved_logical_namespace; pub mod sql_syntax_error; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs index 48e3358156..3ac1cfe81c 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_configured_namespace.rs"] +mod configured_namespace; #[path = "tests/test_environment_names.rs"] mod environment_names; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs new file mode 100644 index 0000000000..b417746180 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_configured_namespace.rs @@ -0,0 +1,39 @@ +use crate::assembly::project::main::configured_namespace::configured_namespace; +use crate::assembly::project::tests::test_types::ConfiguredNamespaceTestCase; + +#[test] +fn given_configured_value_when_reading_namespace_then_matches_python_strip() { + let test_cases = [ + ConfiguredNamespaceTestCase { + description: "surrounding whitespace is stripped", + value: " \tanalytics\n", + expected_namespace: Some("analytics"), + }, + ConfiguredNamespaceTestCase { + description: "a blank value names no namespace", + value: " \u{3000}\u{1c} ", + expected_namespace: None, + }, + ConfiguredNamespaceTestCase { + description: "an empty value names no namespace", + value: "", + expected_namespace: None, + }, + ConfiguredNamespaceTestCase { + description: "inner whitespace is kept", + value: "\u{a0}sales data\u{2028}", + expected_namespace: Some("sales data"), + }, + ]; + + for test_case in test_cases { + let namespace = configured_namespace(test_case.value); + + assert_eq!( + namespace.as_deref(), + test_case.expected_namespace, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index 88b3291f57..5d6acb6b71 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -86,3 +86,9 @@ pub(crate) struct EnvironmentNamesTestCase { pub(crate) values: &'static [Option<&'static str>], pub(crate) expected_names: &'static [&'static str], } + +pub(crate) struct ConfiguredNamespaceTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_namespace: Option<&'static str>, +} diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py deleted file mode 100644 index 052823009c..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ /dev/null @@ -1,687 +0,0 @@ -"""Assemble planner-ready compiled resource objects from compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows import ( - infer_native_query_column_rows, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeColumnQuery, - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.analysis_session.types import ColumnRow, DiagnosticRow -from sqlbuild.compiler.compile._helpers.analysis.cache import ( - build_analysis_cache_context, -) -from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( - model_placeholders as _model_placeholders, -) -from sqlbuild.compiler.compile._helpers.assembly.hand_built_models import ( - hand_built_compiled_model, -) -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) -from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics -from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors -from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( - analyze_model_sql, - expression_source_shapes, - native_project_facts, -) -from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( - assemble_sql_tests, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import ( - resolve_early_model_templates, -) -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, - get_validated_model_cursor_intrinsics, -) -from sqlbuild.compiler.compile._helpers.render.declarations import build_declaration_scope_resolver -from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros, - find_macro_call_names, -) -from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( - scope_index_with_compile_usages, -) -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompiledObjectKey, - CompiledProject, - CompiledSqlScenario, - CompileModelSqlTestInputPayload, - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, - DeclarationScopeResolver, - HandBuiltAnalysis, - MacroContext, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, - CompileInputsView, - ModelInput, - SourceInput, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.profiling.main._record_cpu import record_compile_cpu_timing -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.project_assembly.constants import AUDIT_SLOT, SEED_SLOT, SOURCE_SLOT -from sqlbuild.compiler.project_assembly.main._native_resource import native_resource -from sqlbuild.compiler.project_assembly.models import ( - NativeModelFacts, - NativeProjectFacts, - NativeProjectResources, -) -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.sql_analysis.constants import ( - NATIVE_DIALECT_ALIASES, - NATIVE_FAILURE_PREFIX, -) -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog -from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) -from sqlbuild.spec.contracts.models import ( - SchemaDynamicColumnFamily, -) - - -@dataclass(frozen=True) -class _DynamicContractAnalysisInputs: - families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] - authoritative_column_types_by_table: dict[str, dict[str, str]] - - -def assemble_compiled_project( - *, - inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile | None = None, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - analysis_cache_dir: Path | None = None, - analysis_model_names: frozenset[str] | None = None, -) -> CompiledProjectView: - """Convert attached compile inputs into the planner-ready project view.""" - - sql_analysis_enabled: bool = ( - inputs.effective_settings.sql_analysis and not skip_column_inference - ) - column_types_by_table: dict[str, dict[str, str]] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - complete_binding_schemas: dict[str, dict[str, str]] - column_types_by_table, column_nullability_by_table, complete_binding_schemas = ( - _native.declared_relations(inputs) - ) - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( - _build_dynamic_families_by_table(inputs) - ) - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - profile = replace( - profile, - semantic_known_functions=known_function_names(inputs.sql_function_inputs), - semantic_known_types=known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - quoted_identifiers_ignore_case=ignores_quoted_case( - connection=inputs.effective_connection, dialect=profile.sql_analysis_dialect - ), - sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( - profile.sql_analysis_dialect or "", profile.sql_analysis_dialect - ), - function_return_types={ - **profile.function_return_types, - **{ - f"__sqlbuild_udf_{function.name}".upper(): function.returns - for function in inputs.sql_function_inputs - if not function.return_columns - }, - }, - ) - binding_catalog: Any = create_binding_catalog( - dialect=profile.sql_analysis_dialect or "generic", - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=profile.semantic_known_functions, - known_types=profile.semantic_known_types, - relations={}, - ) - profile = replace(profile, binding_catalog=binding_catalog) - if sql_analysis_enabled: - expression_sources: tuple[SourceInput, ...] = tuple( - source_input - for source_input in inputs.source_inputs - if source_input.source_entry.expression - ) - expression_shapes: tuple[dict[str, str] | None, ...] = expression_source_shapes( - expressions=tuple( - cast(str, source_input.source_entry.expression) - for source_input in expression_sources - ), - profile=profile, - ) - for source_input, shape in zip(expression_sources, expression_shapes, strict=True): - if shape is not None: - declared_types: dict[str, str] = column_types_by_table.get( - source_input.source_entry.name, {} - ) - declared_shape: dict[str, str] = { - name: declared_types.get(name, column_type) - for name, column_type in shape.items() - } - complete_binding_schemas[source_input.source_entry.name] = declared_shape - column_types_by_table[source_input.source_entry.name] = declared_shape - column_nullability_by_table[source_input.source_entry.name] = { - name: column_nullability_by_table.get(source_input.source_entry.name, {}).get( - name, InferredNullability.UNKNOWN - ) - for name in declared_shape - } - binding_catalog.native.update_relations(complete_binding_schemas) - binding_catalog.schemas.update(complete_binding_schemas) - dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs = ( - _DynamicContractAnalysisInputs( - families_by_table=dynamic_families_by_table, - authoritative_column_types_by_table=complete_binding_schemas, - ) - ) - allow_compact_analysis: bool = column_lineage_mode in { - ColumnLineageMode.FAST, - ColumnLineageMode.RICH, - } - rich_type_inference: bool = column_lineage_mode == ColumnLineageMode.RICH - analysis_cache: AnalysisCacheContext | None = ( - build_analysis_cache_context( - root=analysis_cache_dir, - inference_profile=profile, - allow_compact_analysis=allow_compact_analysis, - rich_type_inference=rich_type_inference, - signature_namespace={ - "known_functions": known_function_names(inputs.sql_function_inputs), - "known_types": known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - "target": inputs.effective_target_name, - "vars": inputs.effective_vars, - }, - ) - if sql_analysis_enabled and analysis_model_names != frozenset() - else None - ) - analyses: NativeModelAnalyses | None = None - if sql_analysis_enabled: - with ( - record_compile_timing("model_analysis_ms"), - record_compile_cpu_timing("model_analysis_cpu_ms"), - ): - analyses = analyze_model_sql( - NativeModelAnalysisRequest( - known_functions=known_function_names(inputs.sql_function_inputs), - known_types=known_declared_types( - functions=inputs.sql_function_inputs, column_types=column_types_by_table - ), - model_inputs=tuple( - model_input - for model_input in inputs.model_inputs - if model_input.sql_validation_enabled - and ( - analysis_model_names is None - or _model_name(model_input) in analysis_model_names - ) - ), - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - inference_profile=profile, - allow_compact_analysis=allow_compact_analysis, - rich_type_inference=rich_type_inference, - analysis_cache=analysis_cache, - complete_binding_schemas=complete_binding_schemas, - dynamic_families_by_table=dynamic_families_by_table, - ) - ) - native_session: Any | None = None if analyses is None else analyses.session - native: NativeProjectFacts = native_project_facts( - inputs=inputs, - dialect=profile.sql_analysis_dialect, - analysis_model_names=analysis_model_names, - analyses=analyses, - session=native_session, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=complete_binding_schemas, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ) - resources: NativeProjectResources = native.resources - scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) - report_scope_index_errors(index=scope_index) - effective_target_values: dict[str, object] = resolve_early_model_templates( - values={ - "database": ( - (inputs.effective_target.database if inputs.effective_target is not None else None) - or _connection_database_fallback(inputs=inputs) - ), - "schema": ( - (inputs.effective_target.schema if inputs.effective_target is not None else None) - or _str_or_none(inputs.effective_connection.get("schema")) - ), - }, - effective_vars=inputs.effective_vars, - effective_target_name=inputs.effective_target_name, - run_id=inputs.run_id, - ) - models: tuple[CompiledModelView, ...] = tuple( - _assemble_compiled_model( - model_input=model_input, - sql_analysis_enabled=( - sql_analysis_enabled - and model_input.sql_validation_enabled - and ( - analysis_model_names is None or _model_name(model_input) in analysis_model_names - ) - ), - sql_validation_enabled=( - analysis_model_names is None or _model_name(model_input) in analysis_model_names - ), - native_model=native.models[index], - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, - inference_profile=profile, - analyses=analyses, - allow_compact_analysis=allow_compact_analysis, - ) - for index, model_input in enumerate(inputs.model_inputs) - ) - fields: dict[str, Any] = dict( - binding_catalog=profile.binding_catalog, - run_id=inputs.run_id, - effective_target_name=inputs.effective_target_name, - effective_connection=inputs.effective_connection, - effective_vars=inputs.effective_vars, - effective_target_database=_str_or_none(effective_target_values.get("database")), - effective_target_schema=_str_or_none(effective_target_values.get("schema")), - sql_analysis_dialect=profile.sql_analysis_dialect, - sql_lexical_syntax=inputs.sql_lexical_syntax, - compile_cache_dir=inputs.compile_cache_dir, - settings=inputs.effective_settings, - scenario=resolve_effective_scenario_config( - project_config=inputs.project_config, - local_config=inputs.local_config, - ), - models=models, - sources=_compiled_sources(inputs=inputs, resources=resources), - seeds=_compiled_seeds(inputs=inputs, resources=resources), - functions=_compiled_functions(inputs=inputs, resources=resources), - audits=_compiled_audits(inputs=inputs, resources=resources), - sql_tests=assemble_sql_tests( - inputs=inputs, - macro_mock_queries=lambda test_input: _build_test_model_query_overrides( - test_input=test_input, model_inputs=inputs.model_inputs, inputs=inputs - ), - ), - sql_scenarios=tuple( - _assemble_compiled_sql_scenario(scenario_input) - for scenario_input in inputs.scenario_inputs - ), - loader_functions=inputs.discovered_inputs.loader_functions, - hook_functions=inputs.discovered_inputs.hook_functions, - enforce_explicit_references=inputs.project_config.references.enforce_explicit, - sql_hook_files=inputs.discovered_inputs.sql_hook_files, - materialization_files=inputs.discovered_inputs.materialization_files, - public_enums=inputs.public_enums, - public_constants=inputs.public_constants, - loaded_macros=inputs.loaded_macros, - diagnostics=( - *inputs.diagnostics, - *_project_binding_diagnostics(models=models, analyses=analyses), - ), - external_sql_reference_resolver=inputs.external_sql_reference_resolver, - scope_index=scope_index, - ) - project: CompiledProjectView = _assembled_project(inputs=inputs, fields=fields) - return complete_semantic_diagnostics( - project=project, - profile=profile, - resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, - native_session=native_session, - binding_results=_binding_rows(analyses), - ) - - -def _assembled_project(*, inputs: CompileInputsView, fields: dict[str, Any]) -> CompiledProjectView: - """The native owner's project view when it holds every model; else a `CompiledProject`.""" - - models: tuple[CompiledModelView, ...] = fields["models"] - if inputs.native_project is not None and all( - isinstance(model, _native.NativeCompiledModel) for model in models - ): - return inputs.native_project.project_view( - {**fields, "unmatched_literal_sql_relations": (), "native_session": None} - ) - return CompiledProject( - **fields, - sql_expansions={ - model_input.model_file.file_path: model_input.sql_expansion - for model_input in inputs.model_inputs - if model_input.sql_expansion is not None - }, - ) - - -def _str_or_none(value: object) -> str | None: - return value.strip() if isinstance(value, str) and value.strip() else None - - -def _connection_database_fallback(*, inputs: CompileInputsView) -> str | None: - adapter_name: str = resolve_effective_adapter_name( - project_config=inputs.project_config, - local_config=inputs.local_config, - ) - if adapter_name == BuiltinAdapter.DUCKDB: - return None - return _str_or_none(inputs.effective_connection.get("database")) - - -def _assemble_compiled_model( - *, - model_input: ModelInput, - sql_analysis_enabled: bool, - sql_validation_enabled: bool = True, - native_model: NativeModelFacts, - column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, - column_types_by_table: dict[str, dict[str, str]] | None = None, - dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, - inference_profile: ExpressionInferenceProfile | None = None, - analyses: NativeModelAnalyses | None = None, - allow_compact_analysis: bool = False, -) -> CompiledModelView: - del column_types_by_table, dynamic_contract_analysis_inputs, allow_compact_analysis - model_name: str = model_input.model_file.file_path.stem - syntax_validated: bool = native_model.syntax_valid - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - analysis_query_sql: str = cursor_intrinsics_analysis_sql( - sql=model_input.query_sql, - cursor_type=model_input.config.values.get("cursor_type"), - ) - placeholders: dict[str, str] | None = _model_placeholders(model_input) - if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: - for hook_name in ("pre_hooks", "post_hooks"): - _native.validate_hook_sql_syntax( - model_input.config.values.get(hook_name), - hook_name, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - session_analysis: tuple[_native.NativeModelAnalysisSession, int] | None = None - fallback: list[ColumnRow] | None = None - if sql_analysis_enabled: - position: int | None = None if analyses is None else analyses.positions.get(model_name) - if analyses is None or position is None: - raise _native.NativeCompilerError( - f"{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no " - f"analysis for model '{model_name}'" - ) - session_analysis = ( - cast(_native.NativeModelAnalysisSession, analyses.session), - position, - ) - if model_name not in analyses.succeeded: - if model_input.sql_validation_enabled and not syntax_validated: - _native.validate_sql_syntax( - analysis_query_sql, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - fallback = infer_native_query_column_rows( - queries=( - NativeColumnQuery( - sql=analysis_query_sql, mode="parse", placeholders=placeholders - ), - ), - profile=profile, - column_nullability_by_table=column_nullability_by_table, - )[0][1] - elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: - _native.validate_sql_syntax( - analysis_query_sql, - (model_name, model_input.model_file.file_path), - (placeholders, profile.sql_analysis_dialect), - ) - macro_deps: tuple[str, ...] = model_input.macro_deps or find_macro_call_names( - model_input.macro_source_sql - ) - if isinstance(model_input, _native.NativeModelInput): - return model_input.compiled_model( - ( - native_model.deps, - session_analysis, - fallback, - native_model.dynamic_contract, - list(macro_deps), - ) - ) - return hand_built_compiled_model( - model_input=model_input, - deps=native_model.deps, - analysis=HandBuiltAnalysis( - session=session_analysis, - fallback=fallback, - standalone_proof=native_model.dynamic_contract, - ), - macro_deps=macro_deps, - ) - - -def _model_name(model_input: ModelInput) -> str: - return model_input.model_file.file_path.stem - - -def _build_dynamic_families_by_table( - inputs: CompileInputsView, -) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: - return { - model_input.schema_entry.name: model_input.schema_entry.dynamic_columns - for model_input in inputs.model_inputs - if model_input.config.values.get("contract") == ContractPolicy.ENFORCED - and model_input.schema_entry is not None - and model_input.schema_entry.dynamic_columns - } - - -def _project_binding_diagnostics( - *, models: tuple[CompiledModelView, ...], analyses: NativeModelAnalyses | None -) -> tuple[CompilerDiagnostic, ...]: - """Every analysed model's located binding diagnostics, which the project also reports.""" - - if analyses is None: - return () - diagnostics: list[CompilerDiagnostic] = [] - for model in models: - if model.name in analyses.positions: - diagnostics.extend(model.binding_diagnostics) - return tuple(diagnostics) - - -def _binding_rows(analyses: NativeModelAnalyses | None) -> dict[str, list[DiagnosticRow]]: - """Each analysed model's raw binding rows, which semantic type recovery identifies.""" - - if analyses is None or analyses.session is None: - return {} - rows: list[list[DiagnosticRow]] = analyses.session.binding_rows() - return {name: rows[position] for name, position in analyses.positions.items()} - - -def _compiled_sources( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledSourceView, ...]: - """Each source's view over the compile's native project, raising assembly's error there.""" - - if not inputs.source_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_source_namespaces(list(resources.source_namespaces)) - sources: list[CompiledSourceView] = [] - for index in range(len(inputs.source_inputs)): - _ = native_resource( - resources=resources, slot=SOURCE_SLOT, items=resources.source_namespaces, index=index - ) - sources.append(owner.compiled_source(index)) - return tuple(sources) - - -def _compiled_seeds( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledSeedView, ...]: - """Each seed's view over the compile's native project, raising assembly's error at its seed.""" - - if not inputs.seed_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_seed_namespaces(list(resources.seed_namespaces)) - seeds: list[CompiledSeedView] = [] - for index in range(len(inputs.seed_inputs)): - _ = native_resource( - resources=resources, slot=SEED_SLOT, items=resources.seed_namespaces, index=index - ) - seeds.append(owner.compiled_seed(index)) - return tuple(seeds) - - -def _compiled_functions( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledFunctionView, ...]: - """Each function's view over the compile's native project, with its resolved deps.""" - - if not inputs.sql_function_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_function_deps(list(resources.function_deps)) - return tuple(owner.compiled_function(index) for index in range(len(inputs.sql_function_inputs))) - - -def _compiled_audits( - *, inputs: CompileInputsView, resources: NativeProjectResources -) -> tuple[CompiledAuditView, ...]: - """Each audit's view over the compile's native project, raising assembly's error there.""" - - if not inputs.audit_inputs: - return () - owner: _native.NativeCompiledProject = cast( - _native.NativeCompiledProject, inputs.native_project - ) - owner.record_audit_scope_deps(list(resources.audit_scope_deps)) - audits: list[CompiledAuditView] = [] - for index in range(len(inputs.audit_inputs)): - _ = native_resource( - resources=resources, slot=AUDIT_SLOT, items=resources.audit_scope_deps, index=index - ) - audits.append(owner.compiled_audit(index)) - return tuple(audits) - - -def _build_test_model_query_overrides( - *, - test_input: CompileSqlTestInput, - model_inputs: tuple[ModelInput, ...], - inputs: CompileInputsView, -) -> dict[str, str]: - """Re-expand each model from its pre-macro macro_source_sql with test macro mocks applied.""" - - if not isinstance(test_input.payload, CompileModelSqlTestInputPayload): - return {} - if not test_input.payload.macro_mocks: - return {} - model_macro_context: MacroContext = inputs.macro_context or MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=inputs.project_config, - local_config=inputs.local_config, - ), - sql_analysis_enabled=inputs.effective_settings.sql_analysis, - target_name=inputs.effective_target_name, - vars=inputs.effective_vars, - ) - macro_context: MacroContext = replace(model_macro_context, _enforce_explicit_references=False) - declaration_resolver: DeclarationScopeResolver = build_declaration_scope_resolver( - discovered_inputs=inputs.discovered_inputs, - scope_index=inputs.scope_index, - loaded_macros=inputs.loaded_macros, - lookup=( - inputs.declaration_scope.resolver.lookup - if inputs.declaration_scope is not None - else None - ), - ) - overrides: dict[str, str] = {} - model_input: ModelInput - for model_input in model_inputs: - model_name: str = model_input.model_file.file_path.stem - macro_source_sql: str = model_input.macro_source_sql or model_input.query_sql - overrides[model_name] = get_validated_model_cursor_intrinsics( - sql=expand_sql_macros( - sql=macro_source_sql, - file_path=model_input.model_file.file_path, - loaded_macros=inputs.loaded_macros, - macro_overrides=test_input.payload.macro_mocks, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - ), - config_values=model_input.config.values, - model_name=model_name, - ) - return overrides - - -def _assemble_compiled_sql_scenario( - scenario_input: CompileSqlScenarioInput, -) -> CompiledSqlScenario: - scenario_name: str = scenario_input.scenario_file.name - return CompiledSqlScenario( - key=CompiledObjectKey( - resource_type=CompiledResourceType.SQL_SCENARIO, - name=scenario_name, - ), - name=scenario_name, - scenario_file=scenario_input.scenario_file, - sql_body=scenario_input.sql_body, - authored_ctes=scenario_input.authored_ctes, - expected_ctes=scenario_input.expected_ctes, - assertion_ctes=scenario_input.assertion_ctes, - source_fixture_names=scenario_input.source_fixture_names, - ref_fixture_names=scenario_input.ref_fixture_names, - seed_fixture_names=scenario_input.seed_fixture_names, - dbt_ref_fixture_names=scenario_input.dbt_ref_fixture_names, - expected_model_names=scenario_input.expected_model_names, - assertion_names=scenario_input.assertion_names, - assertion_target_model_names=scenario_input.assertion_target_model_names, - target_model_names=scenario_input.target_model_names, - source_path=scenario_input.scenario_file.relative_path, - ownership_root=scenario_input.scenario_file.ownership_root, - ) diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py deleted file mode 100644 index 1c255587cd..0000000000 --- a/src/sqlbuild/compiler/compile/main/_assemble_project.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Assemble planner-ready compiled project objects.""" - -from __future__ import annotations - -from collections.abc import Callable -from functools import partial -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - with_collected_compile_diagnostics, -) -from sqlbuild.compiler.compile._helpers.explicit_references.checks import ( - explicit_reference_report, -) -from sqlbuild.compiler.compile.models import ( - PythonSqlReferenceReport, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main._record_model_analyses import record_model_analyses -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge - - -@with_collected_compile_diagnostics -def assemble_project( - *, - inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile | None = None, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - analysis_cache_dir: Path | None = None, - analysis_model_names: frozenset[str] | None = None, -) -> CompiledProjectView: - """Convert compile inputs into the planner-ready project view.""" - - assemble: Callable[[], CompiledProjectView] = partial( - assemble_compiled_project, - inputs=inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=analysis_cache_dir, - analysis_model_names=analysis_model_names, - ) - project: CompiledProjectView = _with_native_project( - project=( - run_with_macro_bridge(stage=assemble) if active_macro_bridge() is None else assemble() - ), - native_project=inputs.native_project, - ) - references: PythonSqlReferenceReport = explicit_reference_report( - project=project, - discovered_inputs=inputs.discovered_inputs, - enforce_explicit=inputs.project_config.references.enforce_explicit, - ) - if not references.diagnostics and not references.unmatched: - return project - return project_with( - project=project, - diagnostics=(*project.diagnostics, *references.diagnostics), - unmatched_literal_sql_relations=references.unmatched, - ) - - -def _with_native_project( - *, project: CompiledProjectView, native_project: _native.NativeCompiledProject | None -) -> CompiledProjectView: - """Hand the compile's native project the assembled analyses and keep it on the project.""" - - if native_project is None: - return project - _ = record_model_analyses(project=native_project, models=project.models) - if project.native_project is native_project: - return project - return project_with(project=project, native_project=native_project) diff --git a/src/sqlbuild/compiler/project_assembly/main/_native_resource.py b/src/sqlbuild/compiler/project_assembly/main/_native_resource.py deleted file mode 100644 index 61e833139a..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_native_resource.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Take one resource's native fact, raising the error Python's assembly raises there.""" - -from __future__ import annotations - -from sqlbuild.compiler.project_assembly.models import NativeProjectResources, NativeResourceFailure - - -def native_resource[T]( - *, resources: NativeProjectResources, slot: str, items: tuple[T, ...], index: int -) -> T: - """Return resource `index` of `slot`, raising the error Python's assembly raises there.""" - - failure: NativeResourceFailure | None = resources.failure - if failure is not None and failure.slot == slot and failure.index == index: - raise failure.error - return items[index] From 075fd79f1a714d1509139e17d743dd952ca232b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:16:30 +0100 Subject: [PATCH 13/26] refactor: assemble compiled projects in the native compile driver --- .../_helpers/compile_driver/assembly.rs | 173 +++++++ .../compile_driver/assembly_analysis.rs | 430 ++++++++++++++++++ .../compile_driver/assembly_checks.rs | 188 ++++++++ .../compile_driver/assembly_models.rs | 274 +++++++++++ .../compile_driver/assembly_project.rs | 426 +++++++++++++++++ .../bindings/_helpers/compile_driver/mod.rs | 5 + .../_helpers/compile_driver/project.rs | 4 +- .../_helpers/compile_driver/registration.rs | 3 +- src/sqlbuild/_native.pyi | 10 + .../compile/_helpers/diagnostics/collector.py | 32 +- .../pipeline/_helpers/analysis_selection.py | 4 +- ...ntyped_inputs_cte_facts_and_contracts.json | 72 --- ...tions_untyped_inputs_contracts_pivots.json | 72 --- .../compiler/analysis_session/helpers.py | 31 +- .../test_native_model_analysis.py | 6 +- .../test_native_query_columns.py | 4 +- .../compiler/compiled_project/helpers.py | 10 +- .../compiler/explicit_references/helpers.py | 3 +- .../compiler/project_assembly/helpers.py | 10 +- .../test_native_project_assembly.py | 4 +- .../compiler/semantic_checks/helpers.py | 4 +- .../sqlbuild/integrations/dbt/test_graph.py | 5 +- .../main/test_attempt_import_footprint.py | 2 +- .../compiler/compile/_helpers/helpers.py | 3 +- .../compile/_helpers/test_assembly.py | 17 +- .../_helpers/test_declaration_scopes.py | 3 +- .../compile/_helpers/test_declarations.py | 3 +- .../compile/_helpers/test_model_schemas.py | 3 +- .../_helpers/test_parameterized_sql_tests.py | 6 +- .../sqlbuild/compiler/dag/main/test_build.py | 10 +- .../src/sqlbuild/integrations/dbt/helpers.py | 4 +- 31 files changed, 1578 insertions(+), 243 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs new file mode 100644 index 0000000000..29a3595ccb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -0,0 +1,173 @@ +//! Planner-ready project assembly: diagnostics collection, the macro bridge and explicit references. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; +use crate::bindings::_helpers::compile_driver::project::lineage_mode; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; + +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const RECORD_ANALYSES_MODULE: &str = + "sqlbuild.compiler.compiled_project.main._record_model_analyses"; +const EXPLICIT_REFERENCES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.explicit_references.checks"; +const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; +const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge"; + +const OPTIONAL: [&str; 5] = [ + "inference_profile", + "skip_column_inference", + "column_lineage_mode", + "analysis_cache_dir", + "analysis_model_names", +]; + +/// The keyword arguments of one project assembly. +pub(crate) struct AssemblyRequest { + pub(crate) inputs: Py, + pub(crate) inference_profile: Py, + pub(crate) skip_column_inference: Py, + pub(crate) column_lineage_mode: Py, + pub(crate) analysis_cache_dir: Py, + pub(crate) analysis_model_names: Py, +} + +/// Convert compile inputs into the planner-ready project view. +#[pyfunction] +#[pyo3(signature = (**options))] +fn assemble_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "assemble_project", options, (&["inputs"], &OPTIONAL))?; + let request: AssemblyRequest = AssemblyRequest { + inputs: arguments.value("inputs")?.unbind(), + inference_profile: arguments.value("inference_profile")?.unbind(), + skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), + column_lineage_mode: lineage_mode(&arguments)?.unbind(), + analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), + analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + }; + let collector: Bound<'py, PyAny> = + imported(py, COLLECTOR_MODULE, "collect_compile_diagnostics")?.call0()?; + let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + within(&collector, |violations| { + Ok((referenced_project(py, request)?, violations.clone())) + })?; + let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + if !collected.is_truthy()? { + return Ok(built); + } + let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[("project", built), ("diagnostics", diagnostics.into_any())], + ) +} + +/// `first` followed by `second`, as one tuple. +pub(crate) fn chained<'py>( + first: &Bound<'py, PyAny>, + second: &Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new( + first.py(), + first + .try_iter()? + .chain(second.try_iter()?) + .collect::>>()?, + ) +} + +/// The undecorated `assemble_project`. +fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult> { + let inputs: Bound<'_, PyAny> = request.inputs.bind(py).clone(); + let project: Bound<'_, PyAny> = if imported(py, ACTIVE_BRIDGE_MODULE, "active_macro_bridge")? + .call0()? + .is_none() + { + let stage: Bound<'_, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + assembled_project(arguments.py(), &request).map(Bound::unbind) + }, + )?; + call_with( + &imported(py, RUN_BRIDGE_MODULE, "run_with_macro_bridge")?, + &[("stage", stage.into_any())], + )? + } else { + assembled_project(py, &request)? + }; + let project: Bound<'_, PyAny> = + with_native_project(&project, &inputs.getattr("native_project")?)?; + let references: Bound<'_, PyAny> = call_with( + &imported(py, EXPLICIT_REFERENCES_MODULE, "explicit_reference_report")?, + &[ + ("project", project.clone()), + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ( + "enforce_explicit", + inputs + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + let reference_diagnostics: Bound<'_, PyAny> = references.getattr("diagnostics")?; + let unmatched: Bound<'_, PyAny> = references.getattr("unmatched")?; + if !reference_diagnostics.is_truthy()? && !unmatched.is_truthy()? { + return Ok(project); + } + let diagnostics: Bound<'_, PyTuple> = + chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[ + ("project", project), + ("diagnostics", diagnostics.into_any()), + ("unmatched_literal_sql_relations", unmatched), + ], + ) +} + +/// Hand the compile's native project the assembled analyses and keep it on the project. +fn with_native_project<'py>( + project: &Bound<'py, PyAny>, + native_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + if native_project.is_none() { + return Ok(project.clone()); + } + call_with( + &imported(py, RECORD_ANALYSES_MODULE, "record_model_analyses")?, + &[ + ("project", native_project.clone()), + ("models", project.getattr("models")?), + ], + )?; + if project.getattr("native_project")?.is(native_project) { + return Ok(project.clone()); + } + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[ + ("project", project.clone()), + ("native_project", native_project.clone()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_project, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs new file mode 100644 index 0000000000..3a43e2bbde --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -0,0 +1,430 @@ +//! Project assembly's inference profile, binding catalog, source shapes and model analysis. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; + +use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; + +use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::python_calls::{ + call_with, imported, keywords, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +const NATIVE_DECLARATIONS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.assembly.native_declarations"; +const IDENTIFIER_CASE_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._identifier_case"; +const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; +pub(crate) const NATIVE_STAGES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.native_stages.assembly"; +const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; +pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const PROFILING_CPU_MODULE: &str = "sqlbuild.compiler.profiling.main._record_cpu"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const GENERIC_DIALECT: &str = "generic"; +const UDF_PREFIX: &str = "__sqlbuild_udf_"; + +/// The analysis state every assembled model and the project share. +pub(crate) struct AnalysisContext<'py> { + pub(crate) profile: Bound<'py, PyAny>, + /// Python's `settings.sql_analysis and not skip_column_inference`, as that value. + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) column_types: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) complete_schemas: Bound<'py, PyAny>, + pub(crate) dynamic_families: Bound<'py, PyAny>, + /// The `NativeModelAnalyses`, or `None` when no model was analysed. + pub(crate) analyses: Bound<'py, PyAny>, +} + +/// Whether `model_input` is selected for analysis by `names` (`None` selects every model). +pub(crate) fn selected_for_analysis( + model_input: &Bound<'_, PyAny>, + names: &Bound<'_, PyAny>, +) -> PyResult { + Ok(names.is_none() || names.contains(model_stem(model_input)?)?) +} + +/// The model's name: its file's stem. +pub(crate) fn model_stem<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { + model_input + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem") +} + +/// Python's `left and not right`. +pub(crate) fn and_not<'py>( + left: &Bound<'py, PyAny>, + right: &Bound<'py, PyAny>, +) -> PyResult> { + if !left.is_truthy()? { + return Ok(left.clone()); + } + Ok(PyBool::new(left.py(), !right.is_truthy()?) + .to_owned() + .into_any()) +} + +/// The profile, catalog, declared relations and model analyses of `request`'s project. +pub(crate) fn analysis_context<'py>( + py: Python<'py>, + request: &AssemblyRequest, +) -> PyResult> { + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let sql_analysis_enabled: Bound<'py, PyAny> = and_not( + &inputs + .getattr("effective_settings")? + .getattr("sql_analysis")?, + request.skip_column_inference.bind(py), + )?; + let relations: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "declared_relations")?.call1((inputs,))?; + let column_types: Bound<'py, PyAny> = relations.get_item(0)?; + let nullability: Bound<'py, PyAny> = relations.get_item(1)?; + let complete_schemas: Bound<'py, PyAny> = relations.get_item(2)?; + let dynamic_families: Bound<'py, PyAny> = dynamic_families_by_table(inputs)?.into_any(); + let profile: Bound<'py, PyAny> = + profile_with_catalog(inputs, request.inference_profile.bind(py), &column_types)?; + let mut context: AnalysisContext<'py> = AnalysisContext { + profile, + sql_analysis_enabled, + column_types, + nullability, + complete_schemas, + dynamic_families, + analyses: py.None().into_bound(py), + }; + if context.sql_analysis_enabled.is_truthy()? { + record_expression_shapes(inputs, &context)?; + } + let catalog: Bound<'py, PyAny> = context.profile.getattr("binding_catalog")?; + catalog + .getattr("native")? + .call_method1("update_relations", (&context.complete_schemas,))?; + catalog + .getattr("schemas")? + .call_method1("update", (&context.complete_schemas,))?; + context.analyses = model_analyses(request, &context)?; + Ok(context) +} + +/// `_build_dynamic_families_by_table`: enforced contract models' dynamic column families. +fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let enforced: Bound<'py, PyAny> = + imported(py, PLANNER_TYPES_MODULE, "ContractPolicy")?.getattr("ENFORCED")?; + let families: Bound<'py, PyDict> = PyDict::new(py); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let contract: Bound<'py, PyAny> = model_input + .getattr("config")? + .getattr("values")? + .call_method1("get", ("contract",))?; + if !contract.eq(&enforced)? { + continue; + } + let schema_entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + if schema_entry.is_none() { + continue; + } + let dynamic_columns: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if dynamic_columns.is_truthy()? { + families.set_item(schema_entry.getattr("name")?, dynamic_columns)?; + } + } + Ok(families) +} + +/// Sorted function names and declared types registered with native validation. +pub(crate) fn declared_names<'py>( + inputs: &Bound<'py, PyAny>, + column_types: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = inputs.py(); + let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; + let names: Bound<'py, PyAny> = + imported(py, NATIVE_DECLARATIONS_MODULE, "known_function_names")? + .call1((functions.clone(),))?; + let types: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_DECLARATIONS_MODULE, "known_declared_types")?, + &[ + ("functions", functions), + ("column_types", column_types.clone()), + ], + )?; + Ok((names, types)) +} + +/// The adapter's profile with project declarations, the native dialect and a binding catalog. +fn profile_with_catalog<'py>( + inputs: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + column_types: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let profile: Bound<'py, PyAny> = if inference_profile.is_truthy()? { + inference_profile.clone() + } else { + imported(py, ADAPTER_MODELS_MODULE, "ExpressionInferenceProfile")?.call0()? + }; + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, column_types)?; + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let ignore_case: Bound<'py, PyAny> = call_with( + &imported(py, IDENTIFIER_CASE_MODULE, "ignores_quoted_case")?, + &[ + ("connection", inputs.getattr("effective_connection")?), + ("dialect", dialect.clone()), + ], + )?; + let name: Option = if dialect.is_none() { + None + } else { + Some(dialect.extract()?) + }; + let native: Bound<'py, PyAny> = match native_dialect(name.as_deref()) { + Some(native) if Some(native) != name.as_deref() => PyString::new(py, native).into_any(), + _ => dialect, + }; + let profile: Bound<'py, PyAny> = replace.call( + (profile.clone(),), + Some(&keywords( + py, + &[ + ("semantic_known_functions", known_functions), + ("semantic_known_types", known_types), + ("quoted_identifiers_ignore_case", ignore_case), + ("sql_analysis_dialect", native), + ( + "function_return_types", + function_return_types(inputs, &profile)?.into_any(), + ), + ], + )?), + )?; + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let catalog: Bound<'py, PyAny> = call_with( + &imported(py, BINDING_CATALOG_MODULE, "create_binding_catalog")?, + &[ + ( + "dialect", + if dialect.is_truthy()? { + dialect + } else { + PyString::new(py, GENERIC_DIALECT).into_any() + }, + ), + ( + "quoted_ignore_case", + profile.getattr("quoted_identifiers_ignore_case")?, + ), + ( + "known_functions", + profile.getattr("semantic_known_functions")?, + ), + ("known_types", profile.getattr("semantic_known_types")?), + ("relations", PyDict::new(py).into_any()), + ], + )?; + replace.call( + (profile,), + Some(&keywords(py, &[("binding_catalog", catalog)])?), + ) +} + +/// The profile's function return types with each scalar project function's analysis name. +fn function_return_types<'py>( + inputs: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let types: Bound<'py, PyDict> = PyDict::new(py); + types.call_method1("update", (profile.getattr("function_return_types")?,))?; + let scalar: Bound<'py, PyDict> = PyDict::new(py); + for function in inputs.getattr("sql_function_inputs")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + if function.getattr("return_columns")?.is_truthy()? { + continue; + } + let name: Bound<'py, PyAny> = function.getattr("name")?.str()?.into_any(); + let key: Bound<'py, PyAny> = PyString::new(py, UDF_PREFIX) + .into_any() + .add(name)? + .call_method0("upper")?; + scalar.set_item(key, function.getattr("returns")?)?; + } + types.update(scalar.as_mapping())?; + Ok(types) +} + +/// Record each expression source's analysed shape as its declared relation. +fn record_expression_shapes<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, +) -> PyResult<()> { + let py: Python<'py> = inputs.py(); + let mut sources: Vec> = Vec::new(); + let mut expressions: Vec> = Vec::new(); + for source_input in inputs.getattr("source_inputs")?.try_iter()? { + let source_input: Bound<'py, PyAny> = source_input?; + let expression: Bound<'py, PyAny> = source_input + .getattr("source_entry")? + .getattr("expression")?; + if expression.is_truthy()? { + sources.push(source_input); + expressions.push(expression); + } + } + let shapes: Vec> = call_with( + &imported(py, NATIVE_STAGES_MODULE, "expression_source_shapes")?, + &[ + ("expressions", PyTuple::new(py, expressions)?.into_any()), + ("profile", context.profile.clone()), + ], + )? + .try_iter()? + .collect::>()?; + if shapes.len() != sources.len() { + let relation: &str = if shapes.len() < sources.len() { + "shorter" + } else { + "longer" + }; + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "zip() argument 2 is {relation} than argument 1" + ))); + } + let unknown: Bound<'py, PyAny> = + imported(py, LINEAGE_TYPES_MODULE, "InferredNullability")?.getattr("UNKNOWN")?; + for (source_input, shape) in sources.iter().zip(shapes) { + if shape.is_none() { + continue; + } + let name: Bound<'py, PyAny> = source_input.getattr("source_entry")?.getattr("name")?; + let declared_types: Bound<'py, PyAny> = context + .column_types + .call_method1("get", (&name, PyDict::new(py)))?; + let declared_shape: Bound<'py, PyDict> = PyDict::new(py); + for item in shape.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let column: Bound<'py, PyAny> = item.get_item(0)?; + declared_shape.set_item( + &column, + declared_types.call_method1("get", (&column, item.get_item(1)?))?, + )?; + } + context.complete_schemas.set_item(&name, &declared_shape)?; + context.column_types.set_item(&name, &declared_shape)?; + let nullability: Bound<'py, PyDict> = PyDict::new(py); + for column in declared_shape.keys() { + let known: Bound<'py, PyAny> = context + .nullability + .call_method1("get", (&name, PyDict::new(py)))?; + nullability.set_item(&column, known.call_method1("get", (&column, &unknown))?)?; + } + context.nullability.set_item(&name, nullability)?; + } + Ok(()) +} + +/// Each selected model's native analysis, timed as `model_analysis_ms`, when SQL analysis runs. +fn model_analyses<'py>( + request: &AssemblyRequest, + context: &AnalysisContext<'py>, +) -> PyResult> { + let py: Python<'py> = context.profile.py(); + if !context.sql_analysis_enabled.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); + let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); + let lineage_mode: Bound<'py, PyAny> = imported(py, LINEAGE_TYPES_MODULE, "ColumnLineageMode")?; + let rich: bool = mode.eq(lineage_mode.getattr("RICH")?)?; + let allow_compact: bool = mode.eq(lineage_mode.getattr("FAST")?)? || rich; + let flags: (Bound<'py, PyAny>, Bound<'py, PyAny>) = ( + PyBool::new(py, allow_compact).to_owned().into_any(), + PyBool::new(py, rich).to_owned().into_any(), + ); + let cache: Bound<'py, PyAny> = if names.ne(PyFrozenSet::empty(py)?)? { + analysis_cache(request, context, &flags)? + } else { + py.None().into_bound(py) + }; + let timing: Bound<'py, PyAny> = + imported(py, PROFILING_MODULE, "record_compile_timing")?.call1(("model_analysis_ms",))?; + within(&timing, |_| { + let cpu_timing: Bound<'py, PyAny> = + imported(py, PROFILING_CPU_MODULE, "record_compile_cpu_timing")? + .call1(("model_analysis_cpu_ms",))?; + within(&cpu_timing, |_| { + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, &context.column_types)?; + let mut model_inputs: Vec> = Vec::new(); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + if model_input.getattr("sql_validation_enabled")?.is_truthy()? + && selected_for_analysis(&model_input, names)? + { + model_inputs.push(model_input); + } + } + let analysis_request: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativeModelAnalysisRequest")?, + &[ + ("known_functions", known_functions), + ("known_types", known_types), + ("model_inputs", PyTuple::new(py, model_inputs)?.into_any()), + ("column_nullability_by_table", context.nullability.clone()), + ("column_types_by_table", context.column_types.clone()), + ("inference_profile", context.profile.clone()), + ("allow_compact_analysis", flags.0.clone()), + ("rich_type_inference", flags.1.clone()), + ("analysis_cache", cache.clone()), + ("complete_binding_schemas", context.complete_schemas.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + imported(py, NATIVE_STAGES_MODULE, "analyze_model_sql")?.call1((analysis_request,)) + }) + }) +} + +/// The analysis cache context keyed by the project's declarations, target and vars. +fn analysis_cache<'py>( + request: &AssemblyRequest, + context: &AnalysisContext<'py>, + (allow_compact, rich): &(Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = context.profile.py(); + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + declared_names(inputs, &context.column_types)?; + let namespace: Bound<'py, PyDict> = keywords( + py, + &[ + ("known_functions", known_functions), + ("known_types", known_types), + ("target", inputs.getattr("effective_target_name")?), + ("vars", inputs.getattr("effective_vars")?), + ], + )?; + call_with( + &imported(py, ANALYSIS_CACHE_MODULE, "build_analysis_cache_context")?, + &[ + ("root", request.analysis_cache_dir.bind(py).clone()), + ("inference_profile", context.profile.clone()), + ("allow_compact_analysis", allow_compact.clone()), + ("rich_type_inference", rich.clone()), + ("signature_namespace", namespace.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs new file mode 100644 index 0000000000..dd0c9ae3ab --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -0,0 +1,188 @@ +//! SQL test macro-mock model queries and compiled SQL scenarios of an assembled project. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MODULE; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +/// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. +const SCENARIO_FIELDS: [&str; 12] = [ + "sql_body", + "authored_ctes", + "expected_ctes", + "assertion_ctes", + "source_fixture_names", + "ref_fixture_names", + "seed_fixture_names", + "dbt_ref_fixture_names", + "expected_model_names", + "assertion_names", + "assertion_target_model_names", + "target_model_names", +]; + +/// Re-expand each model from its pre-macro SQL with the test's macro mocks applied. +pub(crate) fn test_model_query_overrides<'py>( + test_input: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let overrides: Bound<'py, PyDict> = PyDict::new(py); + let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + if !payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompileModelSqlTestInputPayload", + )?)? { + return Ok(overrides); + } + let macro_mocks: Bound<'py, PyAny> = payload.getattr("macro_mocks")?; + if !macro_mocks.is_truthy()? { + return Ok(overrides); + } + let macro_context: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?.call( + (model_macro_context(inputs)?,), + Some(&keywords( + py, + &[( + "_enforce_explicit_references", + PyBool::new(py, false).to_owned().into_any(), + )], + )?), + )?; + let declaration_scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let lookup: Bound<'py, PyAny> = if declaration_scope.is_none() { + py.None().into_bound(py) + } else { + declaration_scope.getattr("resolver")?.getattr("lookup")? + }; + let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; + let declaration_resolver: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, + &[ + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ("scope_index", inputs.getattr("scope_index")?), + ("loaded_macros", loaded_macros.clone()), + ("lookup", lookup), + ], + )?; + let expand: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "expand_sql_macros")?; + let intrinsics: Bound<'py, PyAny> = imported( + py, + CURSOR_INTRINSICS_MODULE, + "get_validated_model_cursor_intrinsics", + )?; + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let file_path: Bound<'py, PyAny> = + model_input.getattr("model_file")?.getattr("file_path")?; + let model_name: Bound<'py, PyAny> = file_path.getattr("stem")?; + let macro_source_sql: Bound<'py, PyAny> = model_input.getattr("macro_source_sql")?; + let macro_source_sql: Bound<'py, PyAny> = if macro_source_sql.is_truthy()? { + macro_source_sql + } else { + model_input.getattr("query_sql")? + }; + let expanded: Bound<'py, PyAny> = call_with( + &expand, + &[ + ("sql", macro_source_sql), + ("file_path", file_path), + ("loaded_macros", loaded_macros.clone()), + ("macro_overrides", macro_mocks.clone()), + ("macro_context", macro_context.clone()), + ("declaration_resolver", declaration_resolver.clone()), + ], + )?; + let query: Bound<'py, PyAny> = call_with( + &intrinsics, + &[ + ("sql", expanded), + ( + "config_values", + model_input.getattr("config")?.getattr("values")?, + ), + ("model_name", model_name.clone()), + ], + )?; + overrides.set_item(model_name, query)?; + } + Ok(overrides) +} + +/// The compile's macro context, or one built from the effective adapter, settings and target. +fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let macro_context: Bound<'py, PyAny> = inputs.getattr("macro_context")?; + if macro_context.is_truthy()? { + return Ok(macro_context); + } + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", inputs.getattr("project_config")?), + ("local_config", inputs.getattr("local_config")?), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ("adapter_name", adapter_name), + ( + "sql_analysis_enabled", + inputs + .getattr("effective_settings")? + .getattr("sql_analysis")?, + ), + ("target_name", inputs.getattr("effective_target_name")?), + ("vars", inputs.getattr("effective_vars")?), + ], + ) +} + +/// Each scenario input as a `CompiledSqlScenario`. +pub(crate) fn compiled_sql_scenarios<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let scenario_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompiledSqlScenario")?; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?; + let resource_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_SCENARIO")?; + let mut scenarios: Vec> = Vec::new(); + for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { + let scenario_input: Bound<'py, PyAny> = scenario_input?; + let scenario_file: Bound<'py, PyAny> = scenario_input.getattr("scenario_file")?; + let name: Bound<'py, PyAny> = scenario_file.getattr("name")?; + let key: Bound<'py, PyAny> = call_with( + &key_class, + &[ + ("resource_type", resource_type.clone()), + ("name", name.clone()), + ], + )?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ("key", key), + ("name", name), + ("scenario_file", scenario_file.clone()), + ], + )?; + for field in SCENARIO_FIELDS { + fields.set_item(field, scenario_input.getattr(field)?)?; + } + fields.set_item("source_path", scenario_file.getattr("relative_path")?)?; + fields.set_item("ownership_root", scenario_file.getattr("ownership_root")?)?; + scenarios.push(scenario_class.call((), Some(&fields))?); + } + PyTuple::new(py, scenarios) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs new file mode 100644 index 0000000000..96aa7a4797 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -0,0 +1,274 @@ +//! Each compiled model of an assembled project, with its syntax checks and analysis fallback. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_driver::assembly_analysis::{ + ANALYSIS_MODELS_MODULE, AnalysisContext, model_stem, selected_for_analysis, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SYNTAX_CHECKS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.syntax_checks"; +const CURSOR_INTRINSICS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const INFER_ROWS_MODULE: &str = + "sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows"; +const HAND_BUILT_MODULE: &str = "sqlbuild.compiler.compile._helpers.assembly.hand_built_models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; +const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; + +/// The Python stages each model's assembly calls, looked up once per project. +struct ModelStages<'py> { + analysis_sql: Bound<'py, PyAny>, + placeholders: Bound<'py, PyAny>, + validate_hooks: Bound<'py, PyAny>, + validate_sql: Bound<'py, PyAny>, + macro_calls: Bound<'py, PyAny>, + native_input: Bound<'py, PyAny>, +} + +/// One model's assembly inputs: the attached model, its native facts and its two gates. +struct ModelRequest<'a, 'py> { + model_input: &'a Bound<'py, PyAny>, + native_model: Bound<'py, PyAny>, + sql_analysis_enabled: bool, + sql_validation_enabled: bool, +} + +/// Every model of the project, compiled in input order. +pub(crate) fn compiled_models<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + (native_models, names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let native: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.into_any(); + let stages: ModelStages<'py> = ModelStages { + analysis_sql: imported( + py, + CURSOR_INTRINSICS_MODULE, + "cursor_intrinsics_analysis_sql", + )?, + placeholders: imported(py, SYNTAX_CHECKS_MODULE, "model_placeholders")?, + validate_hooks: native.getattr("validate_hook_sql_syntax")?, + validate_sql: native.getattr("validate_sql_syntax")?, + macro_calls: imported(py, MACROS_MODULE, "find_macro_call_names")?, + native_input: native.getattr("NativeModelInput")?, + }; + let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; + let mut models: Vec> = Vec::new(); + for (index, model_input) in inputs.getattr("model_inputs")?.try_iter()?.enumerate() { + let model_input: Bound<'py, PyAny> = model_input?; + let sql_analysis_enabled: bool = analysis_enabled + && model_input.getattr("sql_validation_enabled")?.is_truthy()? + && selected_for_analysis(&model_input, names)?; + let request: ModelRequest<'_, 'py> = ModelRequest { + model_input: &model_input, + sql_analysis_enabled, + sql_validation_enabled: selected_for_analysis(&model_input, names)?, + native_model: native_models.get_item(index)?, + }; + models.push(compiled_model(&stages, context, &request)?); + } + PyTuple::new(py, models) +} + +/// `_assemble_compiled_model`. +fn compiled_model<'py>( + stages: &ModelStages<'py>, + context: &AnalysisContext<'py>, + request: &ModelRequest<'_, 'py>, +) -> PyResult> { + let model_input: &Bound<'py, PyAny> = request.model_input; + let py: Python<'py> = model_input.py(); + let model_name: Bound<'py, PyAny> = model_stem(model_input)?; + let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; + let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; + let analysis_query_sql: Bound<'py, PyAny> = call_with( + &stages.analysis_sql, + &[ + ("sql", model_input.getattr("query_sql")?), + ( + "cursor_type", + config_values.call_method1("get", ("cursor_type",))?, + ), + ], + )?; + let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; + let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; + let location: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + model_name.clone(), + model_input.getattr("model_file")?.getattr("file_path")?, + ], + )?; + let syntax: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + placeholders.clone(), + context.profile.getattr("sql_analysis_dialect")?, + ], + )?; + let validates: bool = input_validated && !syntax_validated; + if request.sql_validation_enabled && validates { + for hook_name in HOOK_NAMES { + stages.validate_hooks.call1(( + config_values.call_method1("get", (hook_name,))?, + hook_name, + &location, + &syntax, + ))?; + } + } + let mut session_analysis: Bound<'py, PyAny> = py.None().into_bound(py); + let mut fallback: Bound<'py, PyAny> = py.None().into_bound(py); + if request.sql_analysis_enabled { + let analyses: &Bound<'py, PyAny> = &context.analyses; + let position: Bound<'py, PyAny> = if analyses.is_none() { + py.None().into_bound(py) + } else { + analyses + .getattr("positions")? + .call_method1("get", (&model_name,))? + }; + if analyses.is_none() || position.is_none() { + return Err(NativeCompilerError::new_err(format!( + "{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no \ + analysis for model '{}'", + model_name.str()? + ))); + } + session_analysis = PyTuple::new(py, [analyses.getattr("session")?, position])?.into_any(); + if !analyses.getattr("succeeded")?.contains(&model_name)? { + if validates { + stages + .validate_sql + .call1((&analysis_query_sql, &location, &syntax))?; + } + fallback = fallback_columns(context, &analysis_query_sql, &placeholders)?; + } + } else if request.sql_validation_enabled && validates { + stages + .validate_sql + .call1((&analysis_query_sql, &location, &syntax))?; + } + let declared_macro_deps: Bound<'py, PyAny> = model_input.getattr("macro_deps")?; + let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { + declared_macro_deps + } else { + stages + .macro_calls + .call1((model_input.getattr("macro_source_sql")?,))? + }; + let native_model: &Bound<'py, PyAny> = &request.native_model; + if model_input.is_instance(&stages.native_input)? { + let macro_list: Bound<'py, PyList> = + PyList::new(py, macro_deps.try_iter()?.collect::>>()?)?; + return model_input.call_method1( + "compiled_model", + (( + native_model.getattr("deps")?, + session_analysis, + fallback, + native_model.getattr("dynamic_contract")?, + macro_list, + ),), + ); + } + let analysis: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "HandBuiltAnalysis")?, + &[ + ("session", session_analysis), + ("fallback", fallback), + ( + "standalone_proof", + native_model.getattr("dynamic_contract")?, + ), + ], + )?; + call_with( + &imported(py, HAND_BUILT_MODULE, "hand_built_compiled_model")?, + &[ + ("model_input", model_input.clone()), + ("deps", native_model.getattr("deps")?), + ("analysis", analysis), + ("macro_deps", macro_deps), + ], + ) +} + +/// The columns a parse-only native query inference reads from a model whose analysis failed. +fn fallback_columns<'py>( + context: &AnalysisContext<'py>, + analysis_query_sql: &Bound<'py, PyAny>, + placeholders: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = analysis_query_sql.py(); + let query: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativeColumnQuery")?, + &[ + ("sql", analysis_query_sql.clone()), + ("mode", PyString::new(py, "parse").into_any()), + ("placeholders", placeholders.clone()), + ], + )?; + call_with( + &imported(py, INFER_ROWS_MODULE, "infer_native_query_column_rows")?, + &[ + ("queries", PyTuple::new(py, [query])?.into_any()), + ("profile", context.profile.clone()), + ("column_nullability_by_table", context.nullability.clone()), + ], + )? + .get_item(0)? + .get_item(1) +} + +/// `_project_binding_diagnostics`: every analysed model's located binding diagnostics. +pub(crate) fn project_binding_diagnostics<'py>( + models: &Bound<'py, PyTuple>, + analyses: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + if analyses.is_none() { + return Ok(diagnostics); + } + let positions: Bound<'py, PyAny> = analyses.getattr("positions")?; + for model in models.as_any().try_iter()? { + let model: Bound<'py, PyAny> = model?; + if positions.contains(model.getattr("name")?)? { + for diagnostic in model.getattr("binding_diagnostics")?.try_iter()? { + diagnostics.push(diagnostic?); + } + } + } + Ok(diagnostics) +} + +/// `_binding_rows`: each analysed model's raw binding rows by model name. +pub(crate) fn binding_rows<'py>(analyses: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = analyses.py(); + let rows_by_name: Bound<'py, PyDict> = PyDict::new(py); + if analyses.is_none() { + return Ok(rows_by_name); + } + let session: Bound<'py, PyAny> = analyses.getattr("session")?; + if session.is_none() { + return Ok(rows_by_name); + } + let rows: Bound<'py, PyAny> = session.call_method0("binding_rows")?; + for item in analyses + .getattr("positions")? + .call_method0("items")? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + rows_by_name.set_item(item.get_item(0)?, rows.get_item(item.get_item(1)?)?)?; + } + Ok(rows_by_name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs new file mode 100644 index 0000000000..376a3f1339 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -0,0 +1,426 @@ +//! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; + +use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; + +use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly_analysis::{ + AnalysisContext, NATIVE_STAGES_MODULE, analysis_context, and_not, +}; +use crate::bindings::_helpers::compile_driver::assembly_checks::{ + compiled_sql_scenarios, test_model_query_overrides, +}; +use crate::bindings::_helpers::compile_driver::assembly_models::{ + binding_rows, compiled_models, project_binding_diagnostics, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; +const SCOPE_DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.scope"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +pub(crate) const ADAPTER_NAME_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; +const SCENARIO_CONFIG_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const SQL_TESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.sql_tests"; +const RECOVERY_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.recovery"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SOURCE_SLOT: &str = "source"; +const SEED_SLOT: &str = "seed"; +const AUDIT_SLOT: &str = "audit"; + +/// One native resource kind: its inputs, recorded facts, slot and the owner's methods. +struct ResourceKind<'a> { + inputs: &'a str, + facts: &'a str, + slot: Option<&'a str>, + record: &'a str, + compiled: &'a str, +} + +const SOURCES: ResourceKind<'static> = ResourceKind { + inputs: "source_inputs", + facts: "source_namespaces", + slot: Some(SOURCE_SLOT), + record: "record_source_namespaces", + compiled: "compiled_source", +}; +const SEEDS: ResourceKind<'static> = ResourceKind { + inputs: "seed_inputs", + facts: "seed_namespaces", + slot: Some(SEED_SLOT), + record: "record_seed_namespaces", + compiled: "compiled_seed", +}; +const FUNCTIONS: ResourceKind<'static> = ResourceKind { + inputs: "sql_function_inputs", + facts: "function_deps", + slot: None, + record: "record_function_deps", + compiled: "compiled_function", +}; +const AUDITS: ResourceKind<'static> = ResourceKind { + inputs: "audit_inputs", + facts: "audit_scope_deps", + slot: Some(AUDIT_SLOT), + record: "record_audit_scope_deps", + compiled: "compiled_audit", +}; + +/// Convert attached compile inputs into the planner-ready project view. +pub(crate) fn assembled_project<'py>( + py: Python<'py>, + request: &AssemblyRequest, +) -> PyResult> { + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); + let context: AnalysisContext<'py> = analysis_context(py, request)?; + let native_session: Bound<'py, PyAny> = if context.analyses.is_none() { + py.None().into_bound(py) + } else { + context.analyses.getattr("session")? + }; + let native: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_STAGES_MODULE, "native_project_facts")?, + &[ + ("inputs", inputs.clone()), + ("dialect", context.profile.getattr("sql_analysis_dialect")?), + ("analysis_model_names", names.clone()), + ("analyses", context.analyses.clone()), + ("session", native_session.clone()), + ("column_types_by_table", context.column_types.clone()), + ( + "authoritative_column_types_by_table", + context.complete_schemas.clone(), + ), + ("column_nullability_by_table", context.nullability.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + let resources: Bound<'py, PyAny> = native.getattr("resources")?; + let scope_index: Bound<'py, PyAny> = call_with( + &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, + &[("inputs", inputs.clone())], + )?; + call_with( + &imported(py, SCOPE_DIAGNOSTICS_MODULE, "report_scope_index_errors")?, + &[("index", scope_index.clone())], + )?; + let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; + let models: Bound<'py, PyTuple> = + compiled_models(inputs, &context, (&native.getattr("models")?, names))?; + let fields: Bound<'py, PyDict> = project_fields( + inputs, + &context, + (&models, &resources, &scope_index), + &target_values, + )?; + let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; + let binding_results: Bound<'py, PyDict> = binding_rows(&context.analyses)?; + call_with( + &imported(py, RECOVERY_MODULE, "complete_semantic_diagnostics")?, + &[ + ("project", project), + ("profile", context.profile.clone()), + ( + "resource_sql_analysis", + and_not( + &context.sql_analysis_enabled, + &inputs.getattr("no_sql_validation")?, + )?, + ), + ("native_session", native_session), + ("binding_results", binding_results.into_any()), + ], + ) +} + +/// `value.strip()` when `value` is a string with non-blank text, else `None`. +fn str_or_none<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if let Ok(text) = value.downcast::() + && let Some(stripped) = configured_namespace(&text.to_cow()?) + { + return Ok(PyString::new(py, &stripped).into_any()); + } + Ok(py.None().into_bound(py)) +} + +/// The selected target's `field`, or `None` without a selected target. +fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult> { + let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + if target.is_none() { + return Ok(target); + } + target.getattr(field) +} + +/// The target database and schema, with connection fallbacks, resolved as early model templates. +fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let connection: Bound<'py, PyAny> = inputs.getattr("effective_connection")?; + let database: Bound<'py, PyAny> = target_field(inputs, "database")?; + let database: Bound<'py, PyAny> = if database.is_truthy()? { + database + } else { + connection_database_fallback(inputs, &connection)? + }; + let schema: Bound<'py, PyAny> = target_field(inputs, "schema")?; + let schema: Bound<'py, PyAny> = if schema.is_truthy()? { + schema + } else { + str_or_none(&connection.call_method1("get", ("schema",))?)? + }; + call_with( + &imported( + py, + CONTEXT_TEMPLATES_MODULE, + "resolve_early_model_templates", + )?, + &[ + ( + "values", + keywords(py, &[("database", database), ("schema", schema)])?.into_any(), + ), + ("effective_vars", inputs.getattr("effective_vars")?), + ( + "effective_target_name", + inputs.getattr("effective_target_name")?, + ), + ("run_id", inputs.getattr("run_id")?), + ], + ) +} + +/// The connection's database, except under DuckDB, whose database is a file. +fn connection_database_fallback<'py>( + inputs: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", inputs.getattr("project_config")?), + ("local_config", inputs.getattr("local_config")?), + ], + )?; + if adapter_name.eq(imported(py, ADAPTER_TYPES_MODULE, "BuiltinAdapter")?.getattr("DUCKDB")?)? { + return Ok(py.None().into_bound(py)); + } + str_or_none(&connection.call_method1("get", ("database",))?) +} + +/// The compiled project's fields, in the order Python's assembly built them. +fn project_fields<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + (models, resources, scope_index): ( + &Bound<'py, PyTuple>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + target_values: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let discovered: Bound<'py, PyAny> = inputs.getattr("discovered_inputs")?; + let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let fields: Bound<'py, PyDict> = keywords( + py, + &[ + ( + "binding_catalog", + context.profile.getattr("binding_catalog")?, + ), + ("run_id", inputs.getattr("run_id")?), + ( + "effective_target_name", + inputs.getattr("effective_target_name")?, + ), + ( + "effective_connection", + inputs.getattr("effective_connection")?, + ), + ("effective_vars", inputs.getattr("effective_vars")?), + ( + "effective_target_database", + str_or_none(&target_values.call_method1("get", ("database",))?)?, + ), + ( + "effective_target_schema", + str_or_none(&target_values.call_method1("get", ("schema",))?)?, + ), + ( + "sql_analysis_dialect", + context.profile.getattr("sql_analysis_dialect")?, + ), + ("sql_lexical_syntax", inputs.getattr("sql_lexical_syntax")?), + ("compile_cache_dir", inputs.getattr("compile_cache_dir")?), + ("settings", inputs.getattr("effective_settings")?), + ( + "scenario", + call_with( + &imported( + py, + SCENARIO_CONFIG_MODULE, + "resolve_effective_scenario_config", + )?, + &[ + ("project_config", project_config.clone()), + ("local_config", inputs.getattr("local_config")?), + ], + )?, + ), + ("models", models.clone().into_any()), + ], + )?; + for (name, kind) in [ + ("sources", &SOURCES), + ("seeds", &SEEDS), + ("functions", &FUNCTIONS), + ("audits", &AUDITS), + ] { + fields.set_item(name, compiled_resources(inputs, resources, kind)?)?; + } + let inputs_owned: pyo3::Py = inputs.clone().unbind(); + let overrides: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + let py: Python<'_> = arguments.py(); + test_model_query_overrides(&arguments.get_item(0)?, inputs_owned.bind(py)) + .map(|overrides| overrides.into_any().unbind()) + }, + )?; + fields.set_item( + "sql_tests", + call_with( + &imported(py, SQL_TESTS_MODULE, "assemble_sql_tests")?, + &[ + ("inputs", inputs.clone()), + ("macro_mock_queries", overrides.into_any()), + ], + )?, + )?; + fields.set_item("sql_scenarios", compiled_sql_scenarios(inputs)?)?; + for name in ["loader_functions", "hook_functions"] { + fields.set_item(name, discovered.getattr(name)?)?; + } + fields.set_item( + "enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + )?; + for name in ["sql_hook_files", "materialization_files"] { + fields.set_item(name, discovered.getattr(name)?)?; + } + for name in ["public_enums", "public_constants", "loaded_macros"] { + fields.set_item(name, inputs.getattr(name)?)?; + } + let mut diagnostics: Vec> = inputs + .getattr("diagnostics")? + .try_iter()? + .collect::>()?; + diagnostics.extend(project_binding_diagnostics(models, &context.analyses)?); + fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; + fields.set_item( + "external_sql_reference_resolver", + inputs.getattr("external_sql_reference_resolver")?, + )?; + fields.set_item("scope_index", scope_index)?; + Ok(fields) +} + +/// Each resource's view over the compile's native project, raising assembly's error there. +fn compiled_resources<'py>( + inputs: &Bound<'py, PyAny>, + resources: &Bound<'py, PyAny>, + kind: &ResourceKind<'_>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let count: usize = inputs.getattr(kind.inputs)?.len()?; + if count == 0 { + return Ok(PyTuple::empty(py)); + } + let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let facts: Bound<'py, PyAny> = resources.getattr(kind.facts)?; + owner.call_method1( + kind.record, + (PyList::new( + py, + facts.try_iter()?.collect::>>()?, + )?,), + )?; + let mut compiled: Vec> = Vec::with_capacity(count); + for index in 0..count { + if let Some(slot) = kind.slot { + native_resource(resources, (slot, &facts), index)?; + } + compiled.push(owner.call_method1(kind.compiled, (index,))?); + } + PyTuple::new(py, compiled) +} + +/// Resource `index` of `slot`, raising the error Python's assembly raises there. +fn native_resource<'py>( + resources: &Bound<'py, PyAny>, + (slot, items): (&str, &Bound<'py, PyAny>), + index: usize, +) -> PyResult> { + let failure: Bound<'py, PyAny> = resources.getattr("failure")?; + if !failure.is_none() + && failure.getattr("slot")?.eq(slot)? + && failure.getattr("index")?.eq(index)? + { + return Err(PyErr::from_value(failure.getattr("error")?)); + } + items.get_item(index) +} + +/// The native owner's project view when it holds every model; else a `CompiledProject`. +fn project_view<'py>( + inputs: &Bound<'py, PyAny>, + fields: &Bound<'py, PyDict>, + models: &Bound<'py, PyTuple>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let native_model: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeCompiledModel")?; + let mut all_native: bool = true; + for model in models.as_any().try_iter()? { + if !model?.is_instance(&native_model)? { + all_native = false; + break; + } + } + if !owner.is_none() && all_native { + let view_fields: Bound<'py, PyDict> = fields.copy()?; + view_fields.set_item("unmatched_literal_sql_relations", PyTuple::empty(py))?; + view_fields.set_item("native_session", py.None())?; + return owner.call_method1("project_view", (view_fields,)); + } + let sql_expansions: Bound<'py, PyDict> = PyDict::new(py); + for model_input in inputs.getattr("model_inputs")?.try_iter()? { + let model_input: Bound<'py, PyAny> = model_input?; + let expansion: Bound<'py, PyAny> = model_input.getattr("sql_expansion")?; + if !expansion.is_none() { + sql_expansions.set_item( + model_input.getattr("model_file")?.getattr("file_path")?, + expansion, + )?; + } + } + let arguments: Bound<'py, PyDict> = fields.copy()?; + arguments.set_item("sql_expansions", sql_expansions)?; + imported(py, COMPILE_MODELS_MODULE, "CompiledProject")?.call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 6ff32c54f7..5641901225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,4 +1,9 @@ pub(crate) mod analysis; +pub(crate) mod assembly; +pub(crate) mod assembly_analysis; +pub(crate) mod assembly_checks; +pub(crate) mod assembly_models; +pub(crate) mod assembly_project; pub(crate) mod command; pub(crate) mod discovery; pub(crate) mod inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index cb6aaa799a..797e9b18bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; -const ASSEMBLE_MODULE: &str = "sqlbuild.compiler.compile.main._assemble_project"; +const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_conflicts"; @@ -52,7 +52,7 @@ struct ProjectRequest { } /// `column_lineage_mode`, defaulting to `ColumnLineageMode.FAST` when not passed. -fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { +pub(crate) fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { let mode: Bound<'py, PyAny> = arguments.value("column_lineage_mode")?; if arguments.passed("column_lineage_mode")? { return Ok(mode); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index d82b7caa52..46cbaf85e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ - analysis, command, discovery, inputs, pipeline, project, + analysis, assembly, command, discovery, inputs, pipeline, project, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; + assembly::register(module)?; command::register(module)?; project::register(module)?; inputs::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8f034a0efd..8d24163417 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,6 +5,7 @@ from pathlib import Path, PurePath from typing import Any, Literal, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import CompileAnalysis @@ -976,6 +977,15 @@ def build_compiled_project( resolved_connection: dict[str, object] | None = None, analysis_selection: CompileAnalysisSelection | None = None, ) -> CompiledProjectView: ... +def assemble_project( + *, + inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile | None = None, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + analysis_cache_dir: Path | None = None, + analysis_model_names: frozenset[str] | None = None, +) -> CompiledProjectView: ... def build_project_graph_with_analysis_selection( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index 2242fe26a1..9ec47b3b56 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -2,24 +2,17 @@ from __future__ import annotations -from collections.abc import Callable, Iterator +from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from functools import wraps -from typing import cast -import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( CollectedCompileDiagnostics, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompileProjectInputs, CompilerDiagnostic, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with -from sqlbuild.compiler.compiled_project.main.project_with import project_with _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None @@ -38,29 +31,6 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: _ACTIVE_COLLECTOR.reset(token) -def with_collected_compile_diagnostics[**P, R: (CompileInputsView, CompiledProjectView)]( - build: Callable[P, R], -) -> Callable[P, R]: - """Append every diagnostic reported while ``build`` runs to the diagnostics it returns.""" - - @wraps(build) - def collecting(*args: P.args, **kwargs: P.kwargs) -> R: - with collect_compile_diagnostics() as violations: - built: R = build(*args, **kwargs) - if not violations.diagnostics: - return built - diagnostics: tuple[CompilerDiagnostic, ...] = ( - *built.diagnostics, - *violations.diagnostics, - ) - if isinstance(built, CompileProjectInputs | _native.NativeInputsView): - return cast(R, inputs_with(inputs=built, diagnostics=diagnostics)) - project: CompiledProjectView = cast(CompiledProjectView, built) - return cast(R, project_with(project=project, diagnostics=diagnostics)) - - return collecting - - def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 452f64dafa..75b8a930ef 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -4,8 +4,8 @@ from dataclasses import replace +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, @@ -113,7 +113,7 @@ def resolve_analysis_model_names( if selection is None or (not selection.select and not selection.exclude): return None - structural_project: CompiledProjectView = assemble_project( + structural_project: CompiledProjectView = _native.assemble_project( inputs=compile_inputs, inference_profile=inference_profile, skip_column_inference=True, diff --git a/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json b/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json index 4ec7fcb8f0..3a75d38701 100644 --- a/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json +++ b/tests/goldens/python_oracle/analysis_generated_rich_lineage_over_untyped_inputs_cte_facts_and_contracts.json @@ -564,18 +564,6 @@ "c44c8633473bde950e573766" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1140,18 +1128,6 @@ "f48ca046110efa93dc8e7b54" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1716,18 +1692,6 @@ "7f3cc4630798c6d611e9c144" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2292,18 +2256,6 @@ "767dbad2dc0bb5d013f7a196" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2868,18 +2820,6 @@ "165df7e540546e89f76b0819" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -3442,17 +3382,5 @@ [ "catalog", "8808b0d353112b97c19536d1" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" ] ] diff --git a/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json b/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json index b3c3149315..0fe10d8eb3 100644 --- a/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json +++ b/tests/goldens/python_oracle/analysis_generated_stars_ctes_cte_facts_set_operations_untyped_inputs_contracts_pivots.json @@ -564,18 +564,6 @@ "9c7db1355dd2394241b43e81" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1140,18 +1128,6 @@ "b83e75232474640290815c9d" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -1716,18 +1692,6 @@ "032717ee1f3f581899d54bf7" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2292,18 +2256,6 @@ "3d1560ed382e916606e5f85f" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -2868,18 +2820,6 @@ "f19e292dff42d4c08b5da8ad" ], [ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ "shapes", "78e5cd70ce361d823798634b" ], @@ -3442,17 +3382,5 @@ [ "catalog", "23b5038345131be7d003ad7b" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" -], -[ -"proof", -"74234e98afe7498fb5daf1f3" ] ] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index f08738544b..b919d9b2d3 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -14,7 +14,7 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -61,7 +61,11 @@ InferredColumn, ModelSqlAnalysis, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompileInputsView, ModelInput +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompileInputsView, + ModelInput, +) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from sqlbuild.sql_values.types import CollectionRendering @@ -643,19 +647,20 @@ def recorded_shapes( runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native - assembled_model: Callable[..., CompiledModelView] = project_assembly._assemble_compiled_model + completed: Callable[..., CompiledProjectView] = semantic_recovery.complete_semantic_diagnostics - def recorded_model(*arguments: Any, **keywords: Any) -> CompiledModelView: - model: CompiledModelView = assembled_model(*arguments, **keywords) - runs.golden.append(golden_entry("proof", model.dynamic_column_contract)) - return model + def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: + runs.golden.extend( + golden_entry("proof", model.dynamic_column_contract) for model in project.models + ) + return completed(project=project, **keywords) with monkeypatch.context() as patch: - patch.setattr(project_assembly, "_assemble_compiled_model", recorded_model) - patch.setattr(project_assembly, "analyze_model_sql", analysed_three_ways) - patch.setattr(project_assembly, "expression_source_shapes", recorded_shapes) + patch.setattr(semantic_recovery, "complete_semantic_diagnostics", recorded_proofs) + patch.setattr(native_stage_assembly, "analyze_model_sql", analysed_three_ways) + patch.setattr(native_stage_assembly, "expression_source_shapes", recorded_shapes) with suppress(CompileInputError): - _ = project_assembly.assemble_compiled_project( + _ = native_module.assemble_project( inputs=inputs, inference_profile=replace(inference_profile), column_lineage_mode=lineage_mode, @@ -700,8 +705,8 @@ def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: monkeypatch.context() as patch, suppress(CompileInputError, native_module.NativeCompilerError), ): - patch.setattr(project_assembly, "analyze_model_sql", captured) - _ = project_assembly.assemble_compiled_project( + patch.setattr(native_stage_assembly, "analyze_model_sql", captured) + _ = native_module.assemble_project( inputs=inputs, inference_profile=inference_profile, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 1590999301..5590867d80 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -11,6 +11,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -18,7 +19,6 @@ analyze_native_model_sql, ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( @@ -399,7 +399,7 @@ def test_given_internal_native_failure_when_assembling_then_raises_without_pytho duplicate_analysed_model_names(monkeypatch=monkeypatch) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = assemble_compiled_project( + _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), ) @@ -443,7 +443,7 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ) recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) - project: CompiledProjectView = assemble_compiled_project( + project: CompiledProjectView = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), analysis_model_names=test_case.analysed_models, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index cc21e4bde6..3e507ee1c6 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,7 +12,7 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( infer_native_query_columns, @@ -132,7 +132,7 @@ def test_given_session_without_model_analysis_when_assembling_then_raises_native test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - project_assembly, + native_stage_assembly, "analyze_model_sql", lambda _request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index be816afaab..7b2dfaa4e3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -13,12 +13,16 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( @@ -631,7 +635,7 @@ def _hand_built_model(name: str, sql: str) -> CompileModelInput: def hand_built_analysed_project(models: dict[str, str]) -> CompiledProjectView: """Assemble hand-built model inputs, which cross the explicit analysis boundary.""" - return assemble_compiled_project( + return assemble_project( inputs=CompileProjectInputs( project_config=_HAND_BUILT_CONFIG, local_config=LocalConfig(), diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 6cc274642d..fe1c968921 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -7,9 +7,8 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompilerDiagnostic, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 9d1705a5e4..5b7ea297e8 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,10 +13,8 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.assembly import project as project_module -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -159,7 +157,7 @@ def assemble_with( ) -> CompiledProjectView: """Assemble `inputs`.""" - return assemble_compiled_project(inputs=inputs) + return assemble_project(inputs=inputs) def recorded_assembly( @@ -233,9 +231,9 @@ def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: calls: Counter[str] = Counter() _ = [ monkeypatch.setattr( - project_module._native, + _native, name, - _counted(name=name, function=getattr(project_module._native, name), calls=calls), + _counted(name=name, function=getattr(_native, name), calls=calls), ) for name in _PYTHON_RESOURCE_FUNCTIONS ] diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index c099f7b21f..795b552976 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -10,8 +10,8 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.project_assembly.types import ProjectRequestRow @@ -342,7 +342,7 @@ def test_given_native_assembly_panics_when_assembling_then_raises_without_python calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = assemble_compiled_project( + _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect=test_case.dialect), ) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 207414d63b..578c6a74b6 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,7 +13,7 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -208,7 +208,7 @@ def capture(**arguments: Any) -> CompiledProjectView: return complete_semantic_diagnostics(**arguments) with monkeypatch.context() as patch: - patch.setattr(assembly_project, "complete_semantic_diagnostics", capture) + patch.setattr(semantic_recovery, "complete_semantic_diagnostics", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index e3a0c63ed0..f34f336cba 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,8 +5,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( @@ -83,7 +82,7 @@ def test_given_real_dbt_manifest_and_sqlbuild_project_when_building_graph_then_e manifest_source=manifest_source ), ) - project: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + project: CompiledProjectView = assemble_project(inputs=compile_inputs) manifest: DbtManifestIndex = build_dbt_manifest_index( raw_data=json.loads(manifest_source.read_text(encoding="utf-8")) ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index 6ccf45f542..c6345f23a3 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -16,7 +16,7 @@ _FULL_COMPILE_MODULES: tuple[str, ...] = ( "sqlbuild.cli.commands._helpers.compile.output", - "sqlbuild.compiler.compile.main._assemble_project", + "sqlbuild.compiler.compile._helpers.native_stages.assembly", ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 6c528656b8..e63daf8241 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -5,7 +5,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -24,7 +24,6 @@ expand_sql_macros, load_project_macros, ) -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledDirectLogicSqlTestPayload, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index b28042ddf6..6f0a067c5e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -518,7 +517,7 @@ def test_given_compile_inputs_when_assembling_compiled_project_then_returns_expe adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(m.name for m in compiled.models) == test_case.expected_model_names assert tuple(m.deps for m in compiled.models) == test_case.expected_model_deps @@ -623,7 +622,7 @@ def test_given_named_generic_audit_instances_when_assembling_then_each_name_is_p run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(audit.name for audit in compiled.audits) == test_case.expected_names @@ -696,7 +695,7 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph ) seeds: dict[str, CompiledSeedView] = { - seed.name: seed for seed in assemble_compiled_project(inputs=compile_inputs).seeds + seed.name: seed for seed in assemble_project(inputs=compile_inputs).seeds } assert ( @@ -719,9 +718,7 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, selected_target="physical", ) - physical_seeds: tuple[CompiledSeedView, ...] = assemble_compiled_project( - inputs=physical_inputs - ).seeds + physical_seeds: tuple[CompiledSeedView, ...] = assemble_project(inputs=physical_inputs).seeds assert {(seed.destination.database, seed.destination.schema) for seed in physical_seeds} == { test_case.expected_literal_target_location } @@ -798,7 +795,7 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert compiled.effective_target_database == test_case.expected_effective_target_database assert compiled.effective_target_schema == test_case.expected_effective_target_schema @@ -860,7 +857,7 @@ def test_given_sql_hooks_when_assembling_then_applies_adapter_aware_validation( run_id="test_run_id", ) - compiled: CompiledProjectView = assemble_compiled_project( + compiled: CompiledProjectView = assemble_project( inputs=compile_inputs, inference_profile=ExpressionInferenceProfile( sql_analysis_dialect=test_case.dialect, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index f0e082241b..9d6cdffbaf 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,9 +9,8 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index d18c693fa4..e5a19eb161 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index d693e298df..e900e30506 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -7,10 +7,9 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 359efd5874..6b0e9aed81 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import assemble_project from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -247,7 +247,7 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq ) -> None: write_repo_files(tmp_path, test_case.repo_files) compile_inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) assert tuple(test.name for test in compiled.sql_tests) == test_case.expected_names assert tuple(test.key.name for test in compiled.sql_tests) == test_case.expected_names @@ -289,7 +289,7 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq model_test_path.read_text(encoding="utf-8").replace('amount "2.4700"', 'amount "2.4800"'), encoding="utf-8", ) - changed: CompiledProjectView = assemble_compiled_project( + changed: CompiledProjectView = assemble_project( inputs=compile_project_inputs(project_dir=tmp_path) ) changed_fingerprints: tuple[str | None, ...] = tuple( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 839e69e694..621b130840 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -8,8 +8,12 @@ import pytest -from sqlbuild._native import build_compile_inputs, build_dag_json, discover_project_inputs -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_dag_json, + discover_project_inputs, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -234,7 +238,7 @@ def test_given_resource_namespace_defaults_when_building_dag_then_targets_retain discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) - project: CompiledProjectView = assemble_compiled_project(inputs=inputs) + project: CompiledProjectView = assemble_project(inputs=inputs) graph: ProjectGraph = project_graph_from_indexes( project=project, upstream_deps={}, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 490313af2a..4febaed46a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild._native import assemble_project from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -440,7 +440,7 @@ def build_compiled_project_with_model_specs( references=extract_sql_references(sql=sql, syntax=_GENERIC_SQL_SYNTAX), ) ) - return assemble_compiled_project( + return assemble_project( inputs=CompileProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), From 8bae6dbcd11515225eec8fecae143dd7c9b4b5db Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:35:29 +0100 Subject: [PATCH 14/26] refactor: scan SQL quotes, comments and parentheses natively for reference resolution The planner's reference resolution scans with a native SqlCodeScanner at Python string offsets, mirroring the generic and dialect rules of the deleted Python sql_analysis scanning helpers and their main wrappers. --- .../bindings/_helpers/sqltext/code_scanner.rs | 70 +++++ .../src/bindings/_helpers/sqltext/mod.rs | 1 + .../bindings/_helpers/sqltext/registration.rs | 3 +- .../src/code_scanner/_helpers/generic.rs | 23 ++ .../src/code_scanner/_helpers/mod.rs | 2 + .../src/code_scanner/_helpers/offsets.rs | 23 ++ .../src/code_scanner/main/code_scanner.rs | 22 ++ .../src/code_scanner/main/mod.rs | 3 + .../main/scanner_matching_paren.rs | 39 +++ .../code_scanner/main/scanner_non_code_end.rs | 21 ++ .../sqlbuild-sqltext/src/code_scanner/mod.rs | 6 + .../src/code_scanner/models.rs | 13 + .../src/code_scanner/tests.rs | 6 + .../src/code_scanner/tests/helpers.rs | 11 + .../code_scanner/tests/test_code_scanner.rs | 139 ++++++++++ .../src/code_scanner/tests/test_types.rs | 16 ++ crates/sqlbuild-sqltext/src/lib.rs | 1 + src/sqlbuild/_native.pyi | 5 + .../compiler/planner/_helpers/resolve/refs.py | 60 ++--- .../sql_analysis/_helpers/scanning.py | 245 ------------------ .../compiler/sql_analysis/constants.py | 12 - .../sql_analysis/main/_find_matching_paren.py | 21 -- .../sql_analysis/main/_skip_block_comment.py | 9 - .../main/_skip_dialect_non_code.py | 12 - .../sql_analysis/main/_skip_line_comment.py | 9 - .../sql_analysis/main/_skip_quoted_text.py | 9 - .../sql_analysis/_helpers/_test_types.py | 11 +- .../compiler/sql_analysis/_helpers/helpers.py | 18 ++ .../sql_analysis/_helpers/test_scanning.py | 74 ++---- 29 files changed, 462 insertions(+), 422 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/models.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/tests.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs create mode 100644 crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs delete mode 100644 src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py create mode 100644 tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs new file mode 100644 index 0000000000..cf4ea638ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -0,0 +1,70 @@ +//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_sqltext::code_scanner::main::code_scanner::code_scanner; +use sqlbuild_sqltext::code_scanner::main::scanner_matching_paren::scanner_matching_paren; +use sqlbuild_sqltext::code_scanner::main::scanner_non_code_end::scanner_non_code_end; +use sqlbuild_sqltext::code_scanner::models::CodeScanner; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; + +/// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SqlCodeScanner { + scanner: CodeScanner, + context: String, +} + +#[pymethods] +impl SqlCodeScanner { + #[new] + #[pyo3(signature = (sql, syntax, context))] + fn new(sql: &str, syntax: Option, context: String) -> Self { + Self { + scanner: code_scanner(sql, syntax.map(Into::into)), + context, + } + } + + /// The end of the comment or quoted text at `index`, or `None` when code starts there. + fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { + scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) + } + + /// The index of the parenthesis closing the one at `open`. + fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { + scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) + } +} + +impl SqlCodeScanner { + /// `CompileInputError(" contains an unclosed ...")`. + fn error(&self, py: Python<'_>, unclosed: Unclosed) -> PyErr { + let construct: &str = match unclosed { + Unclosed::BlockComment => "block comment", + Unclosed::Quote => "quoted string", + Unclosed::Parenthesis => "parenthesis", + }; + let message: String = format!("{} contains an unclosed {construct}", self.context); + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 4cd903b971..52ccc56e39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod authored_values; +pub(crate) mod code_scanner; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; pub(crate) mod model_headers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 4fe4b866e4..d94a7da68f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - declaration_references, model_headers, placeholders, sql_references, static_sql, + code_scanner, declaration_references, model_headers, placeholders, sql_references, static_sql, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_headers::register(module)?; + code_scanner::register(module)?; static_sql::register(module)?; sql_references::register(module)?; declaration_references::register(module)?; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs new file mode 100644 index 0000000000..39b1b3ad13 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/generic.rs @@ -0,0 +1,23 @@ +//! The compiler's generic comment and quoted-text rules without dialect information. + +use crate::sql_scan::main::comment_end::comment_end; +use crate::sql_scan::main::non_code_end::non_code_end; +use crate::sql_scan::models::{QuotePolicy, Unclosed}; + +/// Python's generic comment or quoted-text end at `index`; backticks never double, a lone `$` is one character. +pub(crate) fn generic_skip_end(sql: &[u8], index: usize) -> Result, Unclosed> { + match sql.get(index) { + Some(b'`') => sql[index + 1..] + .iter() + .position(|byte| *byte == b'`') + .map(|offset| Some(index + 1 + offset + 1)) + .ok_or(Unclosed::Quote), + Some(b'$') => Ok(Some( + non_code_end(sql, index, QuotePolicy::COMPILER)?.unwrap_or(index + 1), + )), + _ => match comment_end(sql, index)? { + Some(end) => Ok(Some(end)), + None => non_code_end(sql, index, QuotePolicy::COMPILER), + }, + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs new file mode 100644 index 0000000000..a54aea47b0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod generic; +pub(crate) mod offsets; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs new file mode 100644 index 0000000000..faee9bec80 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/_helpers/offsets.rs @@ -0,0 +1,23 @@ +//! Conversions between Python code point offsets and UTF-8 byte offsets. + +use crate::code_scanner::models::CodeScanner; + +/// The byte offset of the code point at `index`, or the text length past the end. +pub(crate) fn byte_offset(scanner: &CodeScanner, index: usize) -> usize { + if scanner.char_bytes.is_empty() { + return index.min(scanner.sql.len()); + } + scanner + .char_bytes + .get(index) + .copied() + .unwrap_or(scanner.sql.len()) +} + +/// The code point offset of a byte offset at a code point boundary. +pub(crate) fn char_offset(scanner: &CodeScanner, byte: usize) -> usize { + if scanner.char_bytes.is_empty() { + return byte; + } + scanner.char_bytes.partition_point(|start| *start < byte) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs new file mode 100644 index 0000000000..4049206d36 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/code_scanner.rs @@ -0,0 +1,22 @@ +//! Prepare one SQL text for scanning at Python string offsets. + +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::models::LexicalSyntax; + +/// A scanner over `sql`, under `syntax` when given and the generic compiler rules otherwise. +#[must_use] +pub fn code_scanner(sql: &str, syntax: Option) -> CodeScanner { + let char_bytes: Vec = if sql.is_ascii() { + Vec::new() + } else { + sql.char_indices() + .map(|(offset, _)| offset) + .chain([sql.len()]) + .collect() + }; + CodeScanner { + sql: sql.to_owned(), + char_bytes, + syntax, + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs new file mode 100644 index 0000000000..60ffc7f155 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/mod.rs @@ -0,0 +1,3 @@ +pub mod code_scanner; +pub mod scanner_matching_paren; +pub mod scanner_non_code_end; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs new file mode 100644 index 0000000000..f708777596 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_matching_paren.rs @@ -0,0 +1,39 @@ +//! The parenthesis closing one opened at a Python string offset. + +use crate::code_scanner::_helpers::generic::generic_skip_end; +use crate::code_scanner::_helpers::offsets::{byte_offset, char_offset}; +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use crate::sql_scan::models::Unclosed; + +/// The code point offset of the parenthesis closing the one at `open`, skipping text and comments. +pub fn scanner_matching_paren(scanner: &CodeScanner, open: usize) -> Result { + let sql: &[u8] = scanner.sql.as_bytes(); + let start: usize = byte_offset(scanner, open); + let close: usize = match &scanner.syntax { + Some(syntax) => dialect_matching_paren(sql, start, syntax)?, + None => generic_matching_paren(sql, start)?, + }; + Ok(char_offset(scanner, close)) +} + +fn generic_matching_paren(sql: &[u8], open: usize) -> Result { + let mut depth: usize = 1; + let mut index: usize = open + 1; + while index < sql.len() { + if let Some(end) = generic_skip_end(sql, index)? { + index = end; + continue; + } + if sql[index] == b'(' { + depth += 1; + } else if sql[index] == b')' { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + index += 1; + } + Err(Unclosed::Parenthesis) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs new file mode 100644 index 0000000000..7de499bf89 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/main/scanner_non_code_end.rs @@ -0,0 +1,21 @@ +//! The end of the comment or quoted text at one Python string offset. + +use crate::code_scanner::_helpers::generic::generic_skip_end; +use crate::code_scanner::_helpers::offsets::{byte_offset, char_offset}; +use crate::code_scanner::models::CodeScanner; +use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use crate::sql_scan::models::Unclosed; + +/// Where the comment or quoted text at code point `index` ends; a generic lone `$` is one character. +pub fn scanner_non_code_end( + scanner: &CodeScanner, + index: usize, +) -> Result, Unclosed> { + let sql: &[u8] = scanner.sql.as_bytes(); + let start: usize = byte_offset(scanner, index); + let end: Option = match &scanner.syntax { + Some(syntax) => dialect_non_code_end(sql, start, syntax)?, + None => generic_skip_end(sql, start)?, + }; + Ok(end.map(|end| char_offset(scanner, end))) +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/mod.rs b/crates/sqlbuild-sqltext/src/code_scanner/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/models.rs b/crates/sqlbuild-sqltext/src/code_scanner/models.rs new file mode 100644 index 0000000000..90e68df56c --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/models.rs @@ -0,0 +1,13 @@ +//! One SQL text scanned for code at Python string offsets. + +use crate::sql_scan::models::LexicalSyntax; + +/// A SQL text, its code point byte offsets and the dialect rules it is scanned under, if any. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CodeScanner { + pub(crate) sql: String, + /// The byte offset of each code point and then the text length; empty for ASCII text. + pub(crate) char_bytes: Vec, + /// Dialect rules; without them the compiler's generic quoting and comments apply. + pub(crate) syntax: Option, +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests.rs new file mode 100644 index 0000000000..a046ce3699 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_code_scanner.rs"] +mod code_scanner; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs new file mode 100644 index 0000000000..efc9231d16 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/helpers.rs @@ -0,0 +1,11 @@ +//! Lexical syntax shared by the code scanner tests. + +use crate::sql_scan::models::LexicalSyntax; + +pub(crate) fn dialect() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_string(), "#".to_string()], + nested_block_comments: true, + ..LexicalSyntax::default() + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs new file mode 100644 index 0000000000..8c8914d9d1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_code_scanner.rs @@ -0,0 +1,139 @@ +use crate::code_scanner::main::code_scanner::code_scanner; +use crate::code_scanner::main::scanner_matching_paren::scanner_matching_paren; +use crate::code_scanner::main::scanner_non_code_end::scanner_non_code_end; +use crate::code_scanner::tests::helpers::dialect; +use crate::code_scanner::tests::test_types::{MatchingParenTestCase, NonCodeEndTestCase}; +use crate::sql_scan::models::Unclosed; + +/// Expectations recorded from the Python `sql_analysis` scanners this replaces. +#[test] +fn given_sql_when_finding_non_code_end_then_matches_python_offsets() { + let test_cases = [ + NonCodeEndTestCase { + description: "generic backticks end at the first closing backtick", + sql: "`a``b` x", + syntax: None, + index: 0, + expected_end: Ok(Some(3)), + }, + NonCodeEndTestCase { + description: "dialect backticks double to escape", + sql: "`a``b` x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(Some(6)), + }, + NonCodeEndTestCase { + description: "a generic lone dollar is one character", + sql: "$1 x", + syntax: None, + index: 0, + expected_end: Ok(Some(1)), + }, + NonCodeEndTestCase { + description: "a dialect lone dollar is code", + sql: "$1 x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(None), + }, + NonCodeEndTestCase { + description: "offsets count code points", + sql: "é 'naïve' x", + syntax: None, + index: 2, + expected_end: Ok(Some(9)), + }, + NonCodeEndTestCase { + description: "a dialect line comment prefix", + sql: "é # note\nx", + syntax: Some(dialect()), + index: 2, + expected_end: Ok(Some(9)), + }, + NonCodeEndTestCase { + description: "nested block comments", + sql: "/* a /* b */ c */ x", + syntax: Some(dialect()), + index: 0, + expected_end: Ok(Some(17)), + }, + NonCodeEndTestCase { + description: "generic block comments do not nest", + sql: "/* a /* b */ c */ x", + syntax: None, + index: 0, + expected_end: Ok(Some(12)), + }, + NonCodeEndTestCase { + description: "an unclosed dollar quote", + sql: "$tag$ never", + syntax: None, + index: 0, + expected_end: Err(Unclosed::Quote), + }, + NonCodeEndTestCase { + description: "an unclosed block comment", + sql: "/* never", + syntax: Some(dialect()), + index: 0, + expected_end: Err(Unclosed::BlockComment), + }, + NonCodeEndTestCase { + description: "code", + sql: "select 1", + syntax: None, + index: 0, + expected_end: Ok(None), + }, + ]; + for test_case in test_cases { + let scanner = code_scanner(test_case.sql, test_case.syntax); + assert_eq!( + scanner_non_code_end(&scanner, test_case.index), + test_case.expected_end, + "{}", + test_case.description + ); + } +} + +/// Expectations recorded from the Python `find_matching_paren` this replaces. +#[test] +fn given_open_parenthesis_when_matching_then_skips_text_and_comments() { + let test_cases = [ + MatchingParenTestCase { + description: "nested parentheses", + sql: "f(a, (b), c) x", + open: 1, + expected_close: Ok(11), + }, + MatchingParenTestCase { + description: "parentheses in quotes, comments and backticks", + sql: "f(')', `)`, -- )\n /* ) */ $$)$$) x", + open: 1, + expected_close: Ok(31), + }, + MatchingParenTestCase { + description: "code point offsets", + sql: "ééf('é)') x", + open: 3, + expected_close: Ok(8), + }, + MatchingParenTestCase { + description: "an unclosed parenthesis", + sql: "f(a", + open: 1, + expected_close: Err(Unclosed::Parenthesis), + }, + ]; + for test_case in test_cases { + let scanner = code_scanner(test_case.sql, None); + assert_eq!( + scanner_matching_paren(&scanner, test_case.open), + test_case.expected_close, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs new file mode 100644 index 0000000000..20bcc9e52c --- /dev/null +++ b/crates/sqlbuild-sqltext/src/code_scanner/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +pub(super) struct NonCodeEndTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) syntax: Option, + pub(super) index: usize, + pub(super) expected_end: Result, Unclosed>, +} + +pub(super) struct MatchingParenTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) open: usize, + pub(super) expected_close: Result, +} diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs index 74de9d7d28..8e5ea36cbe 100644 --- a/crates/sqlbuild-sqltext/src/lib.rs +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod code_scanner; pub mod compiler; mod constants; pub mod sql_references; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d24163417..7d1fe46741 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1410,6 +1410,11 @@ class NativeModelValidator: self, model: tuple[str, str], references: list[tuple[str, str, bool]] ) -> int | NativeConfigError | None: ... +class SqlCodeScanner: + def __init__(self, sql: str, syntax: dict[str, object] | None, context: str) -> None: ... + def non_code_end(self, index: int) -> int | None: ... + def matching_paren(self, open: int) -> int: ... + class SqlReferenceScanner: def __init__(self, syntax: dict[str, object]) -> None: ... def extract( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 365cff4e0b..21e5b6bfd6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -5,6 +5,7 @@ import re from collections.abc import Callable, Iterator +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, @@ -32,11 +33,6 @@ reference_call_prefix_pattern_text, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token @@ -51,7 +47,7 @@ _TABLE_FUNCTION_PATTERN: re.Pattern[str] = quoted_reference_call_pattern( SqlReferenceKind.TABLE_FUNCTION ) -_SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) +_REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" _CLAUSE_KEYWORDS: frozenset[str] = frozenset( { "WHERE", @@ -185,11 +181,9 @@ def resolve_udf_references( last_index = match.end() continue - call_suffix_end: int = find_matching_paren( - sql=query_sql, - open_paren_index=call_suffix_start, - context="SQL UDF call", - ) + call_suffix_end: int = _native.SqlCodeScanner( + query_sql, None, "SQL UDF call" + ).matching_paren(call_suffix_start) call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] parts.append( adapter.render_udf_call( @@ -241,11 +235,9 @@ def resolve_table_function_references( last_index = match.end() continue - call_suffix_end: int = find_matching_paren( - sql=query_sql, - open_paren_index=call_suffix_start, - context="SQL table function call", - ) + call_suffix_end: int = _native.SqlCodeScanner( + query_sql, None, "SQL table function call" + ).matching_paren(call_suffix_start) call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] parts.append( adapter.render_table_function_call( @@ -281,33 +273,29 @@ def replace_executable_matches( def _iter_executable_matches(*, sql: str, pattern: re.Pattern[str]) -> Iterator[re.Match[str]]: - index: int = 0 - while index < len(sql): - if sql[index] in _SQL_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context="SQL reference resolution") - continue - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context="SQL reference resolution") - continue - match: re.Match[str] | None = pattern.match(sql, index) - if match is None: - index += 1 - continue - yield match - index = match.end() + return _iter_code_matches( + sql=sql, + pattern=pattern, + scanner=_native.SqlCodeScanner(sql, None, _REFERENCE_SCAN_CONTEXT), + ) def _iter_dialect_executable_matches( *, sql: str, pattern: re.Pattern[str], lexical_syntax: SqlLexicalSyntax +) -> Iterator[re.Match[str]]: + return _iter_code_matches( + sql=sql, + pattern=pattern, + scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), + ) + + +def _iter_code_matches( + *, sql: str, pattern: re.Pattern[str], scanner: _native.SqlCodeScanner ) -> Iterator[re.Match[str]]: index: int = 0 while index < len(sql): - non_code_end: int | None = dialect_non_code_end( - sql=sql, start=index, syntax=lexical_syntax, context="SQL reference resolution" - ) + non_code_end: int | None = scanner.non_code_end(index) if non_code_end is not None: index = non_code_end continue diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py deleted file mode 100644 index b1a580421b..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Character-level SQL scanning implementations.""" - -from __future__ import annotations - -import re -from collections.abc import Iterator - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.constants import ( - SQL_BLOCK_COMMENT_CLOSE, - SQL_BLOCK_COMMENT_OPEN, - SQL_CLOSE_PARENTHESIS, - SQL_DOLLAR_QUOTE_CHARACTER, - SQL_ESCAPABLE_QUOTE_CHARACTERS, - SQL_ESCAPE_CHARACTER, - SQL_ESCAPE_STRING_PREFIX, - SQL_IDENTIFIER_PREFIX, - SQL_OPEN_PARENTHESIS, - SQL_RAW_STRING_PREFIX, - SQL_STRING_PREFIX_CHARACTERS, - SQL_STRING_PREFIX_MAX_LENGTH, - SQL_STRING_QUOTE_CHARACTER, - SQL_TEXT_START_CHARACTERS, - SQL_TRIPLE_QUOTE_LENGTH, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/#'\"`$()]") -_DOLLAR_QUOTE_DELIMITER: re.Pattern[str] = re.compile(r"\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$") - - -def skip_quoted_text_impl(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip quoted text at the position, consuming a non-opening ``$`` as one code character.""" - - end: int | None = quoted_text_end_impl(sql=sql, start=start, context=context) - return start + 1 if end is None else end - - -def quoted_text_end_impl(*, sql: str, start: int, context: str = "SQL") -> int | None: - """Return the end of quoted or dollar-quoted text starting at the position, if any.""" - - quote_character: str = sql[start] - if quote_character == SQL_DOLLAR_QUOTE_CHARACTER: - return _dollar_quoted_text_end(sql=sql, start=start, context=context) - index: int = start + 1 - while True: - index = sql.find(quote_character, index) - if index == -1: - raise CompileInputError(f"{context} contains an unclosed quoted string") - if ( - quote_character in SQL_ESCAPABLE_QUOTE_CHARACTERS - and index + 1 < len(sql) - and sql[index + 1] == quote_character - ): - index += 2 - continue - return index + 1 - - -def dialect_non_code_end_impl( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of the quoted text or comment starting at the position, if any.""" - - if sql.startswith(SQL_BLOCK_COMMENT_OPEN, start): - return _dialect_block_comment_end(sql=sql, start=start, syntax=syntax, context=context) - if any(sql.startswith(prefix, start) for prefix in syntax.line_comment_prefixes): - return skip_line_comment_impl(sql=sql, start=start) - if sql[start] in SQL_TEXT_START_CHARACTERS: - return dialect_quoted_text_end_impl(sql=sql, start=start, syntax=syntax, context=context) - return None - - -def _dialect_block_comment_end( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str -) -> int: - depth: int = 1 - index: int = start + len(SQL_BLOCK_COMMENT_OPEN) - while depth: - closing_index: int = sql.find(SQL_BLOCK_COMMENT_CLOSE, index) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed block comment") - opening_index: int = ( - sql.find(SQL_BLOCK_COMMENT_OPEN, index, closing_index) - if syntax.nested_block_comments - else -1 - ) - if opening_index == -1: - depth -= 1 - index = closing_index + len(SQL_BLOCK_COMMENT_CLOSE) - else: - depth += 1 - index = opening_index + len(SQL_BLOCK_COMMENT_OPEN) - return index - - -def dialect_quoted_text_end_impl( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of quoted text starting at the position under the dialect's escape rules.""" - - quote_character: str = sql[start] - if quote_character == SQL_DOLLAR_QUOTE_CHARACTER: - return _dollar_quoted_text_end(sql=sql, start=start, context=context) - prefix: str = _string_literal_prefix(sql=sql, start=start) - backslash_escapes: bool = quote_character in syntax.backslash_escape_quotes or ( - syntax.escape_string_prefix - and quote_character == SQL_STRING_QUOTE_CHARACTER - and prefix == SQL_ESCAPE_STRING_PREFIX - ) - if syntax.raw_string_prefix and SQL_RAW_STRING_PREFIX in prefix: - backslash_escapes = False - delimiter: str = quote_character - triple_quote: str = quote_character * SQL_TRIPLE_QUOTE_LENGTH - if syntax.triple_quoted_strings and sql.startswith(triple_quote, start): - delimiter = triple_quote - index: int = start + len(delimiter) - while index < len(sql): - if backslash_escapes and sql.startswith(SQL_ESCAPE_CHARACTER, index): - index += 2 - continue - if sql.startswith(delimiter, index): - if delimiter == quote_character and sql.startswith(quote_character, index + 1): - index += 2 - continue - return index + len(delimiter) - index += 1 - raise CompileInputError(f"{context} contains an unclosed quoted string") - - -def _string_literal_prefix(*, sql: str, start: int) -> str: - prefix_start: int = start - while ( - prefix_start > 0 - and start - prefix_start < SQL_STRING_PREFIX_MAX_LENGTH - and sql[prefix_start - 1] in SQL_STRING_PREFIX_CHARACTERS - ): - prefix_start -= 1 - if prefix_start > 0 and _continues_dollar_word(sql[prefix_start - 1]): - return "" - return sql[prefix_start:start].lower() - - -def _dollar_quoted_text_end(*, sql: str, start: int, context: str) -> int | None: - delimiter: re.Match[str] | None = _DOLLAR_QUOTE_DELIMITER.match(sql, start) - if delimiter is None or (start > 0 and _continues_dollar_word(sql[start - 1])): - return None - closing_index: int = sql.find(delimiter.group(), delimiter.end()) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed quoted string") - return closing_index + len(delimiter.group()) - - -def _continues_dollar_word(character: str) -> bool: - return ( - not character.isascii() - or character.isalnum() - or character in {SQL_IDENTIFIER_PREFIX, SQL_DOLLAR_QUOTE_CHARACTER} - ) - - -def skip_line_comment_impl(*, sql: str, start: int) -> int: - """Skip past an SQL line comment.""" - - newline_index: int = sql.find("\n", start) - return len(sql) if newline_index == -1 else newline_index + 1 - - -def skip_block_comment_impl(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past an SQL block comment.""" - - closing_index: int = sql.find("*/", start + 2) - if closing_index == -1: - raise CompileInputError(f"{context} contains an unclosed block comment") - return closing_index + 2 - - -def find_matching_paren_impl( - *, - sql: str, - open_paren_index: int, - context: str = "SQL", - syntax: SqlLexicalSyntax | None = None, -) -> int: - """Find the closing parenthesis matching an opening parenthesis.""" - - depth: int = 1 - index: int = open_paren_index + 1 - while index < len(sql): - special: re.Match[str] | None = _PAREN_SCAN_SPECIAL.search(sql, index) - if special is None: - break - index = special.start() - non_code_end: int | None = _non_code_end( - sql=sql, index=index, context=context, syntax=syntax - ) - if non_code_end is not None: - index = non_code_end - continue - if sql[index] == SQL_OPEN_PARENTHESIS: - depth += 1 - elif sql[index] == SQL_CLOSE_PARENTHESIS: - depth -= 1 - if depth == 0: - return index - index += 1 - raise CompileInputError(f"{context} contains an unclosed parenthesis") - - -def iter_code_positions_impl( - *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None -) -> Iterator[tuple[int, int]]: - """Yield code offsets outside comments, quotes, and parentheses with their nesting depth.""" - - depth: int = 0 - index: int = 0 - while index < len(sql): - non_code_end: int | None = _non_code_end( - sql=sql, index=index, context=context, syntax=syntax - ) - if non_code_end is not None: - index = non_code_end - continue - character: str = sql[index] - if character == SQL_OPEN_PARENTHESIS: - depth += 1 - elif character == SQL_CLOSE_PARENTHESIS: - depth -= 1 - else: - yield index, depth - index += 1 - - -def _non_code_end( - *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None -) -> int | None: - if syntax is not None: - return dialect_non_code_end_impl(sql=sql, start=index, syntax=syntax, context=context) - if sql.startswith("--", index): - return skip_line_comment_impl(sql=sql, start=index) - if sql.startswith("/*", index): - return skip_block_comment_impl(sql=sql, start=index, context=context) - if sql[index] in SQL_TEXT_START_CHARACTERS: - return quoted_text_end_impl(sql=sql, start=index, context=context) - return None diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index be247a0320..d58898fd27 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -3,23 +3,11 @@ from __future__ import annotations SQL_ESCAPABLE_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({*SQL_ESCAPABLE_QUOTE_CHARACTERS, "`"}) -SQL_TEXT_START_CHARACTERS: frozenset[str] = frozenset({*SQL_QUOTE_CHARACTERS, "$"}) -SQL_OPEN_PARENTHESIS: str = "(" -SQL_CLOSE_PARENTHESIS: str = ")" -SQL_STRING_QUOTE_CHARACTER: str = "'" SQL_ESCAPE_CHARACTER: str = "\\" -SQL_ESCAPE_STRING_PREFIX: str = "e" -SQL_RAW_STRING_PREFIX: str = "r" -SQL_STRING_PREFIX_CHARACTERS: frozenset[str] = frozenset("bBeErR") -SQL_STRING_PREFIX_MAX_LENGTH: int = 2 SQL_TRIPLE_QUOTE_LENGTH: int = 3 SQL_LINE_COMMENT_PREFIX: str = "--" SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) SQL_BLOCK_COMMENT_OPEN: str = "/*" -SQL_BLOCK_COMMENT_CLOSE: str = "*/" -SQL_DOLLAR_QUOTE_CHARACTER: str = "$" -SQL_IDENTIFIER_PREFIX: str = "_" SNOWFLAKE_DIALECT_NAME: str = "snowflake" POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 diff --git a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py deleted file mode 100644 index a5b31ae45c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py +++ /dev/null @@ -1,21 +0,0 @@ -"""SQL parenthesis matching entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import find_matching_paren_impl -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def find_matching_paren( - *, - sql: str, - open_paren_index: int, - context: str = "SQL", - syntax: SqlLexicalSyntax | None = None, -) -> int: - """Find the closing parenthesis, ending quoted text and comments per the optional dialect.""" - - return find_matching_paren_impl( - sql=sql, - open_paren_index=open_paren_index, - context=context, - syntax=syntax, - ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py deleted file mode 100644 index 13f2482f2a..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_block_comment.py +++ /dev/null @@ -1,9 +0,0 @@ -"""SQL block comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_block_comment_impl - - -def skip_block_comment(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past an SQL block comment.""" - - return skip_block_comment_impl(sql=sql, start=start, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py deleted file mode 100644 index 2fa5b8c4ba..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_dialect_non_code.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Dialect-aware SQL quoted text and comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import dialect_non_code_end_impl -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def dialect_non_code_end( - *, sql: str, start: int, syntax: SqlLexicalSyntax, context: str = "SQL" -) -> int | None: - """Return the end of the quoted text or comment at the position under the dialect's rules.""" - - return dialect_non_code_end_impl(sql=sql, start=start, syntax=syntax, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py deleted file mode 100644 index 5708863245..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_line_comment.py +++ /dev/null @@ -1,9 +0,0 @@ -"""SQL line comment scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_line_comment_impl - - -def skip_line_comment(*, sql: str, start: int) -> int: - """Skip past an SQL line comment.""" - - return skip_line_comment_impl(sql=sql, start=start) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py b/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py deleted file mode 100644 index b131f0fbf3..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_skip_quoted_text.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Quoted SQL text scanning entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.scanning import skip_quoted_text_impl - - -def skip_quoted_text(*, sql: str, start: int, context: str = "SQL") -> int: - """Skip past a quoted string starting at the supplied position.""" - - return skip_quoted_text_impl(sql=sql, start=start, context=context) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index 9303c99533..64026852d6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -24,15 +24,6 @@ class SkipQuotedTextErrorTestCase: expected_error: str -@dataclass(frozen=True) -class IterCodePositionsTestCase: - """Code-position scan case.""" - - description: str - sql: str - expected_positions: tuple[tuple[int, int], ...] - - @dataclass(frozen=True) class DialectQuotedTextSuccessTestCase: """Successful dialect-aware quoted-text scan case.""" @@ -89,6 +80,6 @@ class DialectMatchingParenTestCase: """Dialect-aware parenthesis matching case.""" description: str - syntax: SqlLexicalSyntax | None + native_syntax: dict[str, object] | None sql: str expected_index: int diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py new file mode 100644 index 0000000000..79b8981a2d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py @@ -0,0 +1,18 @@ +"""Native SQL code scanners for the scanning tests.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + + +def generic_scanner(*, sql: str, context: str = "SQL") -> _native.SqlCodeScanner: + """A native scanner over `sql` under the generic compiler rules.""" + + return _native.SqlCodeScanner(sql, None, context) + + +def dialect_scanner(*, sql: str, syntax: SqlLexicalSyntax) -> _native.SqlCodeScanner: + """A native scanner over `sql` under one dialect's lexical rules.""" + + return _native.SqlCodeScanner(sql, syntax.native_mapping, "SQL") diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index 2d45c8472c..850e3434e6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -2,14 +2,8 @@ import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis._helpers.scanning import ( - dialect_non_code_end_impl, - dialect_quoted_text_end_impl, - find_matching_paren_impl, - iter_code_positions_impl, - skip_quoted_text_impl, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( DialectMatchingParenTestCase, @@ -17,11 +11,14 @@ DialectNonCodeSuccessTestCase, DialectQuotedTextErrorTestCase, DialectQuotedTextSuccessTestCase, - IterCodePositionsTestCase, RequiresDialectScanTestCase, SkipQuotedTextErrorTestCase, SkipQuotedTextSuccessTestCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers.helpers import ( + dialect_scanner, + generic_scanner, +) @pytest.mark.parametrize( @@ -75,7 +72,7 @@ def test_given_quoted_text_when_skipping_then_returns_end_position( ) -> None: sql: str = f"{test_case.quoted_sql} suffix" - result: int = skip_quoted_text_impl(sql=sql, start=0) + result: int | None = generic_scanner(sql=sql).non_code_end(0) assert result == test_case.expected_end @@ -102,7 +99,7 @@ def test_given_unclosed_quote_when_skipping_then_raises_contextual_error( test_case: SkipQuotedTextErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - skip_quoted_text_impl(sql=test_case.sql, start=0, context=test_case.context) + generic_scanner(sql=test_case.sql, context=test_case.context).non_code_end(0) ANSI_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() @@ -197,8 +194,8 @@ def test_given_unclosed_quote_when_skipping_then_raises_contextual_error( def test_given_dialect_quoted_text_when_scanning_then_returns_end_position( test_case: DialectQuotedTextSuccessTestCase, ) -> None: - result: int | None = dialect_quoted_text_end_impl( - sql=test_case.sql, start=test_case.start, syntax=test_case.syntax + result: int | None = dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end( + test_case.start ) assert result == test_case.expected_end @@ -226,7 +223,7 @@ def test_given_unclosed_dialect_quoted_text_when_scanning_then_raises( test_case: DialectQuotedTextErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - dialect_quoted_text_end_impl(sql=test_case.sql, start=0, syntax=test_case.syntax) + dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) NESTED_COMMENT_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax(nested_block_comments=True) @@ -295,9 +292,7 @@ def test_given_unclosed_dialect_quoted_text_when_scanning_then_raises( def test_given_dialect_sql_when_scanning_non_code_then_returns_end_position( test_case: DialectNonCodeSuccessTestCase, ) -> None: - result: int | None = dialect_non_code_end_impl( - sql=test_case.sql, start=0, syntax=test_case.syntax - ) + result: int | None = dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) assert result == test_case.expected_end @@ -324,44 +319,7 @@ def test_given_unclosed_dialect_comment_when_scanning_then_raises( test_case: DialectNonCodeErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - dialect_non_code_end_impl(sql=test_case.sql, start=0, syntax=test_case.syntax) - - -@pytest.mark.parametrize( - "test_case", - ( - IterCodePositionsTestCase( - description="parentheses change depth and are not yielded", - sql="a(b)c", - expected_positions=((0, 0), (2, 1), (4, 0)), - ), - IterCodePositionsTestCase( - description="quotes and comments are skipped", - sql="a'(b'`c`--d\n/*e*/f", - expected_positions=((0, 0), (17, 0)), - ), - IterCodePositionsTestCase( - description="dollar-quoted text is skipped", - sql="a $$'(--$$b", - expected_positions=((0, 0), (1, 0), (10, 0)), - ), - IterCodePositionsTestCase( - description="dollars inside identifiers and parameters remain code", - sql="a$$b $1", - expected_positions=tuple((index, 0) for index in range(7)), - ), - IterCodePositionsTestCase( - description="unbalanced close parenthesis goes below zero depth", - sql=")a", - expected_positions=((1, -1),), - ), - ), - ids=lambda case: case.description, -) -def test_given_sql_when_iterating_code_positions_then_yields_code_offsets_with_depth( - test_case: IterCodePositionsTestCase, -) -> None: - assert tuple(iter_code_positions_impl(sql=test_case.sql)) == test_case.expected_positions + dialect_scanner(sql=test_case.sql, syntax=test_case.syntax).non_code_end(0) @pytest.mark.parametrize( @@ -431,19 +389,19 @@ def test_given_sql_when_checking_dialect_sensitivity_then_reports_differences( ( DialectMatchingParenTestCase( description="backslash-escaped quote hides a parenthesis", - syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + native_syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX.native_mapping, sql="('O\\') ' AS name) tail", expected_index=len("('O\\') ' AS name"), ), DialectMatchingParenTestCase( description="hash comment hides a parenthesis", - syntax=HASH_COMMENT_SYNTAX, + native_syntax=HASH_COMMENT_SYNTAX.native_mapping, sql="(1 # )\n) tail", expected_index=len("(1 # )\n"), ), DialectMatchingParenTestCase( description="generic scanning keeps the backslash literal", - syntax=None, + native_syntax=None, sql="('C:\\') tail", expected_index=len("('C:\\'"), ), @@ -454,6 +412,6 @@ def test_given_dialect_sql_when_matching_parenthesis_then_follows_dialect_rules( test_case: DialectMatchingParenTestCase, ) -> None: assert ( - find_matching_paren_impl(sql=test_case.sql, open_paren_index=0, syntax=test_case.syntax) + _native.SqlCodeScanner(test_case.sql, test_case.native_syntax, "SQL").matching_paren(0) == test_case.expected_index ) From ede9ced0018442badfe32da47923cf125861209f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:57:15 +0100 Subject: [PATCH 15/26] refactor: map uncataloged schema validation results natively Schema validation without a shared catalog now returns diagnostic rows from one native call that validates, maps and positions them, replacing the Python JSON response parsing and the validate_sql_with_schemas_json binding. --- .../semantic_validation/_helpers/catalog.rs | 36 ++++++++++ .../src/bindings/_helpers/analysis/queries.rs | 26 ++++--- src/sqlbuild/_native.pyi | 5 ++ .../_helpers/schema_validation.py | 67 ++----------------- .../compiler/sql_analysis/constants.py | 1 - src/sqlbuild/compiler/sql_analysis/types.py | 8 --- .../compiler/pipeline/test_semantic_triage.py | 4 +- .../compiler/sql_analysis/main/helpers.py | 21 +++++- .../main/test_semantic_diagnostics.py | 62 +++++++++-------- 9 files changed, 121 insertions(+), 109 deletions(-) diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index e166e32913..a771ef8731 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -24,6 +24,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; const ANALYSIS_WORKERS: usize = 4; const ANALYSIS_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; +const DEFAULT_BINDING_MESSAGE: &str = "SQL binding failed"; impl ProjectCatalog { /// Map inferred column types onto the binding names the query projects, in column order. @@ -200,6 +201,32 @@ impl ProjectCatalog { serde_json::to_string(&payloads).map_err(|error| error.to_string()) } + /// Validate each request on its own schema, as `validations_json` does, as diagnostic rows. + pub fn schema_validation_rows( + &self, + dialect: &str, + requests: Vec, + ) -> Result>, String> { + let sqls: Vec = requests.iter().map(|request| request.0.clone()).collect(); + let payload: String = self.validation_payload(requests)?; + let results: Vec = + serde_json::from_str(&validation::validations_json(&payload)?) + .map_err(|error| error.to_string())?; + if results.len() != sqls.len() { + return Err( + "native SQL schema validation returned an invalid batch response".to_owned(), + ); + } + sqls.iter() + .zip(results) + .map(|(sql, result)| { + let rows: Vec = + result.errors.into_iter().map(messaged_row).collect(); + diagnostics::binding_diagnostics(sql, dialect, rows) + }) + .collect() + } + /// Resolve a compact batch now; the job runs later while the catalog keeps changing. pub fn prepare_compact(&self, text: &str) -> Result { let analysis = @@ -586,6 +613,15 @@ impl ProjectCatalog { } } +/// The diagnostic row of `error`, naming the failure when the parser gives no message. +fn messaged_row(error: ValidationError) -> DiagnosticRow { + let mut row: DiagnosticRow = diagnostic_row(error); + if row.1.is_empty() { + DEFAULT_BINDING_MESSAGE.clone_into(&mut row.1); + } + row +} + pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { let severity = match error.severity { polyglot_sql::ValidationSeverity::Error => "error", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index b398ef704f..1fb8cc1b19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -1,9 +1,12 @@ //! Validate, fingerprint and analyze SQL queries for the Python compiler. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRow}; -use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::analysis::conversions; +use crate::bindings::_helpers::boundary::panics::{compiler_error, value_error}; +use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -55,12 +58,17 @@ fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::validations_json(request_json) - }) - .map_err(value_error) +/// Each request validated on its own schema, as binding diagnostic rows mapped to its SQL. +#[pyfunction] +fn schema_validation_rows( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + dialect: &str, + #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, +) -> PyResult>> { + let inner: &sqlbuild_analysis::semantic_validation::models::ProjectCatalog = &catalog.inner; + py.compiler_detach(|| inner.schema_validation_rows(dialect, requests)) + .map_err(compiler_error) } #[pyfunction] @@ -113,7 +121,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(function_definition_hash, module)?)?; module.add_function(wrap_pyfunction!(normalize_query_text, module)?)?; module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; - module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; + module.add_function(wrap_pyfunction!(schema_validation_rows, module)?)?; module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7d1fe46741..dabb0466e1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1817,6 +1817,11 @@ def assemble_project_resource_facts( tuple[str, int, str, str, str | None, int | None] | None, ]: ... def check_native_explicit_references(request_json: str, /) -> str: ... +def schema_validation_rows( + catalog: Any, + dialect: str, + requests: list[tuple[str, list[tuple[str, bool]], dict[str, Mapping[str, str]]]], +) -> list[list[tuple[str, str, int | None, int | None, int | None, int | None, str]]]: ... def python_literal_sql_calls( source: str, start_line: int, placeholder: str ) -> list[tuple[str, str, int]]: ... diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py index eab2c51cf8..583d396b1b 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py @@ -1,22 +1,15 @@ """Orchestrate compile-owned catalogs; native code owns schemas and diagnostics.""" -import json -from typing import Any, cast - import sqlbuild._native as _native from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog -from sqlbuild.compiler.sql_analysis.constants import BINDING_SEVERITIES, NATIVE_DIALECT_ALIASES +from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES from sqlbuild.compiler.sql_analysis.exceptions import SqlAnalysisBoundaryError from sqlbuild.compiler.sql_analysis.models import ( SqlBindingDiagnostic, SqlBindingResult, SqlSchemaValidationRequest, ) -from sqlbuild.compiler.sql_analysis.types import ( - NativeDiagnosticRow, - NativePositionsModule, - NativeValidationModule, -) +from sqlbuild.compiler.sql_analysis.types import NativeDiagnosticRow def get_schema_validations( @@ -50,22 +43,10 @@ def get_schema_validations( results: list[SqlBindingResult] = [SqlBindingResult()] * len(requests) for catalog, group in groups.items(): batch: list[SqlSchemaValidationRequest] = [request for _, request in group] - if all(request.catalog is None for request in batch): - payload: str = catalog.native.validation_payload(catalog.prepare(batch)) - responses: object = json.loads( - cast(NativeValidationModule, _native).validate_sql_with_schemas_json(payload) - ) - if not isinstance(responses, list) or len(responses) != len(batch): - raise SqlAnalysisBoundaryError( - "native SQL schema validation returned an invalid batch response" - ) - for (index, request), response in zip(group, responses, strict=True): - results[index] = _binding_result( - sql=request.sql, dialect=request.dialect, response=response - ) - continue - rows: list[list[NativeDiagnosticRow]] = catalog.native.binding_results( - catalog.prepare(batch) + rows: list[list[NativeDiagnosticRow]] = ( + _native.schema_validation_rows(catalog.native, catalog.dialect, catalog.prepare(batch)) + if all(request.catalog is None for request in batch) + else catalog.native.binding_results(catalog.prepare(batch)) ) if len(rows) != len(batch): raise SqlAnalysisBoundaryError( @@ -78,39 +59,3 @@ def get_schema_validations( def _result(rows: list[NativeDiagnosticRow]) -> SqlBindingResult: return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in rows)) - - -def _binding_result(*, sql: str, dialect: str | None, response: object) -> SqlBindingResult: - if not isinstance(response, dict): - raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") - payload: dict[str, Any] = cast(dict[str, Any], response) - if not isinstance(payload.get("errors"), list): - raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") - rows: list[NativeDiagnosticRow] = [] - for value in payload["errors"]: - if not isinstance(value, dict) or value.get("severity") not in BINDING_SEVERITIES: - continue - if not isinstance(value.get("code"), str): - continue - rows.append( - ( - value["code"], - str(value.get("message") or "SQL binding failed"), - _optional_int(value.get("line")), - _optional_int(value.get("column")), - _optional_int(value.get("start")), - _optional_int(value.get("end")), - str(value["severity"]), - ) - ) - return _result( - cast(NativePositionsModule, _native).binding_diagnostics( - sql=sql, - dialect=NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect or "generic"), - rows=rows, - ) - ) - - -def _optional_int(value: object) -> int | None: - return value if isinstance(value, int) else None diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index d58898fd27..ec62425c4a 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -21,7 +21,6 @@ TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) SQL_QUOTED_IDENTIFIER_DELIMITER: str = '"' CASE_SENSITIVE_BINDING_DIALECTS: frozenset[str] = frozenset({"snowflake", "postgresql"}) -BINDING_SEVERITIES: frozenset[str] = frozenset({"error", "warning"}) NATIVE_DIALECT_ALIASES: dict[str, str] = { "postgres": "postgresql", "motherduck": "duckdb", diff --git a/src/sqlbuild/compiler/sql_analysis/types.py b/src/sqlbuild/compiler/sql_analysis/types.py index 266c536fef..717f5b0af9 100644 --- a/src/sqlbuild/compiler/sql_analysis/types.py +++ b/src/sqlbuild/compiler/sql_analysis/types.py @@ -62,11 +62,3 @@ def BindingPositions( ) -> NativeBindingPositions: ... def normalize_analysis_sql(self, request: dict[str, object]) -> str: ... def normalize_dialect_sql(self, *, sql: str, dialect: str) -> str: ... - - -class NativeValidationModule(Protocol): - """Native schema-validation boundary.""" - - def validate_sql_with_schema_json(self, request_json: str) -> str: ... - - def validate_sql_with_schemas_json(self, request_json: str) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index fd81e00a7d..808fea1149 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -226,7 +226,7 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma repeated_validation: Mock = Mock( side_effect=AssertionError("cold compile repeated fused validation") ) - patch.setattr(native, "validate_sql_with_schemas_json", repeated_validation) + patch.setattr(native, "schema_validation_rows", repeated_validation) assert main(args) == 0 repeated_validation.assert_not_called() capsys.readouterr() @@ -234,7 +234,7 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma forbidden: Mock = Mock( side_effect=AssertionError("warm compile repeated native semantic work") ) - patch.setattr(native, "validate_sql_with_schemas_json", forbidden) + patch.setattr(native, "schema_validation_rows", forbidden) patch.setattr(native, "analyze_project_queries_compact_json", forbidden) assert main(args) == 0 forbidden.assert_not_called() diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index efd7dcee2a..e5f22cb9dc 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -1,7 +1,15 @@ """Helpers for SQL analysis public-entry tests.""" +from typing import cast + +import sqlbuild._native as _native from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog -from sqlbuild.compiler.sql_analysis.types import NativeProjectCatalog +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult +from sqlbuild.compiler.sql_analysis.types import ( + NativeDiagnosticRow, + NativePositionsModule, + NativeProjectCatalog, +) def nested_coalesce_sql(*, function_depth: int) -> str: @@ -27,3 +35,14 @@ def without_catalog(dialect: str) -> None: """Return no catalog, which normalizes batches in turn.""" _ = dialect + + +def mapped_binding_result( + *, sql: str, dialect: str, rows: list[NativeDiagnosticRow] +) -> SqlBindingResult: + """Native polyglot diagnostic rows mapped to SQLBuild binding diagnostics for `sql`.""" + + mapped: list[NativeDiagnosticRow] = cast(NativePositionsModule, _native).binding_diagnostics( + sql=sql, dialect=dialect, rows=rows + ) + return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in mapped)) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py index 528eb27bdc..decef34399 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py @@ -7,12 +7,14 @@ import pytest -import sqlbuild._native as native -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest +from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( SemanticDiagnosticMappingCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import mapped_binding_result + +DIALECTS: tuple[str, ...] = ("duckdb", "snowflake", "postgresql", "bigquery") @pytest.mark.parametrize( @@ -48,7 +50,6 @@ ids=lambda case: case.description, ) def test_given_native_semantic_errors_when_validating_then_codes_and_locations_survive( - monkeypatch: pytest.MonkeyPatch, test_case: SemanticDiagnosticMappingCase, ) -> None: codes: dict[str, str] = {"E202": "B101", "E203": "B102", "E222": "B004", "E223": "B005"} @@ -56,30 +57,37 @@ def test_given_native_semantic_errors_when_validating_then_codes_and_locations_s {f"E{number}": f"B{number}" for number in (*range(210, 220), 230, 231, 232, 233, 234)} ) codes.update({"W210": "B217", "W213": "B218"}) - - def validate(payload: str) -> str: - requests: list[dict[str, Any]] = json.loads(payload) - assert all(request["options"]["semantic"] for request in requests) - assert all(request["options"]["check_types"] for request in requests) - errors: list[dict[str, object]] = [ - { - "code": code, - "severity": {True: "warning", False: "error"}[code.startswith("W")], - "message": "Invalid expression", - "line": 2, - "column": 8, - } - for code in codes - ] - return json.dumps([{"errors": errors}] * len(requests)) - - monkeypatch.setattr(native, "validate_sql_with_schemas_json", validate) - results: tuple[SqlBindingResult, ...] = get_schema_validations( - requests=tuple( - SqlSchemaValidationRequest(sql="SELECT\n invalid", dialect=dialect, schema={}) - for dialect in ("duckdb", "snowflake", "postgres", "bigquery") + rows: list[tuple[str, str, int | None, int | None, int | None, int | None, str]] = [ + ( + code, + "Invalid expression", + 2, + 8, + None, + None, + {True: "warning", False: "error"}[code.startswith("W")], ) - ) + for code in codes + ] + payloads: list[dict[str, Any]] = [ + json.loads( + create_binding_catalog( + dialect=dialect, + quoted_ignore_case=False, + known_functions=(), + known_types=(), + relations={}, + ).native.validation_payload([("SELECT\n invalid", [], {})]) + )[0] + for dialect in DIALECTS + ] + results: list[SqlBindingResult] = [ + mapped_binding_result(sql="SELECT\n invalid", dialect=dialect, rows=rows) + for dialect in DIALECTS + ] + + assert all(payload["options"]["semantic"] for payload in payloads) + assert all(payload["options"]["check_types"] for payload in payloads) for result in results: assert ( tuple(diagnostic.code for diagnostic in result.diagnostics) == test_case.expected_codes From eb233d07b678444a09d9dd496fc3864bc9b7b333 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:57:51 +0100 Subject: [PATCH 16/26] refactor: delete unused SQL analysis entry points resolve_binding_column and is_polyglot_available have no callers. --- .../sql_analysis/_helpers/polyglot.py | 6 ---- .../compiler/sql_analysis/constants.py | 1 - .../main/_is_polyglot_available.py | 11 -------- .../main/_resolve_binding_column.py | 28 ------------------- 4 files changed, 46 deletions(-) delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py index ab5ea62974..15badf7b70 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py @@ -80,9 +80,3 @@ def import_polyglot_sql() -> Any: """Return SQLBuild's required Polyglot SQL module.""" return import_polyglot() - - -def is_polyglot_available() -> bool: - """Return true because Polyglot SQL is a required dependency.""" - - return True diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index ec62425c4a..c51f1ac0f5 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -8,7 +8,6 @@ SQL_LINE_COMMENT_PREFIX: str = "--" SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) SQL_BLOCK_COMMENT_OPEN: str = "/*" -SNOWFLAKE_DIALECT_NAME: str = "snowflake" POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 diff --git a/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py b/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py deleted file mode 100644 index 352473adcd..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_is_polyglot_available.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public Polyglot availability entry.""" - -from sqlbuild.compiler.sql_analysis._helpers.polyglot import ( - is_polyglot_available as _is_polyglot_available, -) - - -def is_polyglot_available() -> bool: - """Return whether Polyglot SQL can be imported.""" - - return _is_polyglot_available() diff --git a/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py b/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py deleted file mode 100644 index 0165127490..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Resolve schema column spellings using dialect identifier semantics.""" - -from collections.abc import Mapping - -from sqlbuild.compiler.sql_analysis.constants import ( - CASE_SENSITIVE_BINDING_DIALECTS, - NATIVE_DIALECT_ALIASES, - SNOWFLAKE_DIALECT_NAME, - SQL_QUOTED_IDENTIFIER_DELIMITER, -) - - -def resolve_binding_column( - *, name: str, columns: Mapping[str, str], dialect: str | None, ignore_quoted_case: bool = False -) -> str | None: - dialect = NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect) - - def identity(value: str) -> str: - quoted: bool = value.startswith(SQL_QUOTED_IDENTIFIER_DELIMITER) and value.endswith( - SQL_QUOTED_IDENTIFIER_DELIMITER - ) - raw: str = value[1:-1].replace('""', '"') if quoted else value - if quoted and dialect in CASE_SENSITIVE_BINDING_DIALECTS and not ignore_quoted_case: - return raw - return raw.upper() if dialect == SNOWFLAKE_DIALECT_NAME else raw.casefold() - - expected: str = identity(name) - return next((column for column in columns if identity(column) == expected), None) From a1fda03ab16f414f210a763d593c8a1ce5784715 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:16:58 +0100 Subject: [PATCH 17/26] refactor: compute audit-gate and hook-read ordering edges natively The planner's attached-audit gate edges and hook-read edges come from the native explicit-reference graph logic, which now keeps each edge's audit and target as Python did. The Python fixed-point and hook-edge implementations are deleted; Python only collects the project facts. --- .../_helpers/audit_gates.rs | 30 ++-- .../_helpers/hook_reads.rs | 36 ++--- .../main/check_explicit_references.rs | 2 +- .../main/execution_gate_edges.rs | 14 ++ .../src/explicit_references/main/mod.rs | 1 + .../src/explicit_references/models.rs | 47 +++++- .../_helpers/analysis/explicit_references.rs | 48 +++++- src/sqlbuild/_native.pyi | 6 + .../compiler/graph/_helpers/audit_gates.py | 115 -------------- .../graph/_helpers/execution_gates.py | 141 ++++++++++++++++++ .../compiler/graph/_helpers/hook_reads.py | 76 ---------- .../graph/main/_attached_audit_gate_edges.py | 2 +- .../compiler/graph/main/_hook_read_edges.py | 2 +- .../graph/main/_model_python_hook_names.py | 2 +- 14 files changed, 282 insertions(+), 240 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/explicit_references/main/execution_gate_edges.rs delete mode 100644 src/sqlbuild/compiler/graph/_helpers/audit_gates.py create mode 100644 src/sqlbuild/compiler/graph/_helpers/execution_gates.py delete mode 100644 src/sqlbuild/compiler/graph/_helpers/hook_reads.py diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs index d2c4b0a0b0..cdeac2df37 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/audit_gates.rs @@ -7,30 +7,22 @@ use crate::explicit_references::constants::{ AUDIT_GATE_CODE, MODEL_TYPE, READABLE_TYPES, SEED_TYPE, SOURCE_TYPE, TABLE_FUNCTION_TYPE, UDF_TYPE, }; -use crate::explicit_references::models::{AuditGate, ReferenceDiagnostic, ReferenceRequest}; +use crate::explicit_references::models::{ + AuditGate, GateEdge, GraphResource, ReferenceDiagnostic, ReferenceRequest, +}; use crate::explicit_references::types::ResourceKey; -/// One ordering edge: `gated` waits for `read`, which an audit on `target` reads. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct GateEdge { - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, -} - /// An audit's target and the other resources it reads. struct Gate<'a> { + audit_name: &'a str, target: &'a ResourceKey, reads: Vec<&'a ResourceKey>, } /// `attached_audit_gate_edges_impl`: edges to a fixed point, as sources gain triggering models. -pub(crate) fn gate_edges(request: &ReferenceRequest) -> Vec { +pub(crate) fn gate_edges(resources: &[GraphResource], audits: &[AuditGate]) -> Vec { let mut triggers: HashMap> = HashMap::new(); - for resource in request - .resources - .iter() - .filter(|item| item.key.0 == MODEL_TYPE) - { + for resource in resources.iter().filter(|item| item.key.0 == MODEL_TYPE) { for dep in resource.deps.iter().filter(|dep| dep.0 == SOURCE_TYPE) { triggers .entry(dep.clone()) @@ -38,7 +30,7 @@ pub(crate) fn gate_edges(request: &ReferenceRequest) -> Vec { .insert(resource.key.clone()); } } - let gates: Vec> = request.audits.iter().filter_map(gate).collect(); + let gates: Vec> = audits.iter().filter_map(gate).collect(); let mut edges: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); let mut changed = true; @@ -105,7 +97,11 @@ pub(crate) fn gate_diagnostics( fn gate(audit: &AuditGate) -> Option> { let target = audit.target.as_ref()?; let reads: Vec<&ResourceKey> = extra_reads(audit, target); - (!reads.is_empty()).then_some(Gate { target, reads }) + (!reads.is_empty()).then_some(Gate { + audit_name: &audit.name, + target, + reads, + }) } fn extra_reads<'a>(audit: &'a AuditGate, target: &ResourceKey) -> Vec<&'a ResourceKey> { @@ -135,6 +131,8 @@ fn edges_of( for read in &gate.reads { for key in &gated { edges.push(GateEdge { + audit_name: gate.audit_name.to_owned(), + target: gate.target.clone(), gated: key.clone(), read: (*read).clone(), }); diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs index a81fda7c77..077c9b0930 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/hook_reads.rs @@ -2,32 +2,16 @@ use std::collections::HashSet; -use crate::explicit_references::_helpers::audit_gates::GateEdge; use crate::explicit_references::_helpers::graph::{Upstream, lineage_upstream, reaches}; use crate::explicit_references::constants::{ HOOK_READ_CODE, MODEL_TYPE, PYTHON_DESTINATION, SEED_TYPE, SOURCE_TYPE, SQL_DESTINATION, }; -use crate::explicit_references::models::{PythonHook, ReferenceDiagnostic, ReferenceRequest}; +use crate::explicit_references::models::{ + GateEdge, HookEdge, HookType, ModelReferences, PythonHook, ReferenceDiagnostic, + ReferenceRequest, +}; use crate::explicit_references::types::ResourceKey; -/// How diagnostics name a hook. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum HookType { - Python, - Sql, - InlineSql, -} - -/// One ordering edge: model `gated` waits for `read`, which one of its hooks reads. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct HookEdge { - hook_name: String, - gated: ResourceKey, - read: ResourceKey, - hook_type: HookType, - path: Option, -} - /// The `CompileInputError` message for the first unknown read, hooks taken by name. pub(crate) fn unknown_read(request: &ReferenceRequest) -> Option { let mut hooks: Vec<&PythonHook> = request.hooks.iter().collect(); @@ -59,7 +43,7 @@ pub(crate) fn hook_diagnostics( request: &ReferenceRequest, gates: &[GateEdge], ) -> Vec { - let mut edges: Vec = hook_edges(request); + let mut edges: Vec = hook_edges(&request.models, &request.hooks); if edges.is_empty() { return Vec::new(); } @@ -91,19 +75,17 @@ pub(crate) fn hook_diagnostics( } /// `hook_read_edges_impl`: Python then SQL hook reads per model, deduplicated, minus self-reads. -fn hook_edges(request: &ReferenceRequest) -> Vec { +pub(crate) fn hook_edges(models: &[ModelReferences], hooks: &[PythonHook]) -> Vec { let mut candidates: Vec = Vec::new(); - for model in &request.models { + for model in models { let gated: ResourceKey = (MODEL_TYPE.to_owned(), model.name.clone()); for hook_name in &model.python_hooks { - let reads: &[ResourceKey] = request - .hooks + let reads: &[ResourceKey] = hooks .iter() .rev() .find(|hook| &hook.name == hook_name && !hook.reads.is_empty()) .map_or(&[], |hook| hook.reads.as_slice()); - let path: Option = request - .hooks + let path: Option = hooks .iter() .rev() .find(|hook| &hook.name == hook_name) diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs b/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs index 21f50afd64..c3e106bffe 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/check_explicit_references.rs @@ -23,7 +23,7 @@ fn checked(request: &ReferenceRequest) -> Result ExecutionGateEdges { + ExecutionGateEdges { + audit_gates: gate_edges(&request.resources, &request.audits), + hook_reads: hook_edges(&request.models, &request.hooks), + } +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs index 7cb27294a1..9e7e305311 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs @@ -1 +1,2 @@ pub mod check_explicit_references; +pub mod execution_gate_edges; diff --git a/crates/sqlbuild-analysis/src/explicit_references/models.rs b/crates/sqlbuild-analysis/src/explicit_references/models.rs index 6c730de62d..21021e13a4 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/models.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/models.rs @@ -34,7 +34,7 @@ pub struct GraphResource { } /// A relation name with the qualifiers written or configured. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Deserialize, Serialize)] pub struct Relation { pub name: String, pub schema: Option, @@ -46,6 +46,7 @@ pub struct Relation { pub struct ModelReferences { pub name: String, pub path: String, + #[serde(default)] pub destination: Relation, /// `model_python_hook_names(model)`. pub python_hooks: Vec, @@ -54,6 +55,50 @@ pub struct ModelReferences { pub migrate_from: Option, } +/// The facts execution ordering reads: models with hooks, audits and Python hooks. +#[derive(Debug, Clone, Deserialize)] +pub struct ExecutionGateRequest { + /// Models in project order with their dependencies. + pub resources: Vec, + pub models: Vec, + pub audits: Vec, + pub hooks: Vec, +} + +/// One ordering edge: `gated` waits for `read`, which audit `audit_name` on `target` reads. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GateEdge { + pub audit_name: String, + pub target: ResourceKey, + pub gated: ResourceKey, + pub read: ResourceKey, +} + +/// How diagnostics name a hook. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HookType { + Python, + Sql, + InlineSql, +} + +/// One ordering edge: model `gated` waits for `read`, which one of its hooks reads. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HookEdge { + pub hook_name: String, + pub gated: ResourceKey, + pub read: ResourceKey, + pub hook_type: HookType, + pub path: Option, +} + +/// Python's attached-audit gate edges and hook-read edges, in its order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ExecutionGateEdges { + pub audit_gates: Vec, + pub hook_reads: Vec, +} + /// One SQL hook entry with declared reads: its hook key, position and identity. #[derive(Debug, Clone, Deserialize)] pub struct SqlHook { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 0d09ab978a..c578031f6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -5,7 +5,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use serde_json::{Value, json}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; -use sqlbuild_analysis::explicit_references::models::{ReferenceReport, ReferenceRequest}; +use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::models::{ + ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, ReferenceReport, + ReferenceRequest, +}; +use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; @@ -66,6 +71,46 @@ fn python_literal_sql_calls( .collect() } +/// `(audit name, target, gated, read)` for one attached-audit ordering edge. +type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); +/// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. +type HookRow = ( + String, + ResourceKey, + ResourceKey, + &'static str, + Option, +); + +/// The attached-audit gate edges and hook-read edges of one project's facts, in Python's order. +#[pyfunction] +fn native_execution_gate_edges(request_json: &str) -> PyResult<(Vec, Vec)> { + let request: ExecutionGateRequest = serde_json::from_str(request_json).map_err(|error| { + compiler_error(native_failure( + EXPLICIT_REFERENCES_CONTEXT, + &error.to_string(), + )) + })?; + let edges: ExecutionGateEdges = execution_gate_edges(&request); + Ok(( + edges.audit_gates.into_iter().map(gate_row).collect(), + edges.hook_reads.into_iter().map(hook_row).collect(), + )) +} + +fn gate_row(edge: GateEdge) -> GateRow { + (edge.audit_name, edge.target, edge.gated, edge.read) +} + +fn hook_row(edge: HookEdge) -> HookRow { + let hook_type: &'static str = match edge.hook_type { + HookType::Python => "Python", + HookType::Sql => "SQL", + HookType::InlineSql => "inline SQL", + }; + (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) +} + fn report_value(report: &ReferenceReport) -> PyResult { serde_json::to_value(report).map_err(|error| { compiler_error(native_failure( @@ -78,5 +123,6 @@ fn report_value(report: &ReferenceReport) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(check_native_explicit_references, module)?)?; module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; + module.add_function(wrap_pyfunction!(native_execution_gate_edges, module)?)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index dabb0466e1..ecbfb6acfc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1817,6 +1817,12 @@ def assemble_project_resource_facts( tuple[str, int, str, str, str | None, int | None] | None, ]: ... def check_native_explicit_references(request_json: str, /) -> str: ... +def native_execution_gate_edges( + request_json: str, / +) -> tuple[ + list[tuple[str, tuple[str, str], tuple[str, str], tuple[str, str]]], + list[tuple[str, tuple[str, str], tuple[str, str], str, str | None]], +]: ... def schema_validation_rows( catalog: Any, dialect: str, diff --git a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py deleted file mode 100644 index 193ed835e1..0000000000 --- a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Ordering edges that let attached audits gate their target.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.graph.models import AttachedAuditGateEdge - -_READABLE_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( - { - CompiledResourceType.MODEL, - CompiledResourceType.SEED, - CompiledResourceType.SOURCE, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - } -) - - -@dataclass(frozen=True) -class _AuditGate: - audit_name: str - target_kind: AttachedAuditTargetKind - target: CompiledObjectKey - reads: tuple[CompiledObjectKey, ...] - - -def attached_audit_gate_edges_impl( - *, project: CompiledProjectView -) -> tuple[AttachedAuditGateEdge, ...]: - """Return edges from each extra audit read to its model or seed target, or source triggers.""" - - source_triggers: dict[CompiledObjectKey, set[CompiledObjectKey]] = {} - for model in project.models: - for dep in model.deps: - if dep.resource_type == CompiledResourceType.SOURCE: - source_triggers.setdefault(dep, set()).add(model.key) - gates: tuple[_AuditGate, ...] = _audit_gates(project=project) - edges: dict[AttachedAuditGateEdge, None] = {} - changed: bool = True - while changed: - changed = False - gate: _AuditGate - for gate in gates: - edge: AttachedAuditGateEdge - for edge in _gate_edges(gate=gate, source_triggers=source_triggers): - if edge in edges: - continue - edges[edge] = None - changed = True - if ( - edge.read.resource_type == CompiledResourceType.SOURCE - and edge.gated.resource_type == CompiledResourceType.MODEL - ): - source_triggers.setdefault(edge.read, set()).add(edge.gated) - return tuple(edges) - - -def _audit_gates(*, project: CompiledProjectView) -> tuple[_AuditGate, ...]: - gates: list[_AuditGate] = [] - audit: CompiledAuditView - for audit in project.audits: - if audit.attached_target_kind is None or audit.attached_target_name is None: - continue - target_kind: AttachedAuditTargetKind = AttachedAuditTargetKind(audit.attached_target_kind) - target: CompiledObjectKey = CompiledObjectKey( - resource_type=target_kind.resource_type, name=audit.attached_target_name - ) - reads: tuple[CompiledObjectKey, ...] = _extra_reads(audit=audit, target=target) - if reads: - gates.append( - _AuditGate( - audit_name=audit.name, target_kind=target_kind, target=target, reads=reads - ) - ) - return tuple(gates) - - -def _gate_edges( - *, - gate: _AuditGate, - source_triggers: dict[CompiledObjectKey, set[CompiledObjectKey]], -) -> tuple[AttachedAuditGateEdge, ...]: - gated_keys: tuple[CompiledObjectKey, ...] = ( - tuple(sorted(source_triggers.get(gate.target, ()), key=lambda key: key.name)) - if gate.target_kind is AttachedAuditTargetKind.SOURCE - else (gate.target,) - ) - edges: list[AttachedAuditGateEdge] = [] - read: CompiledObjectKey - for read in gate.reads: - edges.extend( - AttachedAuditGateEdge( - audit_name=gate.audit_name, target=gate.target, gated=gated, read=read - ) - for gated in gated_keys - ) - return tuple(edges) - - -def _extra_reads( - *, audit: CompiledAuditView, target: CompiledObjectKey -) -> tuple[CompiledObjectKey, ...]: - return tuple( - read - for read in audit.scope_deps - if read != target and read.resource_type in _READABLE_RESOURCE_TYPES - ) diff --git a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py new file mode 100644 index 0000000000..d2180b5c69 --- /dev/null +++ b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py @@ -0,0 +1,141 @@ +"""Ordering edges attached audits and model hooks add, computed natively from project facts.""" + +from __future__ import annotations + +from itertools import chain +from pathlib import Path + +import orjson + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledAuditView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge +from sqlbuild.compiler.graph.types import HookReadType, NativeKey +from sqlbuild.python_nodes.models import SqlResourceRef + +_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") + + +def model_python_hook_names_impl(model: CompiledModelView) -> tuple[str, ...]: + """Return the Python hooks attached to one model, in lifecycle order.""" + + names: list[str] = [] + for hook_key in _MODEL_HOOK_KEYS: + entries: object = model.config.values.get(hook_key) + if isinstance(entries, PythonHookEntry): + names.append(entries.name) + elif isinstance(entries, list | tuple): + names.extend(entry.name for entry in entries if isinstance(entry, PythonHookEntry)) + return tuple(dict.fromkeys(names)) + + +def attached_audit_gate_edges_impl( + *, project: CompiledProjectView +) -> tuple[AttachedAuditGateEdge, ...]: + """Return edges from each extra audit read to its model or seed target, or source triggers.""" + + gates, _ = _native.native_execution_gate_edges(_request(project)) + return tuple( + AttachedAuditGateEdge( + audit_name=audit_name, target=_key(target), gated=_key(gated), read=_key(read) + ) + for audit_name, target, gated, read in gates + ) + + +def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: + """Return edges from each hook read to the model that runs the hook.""" + + _, hooks = _native.native_execution_gate_edges(_request(project)) + return tuple( + HookReadEdge( + hook_name=hook_name, + gated=_key(gated), + read=_key(read), + hook_type=HookReadType(hook_type), + hook_path=None if path is None else Path(path), + ) + for hook_name, gated, read, hook_type, path in hooks + ) + + +def _key(pair: NativeKey) -> CompiledObjectKey: + return CompiledObjectKey(resource_type=CompiledResourceType(pair[0]), name=pair[1]) + + +def _request(project: CompiledProjectView) -> str: + return orjson.dumps( + { + "resources": [ + {"key": model.key_row, "deps": model.dep_rows} for model in project.models + ], + "models": [_model_row(model) for model in project.models], + "audits": [_audit_row(audit) for audit in project.audits], + "hooks": [ + { + "name": hook.name, + "path": hook.relative_path.as_posix(), + "reads": _ref_rows(hook.reads), + } + for hook in project.hook_functions + ], + } + ).decode() + + +def _model_row(model: CompiledModelView) -> dict[str, object]: + return { + "name": model.name, + "path": str(model.relative_path), + "python_hooks": list(model_python_hook_names_impl(model)), + "sql_hooks": list( + chain.from_iterable( + _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) + for hook_key in _MODEL_HOOK_KEYS + ) + ), + } + + +def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: + items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) + return [ + { + "hook_key": hook_key, + "index": index, + "name": entry.name, + "path": None if entry.relative_path is None else str(entry.relative_path), + "reads": _ref_rows(entry.reads), + } + for index, entry in enumerate(items) + if isinstance(entry, SqlHookEntry) and entry.reads + ] + + +def _audit_row(audit: CompiledAuditView) -> dict[str, object]: + target: tuple[str, str] | None = ( + ( + str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), + audit.attached_target_name, + ) + if audit.attached_target_kind is not None and audit.attached_target_name is not None + else None + ) + return { + "name": audit.name, + "path": "", + "target": target, + "scope_deps": [(str(key.resource_type), key.name) for key in audit.scope_deps], + } + + +def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: + return [(str(ref.kind), ref.name) for ref in refs] diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py deleted file mode 100644 index 003c47ceb4..0000000000 --- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Ordering edges from the resources model hooks read.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl -from sqlbuild.compiler.graph.models import HookReadEdge -from sqlbuild.compiler.graph.types import HookReadType -from sqlbuild.python_nodes.models import SqlResourceRef - -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") - - -def model_python_hook_names_impl(model: CompiledModelView) -> tuple[str, ...]: - """Return the Python hooks attached to one model, in lifecycle order.""" - - names: list[str] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - if isinstance(entries, PythonHookEntry): - names.append(entries.name) - elif isinstance(entries, list | tuple): - names.extend(entry.name for entry in entries if isinstance(entry, PythonHookEntry)) - return tuple(dict.fromkeys(names)) - - -def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: - """Return edges from each hook read to the model that runs the hook.""" - - reads_by_hook: dict[str, tuple[SqlResourceRef, ...]] = { - hook.name: hook.reads for hook in project.hook_functions if hook.reads - } - paths_by_hook: dict[str, Path] = { - hook.name: hook.relative_path for hook in project.hook_functions - } - candidates: list[HookReadEdge] = [] - for model in project.models: - for hook_name in model_python_hook_names_impl(model): - candidates.extend( - HookReadEdge( - hook_name=hook_name, - gated=model.key, - read=sql_ref_key_impl(ref), - hook_path=paths_by_hook.get(hook_name), - ) - for ref in reads_by_hook.get(hook_name, ()) - ) - candidates.extend(_sql_hook_read_edges(model)) - return tuple(dict.fromkeys(edge for edge in candidates if edge.read != edge.gated)) - - -def _sql_hook_read_edges(model: CompiledModelView) -> tuple[HookReadEdge, ...]: - edges: list[HookReadEdge] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - items: tuple[object, ...] = ( - tuple(entries) if isinstance(entries, list | tuple) else (entries,) - ) - for index, entry in enumerate(items): - if not isinstance(entry, SqlHookEntry) or not entry.reads: - continue - named: bool = entry.name is not None - edges.extend( - HookReadEdge( - hook_name=entry.name if entry.name is not None else f"{hook_key}[{index}]", - gated=model.key, - read=sql_ref_key_impl(ref), - hook_type=HookReadType.SQL if named else HookReadType.INLINE_SQL, - hook_path=entry.relative_path or model.relative_path, - ) - for ref in entry.reads - ) - return tuple(edges) diff --git a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py index d9ab03745d..292bd4d4bb 100644 --- a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py +++ b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl +from sqlbuild.compiler.graph._helpers.execution_gates import attached_audit_gate_edges_impl from sqlbuild.compiler.graph.models import AttachedAuditGateEdge diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py index 3dffa67abe..9d4fdaf66c 100644 --- a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py +++ b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_edges_impl +from sqlbuild.compiler.graph._helpers.execution_gates import hook_read_edges_impl from sqlbuild.compiler.graph.models import HookReadEdge diff --git a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py index b80abfb8c6..2a39037d7c 100644 --- a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py +++ b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.graph._helpers.hook_reads import model_python_hook_names_impl +from sqlbuild.compiler.graph._helpers.execution_gates import model_python_hook_names_impl def model_python_hook_names(*, model: CompiledModelView) -> tuple[str, ...]: From 84802f11e483d5b528706899ff652eeba2bb417b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:18:42 +0100 Subject: [PATCH 18/26] refactor: read the project lineage graph directly build_native_project_graph only returned project.lineage_graph; its callers now read the attribute. --- src/sqlbuild/compiler/graph/_helpers/native_graph.py | 6 ------ .../compiler/graph/main/_native_project_graph.py | 11 ----------- src/sqlbuild/compiler/planner/_helpers/graph/scope.py | 5 +---- src/sqlbuild/compiler/planner/_helpers/reuse/clone.py | 3 +-- 4 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 src/sqlbuild/compiler/graph/main/_native_project_graph.py diff --git a/src/sqlbuild/compiler/graph/_helpers/native_graph.py b/src/sqlbuild/compiler/graph/_helpers/native_graph.py index 9821d8e8dd..d22ce9832d 100644 --- a/src/sqlbuild/compiler/graph/_helpers/native_graph.py +++ b/src/sqlbuild/compiler/graph/_helpers/native_graph.py @@ -43,12 +43,6 @@ def native_keys(keys: Iterable[CompiledObjectKey]) -> list[NativeKey]: return [native_key(key) for key in keys] -def build_native_project_graph_impl(project: CompiledProjectView) -> _native.NativeProjectGraph: - """The project's native graph, built once per compiled project.""" - - return project.lineage_graph - - def project_lineage_views_impl(project: CompiledProjectView) -> LineageGraphViews: """The project's lineage edges and selector indexes as fresh dicts.""" diff --git a/src/sqlbuild/compiler/graph/main/_native_project_graph.py b/src/sqlbuild/compiler/graph/main/_native_project_graph.py deleted file mode 100644 index 5032d11637..0000000000 --- a/src/sqlbuild/compiler/graph/main/_native_project_graph.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Native project graph entrypoint.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.graph._helpers.native_graph import build_native_project_graph_impl - - -def build_native_project_graph(project: CompiledProjectView) -> _native.NativeProjectGraph: - """Index the project's lineage edges, tags, folders and names natively.""" - - return build_native_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index 04af510c1f..1259b69cca 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -5,7 +5,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.graph.main._lineage_graph_names import lineage_graph_names -from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, @@ -47,9 +46,7 @@ def build_planner_scope( resolved_selected_keys: frozenset[CompiledObjectKey] = ( selected_keys if selected_keys is not None - else resolve_graph_selectors( - graph=build_native_project_graph(project), select=select, exclude=exclude - ) + else resolve_graph_selectors(graph=project.lineage_graph, select=select, exclude=exclude) ) executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( _upstream_source_selector_keys( diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index d5320eac16..02625c3ac9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -15,7 +15,6 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, build_execution_upstream_deps, @@ -69,7 +68,7 @@ def build_clone_plan_output( upstream_deps ) selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selectors( - graph=build_native_project_graph(project), select=select, exclude=exclude + graph=project.lineage_graph, select=select, exclude=exclude ) return PlanOutput( execution_order=topologically_order_keys(upstream=upstream_deps), From ab727d30341394005684361728af4cb28173bbc6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:03:05 +0100 Subject: [PATCH 19/26] refactor: build fingerprint table SQL and decode latest fingerprint rows natively --- .../fingerprint_storage/_helpers/base64.rs | 113 ++++++++ .../fingerprint_storage/_helpers/literals.rs | 20 ++ .../src/fingerprint_storage/_helpers/mod.rs | 4 + .../_helpers/table_columns.rs | 8 + .../src/fingerprint_storage/_helpers/utf8.rs | 31 +++ .../src/fingerprint_storage/constants.rs | 33 +++ .../src/fingerprint_storage/errors.rs | 37 +++ .../main/create_table_sql.rs | 26 ++ .../main/decode_storage.rs | 10 + .../main/insert_batch_lengths.rs | 42 +++ .../fingerprint_storage/main/insert_sql.rs | 41 +++ .../fingerprint_storage/main/latest_index.rs | 39 +++ .../src/fingerprint_storage/main/mod.rs | 8 + .../main/read_latest_sql.rs | 18 ++ .../main/relevant_read_sql.rs | 34 +++ .../main/selected_positions.rs | 24 ++ .../src/fingerprint_storage/mod.rs | 10 + .../src/fingerprint_storage/models.rs | 41 +++ .../src/fingerprint_storage/tests.rs | 6 + .../src/fingerprint_storage/tests/helpers.rs | 19 ++ .../fingerprint_storage/tests/test_storage.rs | 209 +++++++++++++++ .../fingerprint_storage/tests/test_types.rs | 28 ++ crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/fingerprints/latest_rows.rs | 241 ++++++++++++++++++ .../src/bindings/_helpers/fingerprints/mod.rs | 3 + .../_helpers/fingerprints/registration.rs | 10 + .../_helpers/fingerprints/storage_sql.rs | 116 +++++++++ .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 7 +- src/sqlbuild/_native.pyi | 25 ++ .../compiler/fingerprints/_helpers/latest.py | 25 -- .../compiler/fingerprints/_helpers/sql.py | 178 ++----------- .../fingerprints/_helpers/write_many.py | 68 +---- .../fingerprints/main/_select_latest.py | 15 +- .../compiler/fingerprints/main/read.py | 78 +----- 35 files changed, 1247 insertions(+), 322 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/models.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs delete mode 100644 src/sqlbuild/compiler/fingerprints/_helpers/latest.py diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs new file mode 100644 index 0000000000..5de163d71b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/base64.rs @@ -0,0 +1,113 @@ +//! Standard-alphabet base64 as Python's `base64.b64encode` and strict `b64decode` apply it. + +const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const PAD: u8 = b'='; +const SECOND_BYTE: usize = 1; +const THIRD_BYTE: usize = 2; +const PAD_START_POSITION: usize = 2; +const QUAD_LENGTH: usize = 4; + +/// `base64.b64encode(value).decode("ascii")`. +pub(crate) fn encode(value: &[u8]) -> String { + let mut encoded = String::with_capacity(value.len().div_ceil(3) * 4); + for chunk in value.chunks(3) { + let first = chunk[0]; + let second = chunk.get(1).copied().unwrap_or(0); + let third = chunk.get(2).copied().unwrap_or(0); + encoded.push(char::from(ALPHABET[usize::from(first >> 2)])); + encoded.push(char::from( + ALPHABET[usize::from(((first & 0x03) << 4) | (second >> 4))], + )); + if chunk.len() > SECOND_BYTE { + encoded.push(char::from( + ALPHABET[usize::from(((second & 0x0f) << 2) | (third >> 6))], + )); + } else { + encoded.push(char::from(PAD)); + } + if chunk.len() > THIRD_BYTE { + encoded.push(char::from(ALPHABET[usize::from(third & 0x3f)])); + } else { + encoded.push(char::from(PAD)); + } + } + encoded +} + +/// `binascii.a2b_base64(text, strict_mode=True)`, with CPython's error messages. +pub(crate) fn decode_strict(text: &[u8]) -> Result, String> { + if text.first() == Some(&PAD) { + return Err("Leading padding not allowed".to_owned()); + } + let mut decoded: Vec = Vec::with_capacity(text.len() / 4 * 3); + let mut quad_position: usize = 0; + let mut left_char: u8 = 0; + let mut pads: usize = 0; + let mut padding_started = false; + for (index, &character) in text.iter().enumerate() { + if character == PAD { + padding_started = true; + if quad_position == 0 { + return Err("Excess padding not allowed".to_owned()); + } + if quad_position >= PAD_START_POSITION { + pads += 1; + if quad_position + pads >= QUAD_LENGTH { + if index + 1 < text.len() { + return Err("Excess data after padding".to_owned()); + } + return Ok(decoded); + } + } + continue; + } + let Some(value) = sextet(character) else { + return Err("Only base64 data is allowed".to_owned()); + }; + if padding_started { + return Err("Discontinuous padding not allowed".to_owned()); + } + pads = 0; + match quad_position { + 0 => { + quad_position = 1; + left_char = value; + } + 1 => { + quad_position = 2; + decoded.push((left_char << 2) | (value >> 4)); + left_char = value & 0x0f; + } + 2 => { + quad_position = 3; + decoded.push((left_char << 4) | (value >> 2)); + left_char = value & 0x03; + } + _ => { + quad_position = 0; + decoded.push((left_char << 6) | value); + left_char = 0; + } + } + } + match quad_position { + 0 => Ok(decoded), + 1 => Err(format!( + "Invalid base64-encoded string: number of data characters ({}) cannot be 1 more \ + than a multiple of 4", + decoded.len() / 3 * 4 + 1 + )), + _ => Err("Incorrect padding".to_owned()), + } +} + +fn sextet(character: u8) -> Option { + match character { + b'A'..=b'Z' => Some(character - b'A'), + b'a'..=b'z' => Some(character - b'a' + 26), + b'0'..=b'9' => Some(character - b'0' + 52), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs new file mode 100644 index 0000000000..96ece2e4d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/literals.rs @@ -0,0 +1,20 @@ +//! SQL string literals for fingerprint rows. + +/// `'value'` with single quotes doubled. +pub(crate) fn string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// `NULL`, or the quoted literal. +pub(crate) fn optional_string_literal(value: Option<&str>) -> String { + value.map_or_else(|| "NULL".to_owned(), string_literal) +} + +/// `'a', 'b'` for an `IN` list. +pub(crate) fn literal_list(values: &[String]) -> String { + values + .iter() + .map(|value| string_literal(value)) + .collect::>() + .join(", ") +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs new file mode 100644 index 0000000000..7a2f4a0fa2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod base64; +pub(crate) mod literals; +pub(crate) mod table_columns; +pub(crate) mod utf8; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs new file mode 100644 index 0000000000..4fd7db8669 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/table_columns.rs @@ -0,0 +1,8 @@ +//! The stored column list. + +use crate::fingerprint_storage::constants::FINGERPRINT_COLUMNS; + +/// Every stored column, comma separated. +pub(crate) fn column_list() -> String { + FINGERPRINT_COLUMNS.join(", ") +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs new file mode 100644 index 0000000000..b872db77d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/_helpers/utf8.rs @@ -0,0 +1,31 @@ +//! UTF-8 decoding failures described as CPython's decoder reports them. + +use crate::fingerprint_storage::errors::StorageDecodeError; + +/// `bytes` as text, or Python's `UnicodeDecodeError` range and reason. +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + let error = match String::from_utf8(bytes) { + Ok(text) => return Ok(text), + Err(error) => error, + }; + let utf8_error = error.utf8_error(); + let bytes: Vec = error.into_bytes(); + let start: usize = utf8_error.valid_up_to(); + let (end, reason): (usize, &'static str) = match utf8_error.error_len() { + None => (bytes.len(), "unexpected end of data"), + Some(length) if length == 1 && invalid_start_byte(bytes[start]) => { + (start + 1, "invalid start byte") + } + Some(length) => (start + length, "invalid continuation byte"), + }; + Err(StorageDecodeError::Utf8 { + bytes, + start, + end, + reason, + }) +} + +fn invalid_start_byte(byte: u8) -> bool { + matches!(byte, 0x80..=0xC1 | 0xF5..=0xFF) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs new file mode 100644 index 0000000000..17aad5e650 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -0,0 +1,33 @@ +//! Fingerprint table column names, in table order. + +pub const COLUMN_NODE_TYPE: &str = "node_type"; +pub const COLUMN_NODE_NAME: &str = "node_name"; +pub const COLUMN_TARGET_DATABASE: &str = "target_database"; +pub const COLUMN_TARGET_SCHEMA: &str = "target_schema"; +pub const COLUMN_TARGET_NAME: &str = "target_name"; +pub const COLUMN_RUN_ID: &str = "run_id"; +pub const COLUMN_DEFINITION_HASH: &str = "definition_hash"; +pub const COLUMN_VERSION_HASH: &str = "version_hash"; +pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; +pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; +pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; +pub const COLUMN_TIMESTAMP: &str = "ts"; + +/// Every stored column, in table order. +pub const FINGERPRINT_COLUMNS: [&str; 12] = [ + COLUMN_NODE_TYPE, + COLUMN_NODE_NAME, + COLUMN_TARGET_DATABASE, + COLUMN_TARGET_SCHEMA, + COLUMN_TARGET_NAME, + COLUMN_RUN_ID, + COLUMN_DEFINITION_HASH, + COLUMN_VERSION_HASH, + COLUMN_SCHEMA_FINGERPRINT, + COLUMN_DEFINITION_B64, + COLUMN_METADATA_JSON_B64, + COLUMN_TIMESTAMP, +]; + +/// The node type that wins a node-name collision between latest rows. +pub const NODE_TYPE_MODEL: &str = "model"; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs new file mode 100644 index 0000000000..6a10d0fb8b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/errors.rs @@ -0,0 +1,37 @@ +//! Failures of encoding and decoding fingerprint rows. + +use std::fmt::{self, Display}; + +/// Invalid insert input, with Python's `FingerprintInputError` message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FingerprintStorageError { + EmptyInsert, + NonPositiveBatchSize, + NonPositiveBatchBytes, +} + +impl Display for FingerprintStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::EmptyInsert => "fingerprint insert requires at least one row", + Self::NonPositiveBatchSize => "fingerprint batch size must be positive", + Self::NonPositiveBatchBytes => "fingerprint batch byte limit must be positive", + }) + } +} + +impl std::error::Error for FingerprintStorageError {} + +/// Why stored base64 text is not base64-encoded UTF-8, as Python's decoders report it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageDecodeError { + /// `binascii.Error` from strict base64 decoding. + Base64(String), + /// `UnicodeDecodeError` from UTF-8 decoding the base64 payload. + Utf8 { + bytes: Vec, + start: usize, + end: usize, + reason: &'static str, + }, +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs new file mode 100644 index 0000000000..7c0a00587e --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -0,0 +1,26 @@ +//! Fingerprint table DDL. + +use crate::fingerprint_storage::constants::{ + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, + COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, + COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, +}; + +/// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. +pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: &str) -> String { + format!( + "CREATE TABLE IF NOT EXISTS {qualified_name} (\ + {COLUMN_NODE_TYPE} {string_type} NOT NULL, \ + {COLUMN_NODE_NAME} {string_type} NOT NULL, \ + {COLUMN_TARGET_DATABASE} {string_type}, \ + {COLUMN_TARGET_SCHEMA} {string_type}, \ + {COLUMN_TARGET_NAME} {string_type}, \ + {COLUMN_RUN_ID} {string_type} NOT NULL, \ + {COLUMN_DEFINITION_HASH} {string_type} NOT NULL, \ + {COLUMN_VERSION_HASH} {string_type} NOT NULL, \ + {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ + {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ + {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + ) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs new file mode 100644 index 0000000000..3c224af96d --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/decode_storage.rs @@ -0,0 +1,10 @@ +//! Stored definition text decoding. + +use crate::fingerprint_storage::_helpers::base64::decode_strict; +use crate::fingerprint_storage::_helpers::utf8::decode_utf8; +use crate::fingerprint_storage::errors::StorageDecodeError; + +/// Stored ASCII base64 text decoded to UTF-8, failing as Python's strict decoders do. +pub fn decode_storage(storage: &str) -> Result { + decode_utf8(decode_strict(storage.as_bytes()).map_err(StorageDecodeError::Base64)?) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs new file mode 100644 index 0000000000..b6f2f90abb --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_batch_lengths.rs @@ -0,0 +1,42 @@ +//! Bounded fingerprint insert batches. + +use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::insert_sql::insert_sql; +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Batch lengths in order, each within `batch_size` rows and `batch_max_bytes` of row SQL. +pub fn insert_batch_lengths( + qualified_name: &str, + fingerprints: &[StoredFingerprint], + batch_size: i64, + batch_max_bytes: i64, +) -> Result, FingerprintStorageError> { + if batch_size < 1 { + return Err(FingerprintStorageError::NonPositiveBatchSize); + } + if batch_max_bytes < 1 { + return Err(FingerprintStorageError::NonPositiveBatchBytes); + } + let mut lengths: Vec = Vec::new(); + let mut current_length: usize = 0; + let mut current_bytes: i64 = 0; + for fingerprint in fingerprints { + let single_row_bytes: i64 = + i64::try_from(insert_sql(qualified_name, std::slice::from_ref(fingerprint))?.len()) + .unwrap_or(i64::MAX); + if current_length > 0 + && (i64::try_from(current_length).unwrap_or(i64::MAX) >= batch_size + || current_bytes.saturating_add(single_row_bytes) > batch_max_bytes) + { + lengths.push(current_length); + current_length = 0; + current_bytes = 0; + } + current_length += 1; + current_bytes = current_bytes.saturating_add(single_row_bytes); + } + if current_length > 0 { + lengths.push(current_length); + } + Ok(lengths) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs new file mode 100644 index 0000000000..25e54bf696 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -0,0 +1,41 @@ +//! Append-only fingerprint inserts. + +use crate::fingerprint_storage::_helpers::base64::encode; +use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; +use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::models::StoredFingerprint; + +/// One multi-row `INSERT` for a non-empty batch. +pub fn insert_sql( + qualified_name: &str, + fingerprints: &[StoredFingerprint], +) -> Result { + if fingerprints.is_empty() { + return Err(FingerprintStorageError::EmptyInsert); + } + let rows: Vec = fingerprints.iter().map(row_values).collect(); + Ok(format!( + "INSERT INTO {qualified_name} ({}) VALUES {}", + column_list(), + rows.join(", ") + )) +} + +fn row_values(fingerprint: &StoredFingerprint) -> String { + let values: [String; 12] = [ + string_literal(&fingerprint.node_type), + string_literal(&fingerprint.node_name), + optional_string_literal(fingerprint.target_database.as_deref()), + optional_string_literal(fingerprint.target_schema.as_deref()), + optional_string_literal(fingerprint.target_name.as_deref()), + format!("'{}'", fingerprint.run_id), + format!("'{}'", fingerprint.definition_hash), + format!("'{}'", fingerprint.version_hash), + format!("'{}'", fingerprint.schema_fingerprint), + format!("'{}'", encode(fingerprint.definition.as_bytes())), + format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), + format!("'{}'", fingerprint.ts), + ]; + format!("({})", values.join(", ")) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs new file mode 100644 index 0000000000..6a3a61f131 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/latest_index.rs @@ -0,0 +1,39 @@ +//! Latest fingerprint rows indexed by identity and by node name. + +use std::collections::HashMap; + +use crate::fingerprint_storage::constants::NODE_TYPE_MODEL; +use crate::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; + +/// Kept row positions in Python dict order, a model winning a node-name collision. +pub fn latest_index(identities: &[FingerprintIdentity<'_>]) -> LatestIndex { + let mut by_name: Vec = Vec::new(); + let mut name_slots: HashMap<&str, usize> = HashMap::new(); + let mut by_identity: Vec = Vec::new(); + let mut identity_slots: HashMap<(&str, &str), usize> = HashMap::new(); + for (position, identity) in identities.iter().enumerate() { + let key = (identity.node_type, identity.node_name); + match identity_slots.get(&key) { + Some(&slot) => by_identity[slot] = position, + None => { + let _ = identity_slots.insert(key, by_identity.len()); + by_identity.push(position); + } + } + match name_slots.get(identity.node_name) { + Some(&slot) => { + if identity.node_type == NODE_TYPE_MODEL { + by_name[slot] = position; + } + } + None => { + let _ = name_slots.insert(identity.node_name, by_name.len()); + by_name.push(position); + } + } + } + LatestIndex { + by_name, + by_identity, + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs new file mode 100644 index 0000000000..54c1f22e6f --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -0,0 +1,8 @@ +pub mod create_table_sql; +pub mod decode_storage; +pub mod insert_batch_lengths; +pub mod insert_sql; +pub mod latest_index; +pub mod read_latest_sql; +pub mod relevant_read_sql; +pub mod selected_positions; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs new file mode 100644 index 0000000000..b6d1baeac6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/read_latest_sql.rs @@ -0,0 +1,18 @@ +//! The latest-row read of the fingerprint table. + +use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::{ + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_TIMESTAMP, +}; + +/// A windowed `SELECT` of the latest row per node identity. +pub fn read_latest_sql(qualified_name: &str) -> String { + let columns: String = column_list(); + format!( + "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (\ + PARTITION BY {COLUMN_NODE_TYPE}, {COLUMN_NODE_NAME} \ + ORDER BY {COLUMN_TIMESTAMP} DESC, {COLUMN_RUN_ID} DESC) AS __sqlbuild_latest_rank \ + FROM {qualified_name}) AS __sqlbuild_latest_fingerprints \ + WHERE __sqlbuild_latest_rank = 1" + ) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs new file mode 100644 index 0000000000..487a79bea2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/relevant_read_sql.rs @@ -0,0 +1,34 @@ +//! The latest-row read narrowed to the relevant nodes. + +use crate::fingerprint_storage::_helpers::literals::literal_list; +use crate::fingerprint_storage::models::RelevantRead; + +/// The read limited to `node_names` plus unfiltered node types; `None` names keep every row. +pub fn relevant_read_sql( + read_sql: &str, + node_names: Option<&[String]>, + filtered_node_types: &[String], +) -> RelevantRead { + let Some(node_names) = node_names else { + return RelevantRead::Sql(read_sql.to_owned()); + }; + if node_names.is_empty() && filtered_node_types.is_empty() { + return RelevantRead::Empty; + } + let mut predicate = String::new(); + if !node_names.is_empty() { + predicate = format!("node_name IN ({})", literal_list(node_names)); + } + if !filtered_node_types.is_empty() { + let identity_predicate: String = + format!("node_type NOT IN ({})", literal_list(filtered_node_types)); + predicate = if predicate.is_empty() { + identity_predicate + } else { + format!("{predicate} OR {identity_predicate}") + }; + } + RelevantRead::Sql(format!( + "SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE {predicate}" + )) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs new file mode 100644 index 0000000000..b38d2bda84 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/selected_positions.rs @@ -0,0 +1,24 @@ +//! In-memory node selection over unfiltered latest rows. + +use std::collections::HashSet; + +use crate::fingerprint_storage::models::FingerprintIdentity; + +/// Positions of the rows a narrowed latest read would have returned. +pub fn selected_positions( + identities: &[FingerprintIdentity<'_>], + node_names: &[String], + filtered_node_types: &[String], +) -> Vec { + let names: HashSet<&str> = node_names.iter().map(String::as_str).collect(); + let types: HashSet<&str> = filtered_node_types.iter().map(String::as_str).collect(); + identities + .iter() + .enumerate() + .filter(|(_, identity)| { + names.contains(identity.node_name) + || (!types.is_empty() && !types.contains(identity.node_type)) + }) + .map(|(position, _)| position) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs new file mode 100644 index 0000000000..35a2bdafe7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/mod.rs @@ -0,0 +1,10 @@ +//! The append-only fingerprint table: its DDL, latest-row reads, batched inserts and row decoding. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs new file mode 100644 index 0000000000..fdd309019b --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/models.rs @@ -0,0 +1,41 @@ +//! Fingerprint rows, read narrowing and latest-row indexes. + +/// One fingerprint row to insert, with its timestamp already in ISO format. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredFingerprint { + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: String, +} + +/// How a latest-fingerprint read is narrowed to the relevant nodes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RelevantRead { + /// Read every latest row with this SQL. + Sql(String), + /// Nothing can match; skip the read. + Empty, +} + +/// One latest fingerprint's identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FingerprintIdentity<'a> { + pub node_type: &'a str, + pub node_name: &'a str, +} + +/// Positions of the latest rows kept per node name and per identity, in Python dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LatestIndex { + pub by_name: Vec, + pub by_identity: Vec, +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs new file mode 100644 index 0000000000..2f1e84c032 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_storage.rs"] +mod storage; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs new file mode 100644 index 0000000000..9af85b3a35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/helpers.rs @@ -0,0 +1,19 @@ +use crate::fingerprint_storage::models::StoredFingerprint; + +/// A model fingerprint row with `definition` and fixed other fields. +pub(crate) fn stored_fingerprint(definition: String) -> StoredFingerprint { + StoredFingerprint { + node_type: "model".to_owned(), + node_name: "o'rders".to_owned(), + target_database: None, + target_schema: Some("main".to_owned()), + target_name: Some("orders".to_owned()), + run_id: "run".to_owned(), + definition_hash: "d".to_owned(), + version_hash: String::new(), + schema_fingerprint: "s".to_owned(), + definition, + metadata_json: "{}".to_owned(), + ts: "2026-01-01T00:00:00".to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs new file mode 100644 index 0000000000..59c72c1dd6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -0,0 +1,209 @@ +use crate::fingerprint_storage::errors::{FingerprintStorageError, StorageDecodeError}; +use crate::fingerprint_storage::main::decode_storage::decode_storage; +use crate::fingerprint_storage::main::insert_batch_lengths::insert_batch_lengths; +use crate::fingerprint_storage::main::insert_sql::insert_sql; +use crate::fingerprint_storage::main::latest_index::latest_index; +use crate::fingerprint_storage::models::{FingerprintIdentity, LatestIndex, StoredFingerprint}; +use crate::fingerprint_storage::tests::helpers::stored_fingerprint; +use crate::fingerprint_storage::tests::test_types::{ + BatchTestCase, DecodeTestCase, InsertTestCase, LatestIndexTestCase, +}; + +#[test] +fn given_stored_text_when_decoding_then_python_strict_base64_and_utf8_rules_apply() { + let test_cases = [ + DecodeTestCase { + description: "padded multi-byte text decodes", + storage: "4oKsQQ==", + expected_decoded: Ok("\u{20ac}A"), + }, + DecodeTestCase { + description: "empty storage is empty text", + storage: "", + expected_decoded: Ok(""), + }, + DecodeTestCase { + description: "leading padding is rejected", + storage: "=", + expected_decoded: Err(StorageDecodeError::Base64( + "Leading padding not allowed".to_owned(), + )), + }, + DecodeTestCase { + description: "a missing pad is incorrect padding", + storage: "QQ=", + expected_decoded: Err(StorageDecodeError::Base64("Incorrect padding".to_owned())), + }, + DecodeTestCase { + description: "one data character past a quad reports the count", + storage: "QUJDR", + expected_decoded: Err(StorageDecodeError::Base64( + "Invalid base64-encoded string: number of data characters (5) cannot be 1 more \ + than a multiple of 4" + .to_owned(), + )), + }, + DecodeTestCase { + description: "data after padding is rejected", + storage: "QQ=A", + expected_decoded: Err(StorageDecodeError::Base64( + "Discontinuous padding not allowed".to_owned(), + )), + }, + DecodeTestCase { + description: "a truncated sequence is an unexpected end", + storage: "4oI=", + expected_decoded: Err(StorageDecodeError::Utf8 { + bytes: vec![0xe2, 0x82], + start: 0, + end: 2, + reason: "unexpected end of data", + }), + }, + DecodeTestCase { + description: "a surrogate lead is an invalid continuation", + storage: "7aCA", + expected_decoded: Err(StorageDecodeError::Utf8 { + bytes: vec![0xed, 0xa0, 0x80], + start: 0, + end: 1, + reason: "invalid continuation byte", + }), + }, + ]; + for test_case in test_cases { + assert_eq!( + decode_storage(test_case.storage).as_deref(), + test_case.expected_decoded.as_ref().copied(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprints_when_batching_then_row_and_byte_limits_split_batches() { + let test_cases = [ + BatchTestCase { + description: "the row limit splits batches", + definition_lengths: &[1, 1, 1, 1, 1], + batch_size: 2, + batch_max_bytes: 1_000_000, + expected_lengths: Ok(&[2, 2, 1]), + }, + BatchTestCase { + description: "an oversized row still forms its own batch", + definition_lengths: &[10, 5_000, 10], + batch_size: 50, + batch_max_bytes: 1_000, + expected_lengths: Ok(&[1, 1, 1]), + }, + BatchTestCase { + description: "no fingerprints form no batches", + definition_lengths: &[], + batch_size: 50, + batch_max_bytes: 1_000, + expected_lengths: Ok(&[]), + }, + BatchTestCase { + description: "a zero batch size is rejected", + definition_lengths: &[1], + batch_size: 0, + batch_max_bytes: 1_000, + expected_lengths: Err(FingerprintStorageError::NonPositiveBatchSize), + }, + ]; + for test_case in test_cases { + let fingerprints: Vec = test_case + .definition_lengths + .iter() + .map(|length| stored_fingerprint("x".repeat(*length))) + .collect(); + assert_eq!( + insert_batch_lengths( + "main._sqlbuild_fingerprints", + &fingerprints, + test_case.batch_size, + test_case.batch_max_bytes, + ), + test_case.expected_lengths.map(<[usize]>::to_vec), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() { + let test_cases = [ + InsertTestCase { + description: "one row quotes names and encodes the definition", + definitions: &["ab"], + expected_sql: Ok( + "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ + target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ + definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + ), + }, + InsertTestCase { + description: "an empty batch is rejected", + definitions: &[], + expected_sql: Err(FingerprintStorageError::EmptyInsert), + }, + ]; + for test_case in test_cases { + let fingerprints: Vec = test_case + .definitions + .iter() + .map(|definition| stored_fingerprint((*definition).to_owned())) + .collect(); + assert_eq!( + insert_sql("main.t", &fingerprints), + test_case.expected_sql.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_name_collisions_when_indexing_latest_rows_then_models_win_and_order_is_kept() { + let test_cases = [ + LatestIndexTestCase { + description: "a later model replaces an earlier name; other types do not", + identities: &[ + ("udf", "orders"), + ("seed", "customers"), + ("model", "orders"), + ("hook", "orders"), + ("udf", "orders"), + ], + expected_index: LatestIndex { + by_name: vec![2, 1], + by_identity: vec![4, 1, 2, 3], + }, + }, + LatestIndexTestCase { + description: "no rows index nothing", + identities: &[], + expected_index: LatestIndex::default(), + }, + ]; + for test_case in test_cases { + let identities: Vec> = test_case + .identities + .iter() + .map(|(node_type, node_name)| FingerprintIdentity { + node_type, + node_name, + }) + .collect(); + assert_eq!( + latest_index(&identities), + test_case.expected_index, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs new file mode 100644 index 0000000000..048e5cf649 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::fingerprint_storage::errors::{FingerprintStorageError, StorageDecodeError}; +use crate::fingerprint_storage::models::LatestIndex; + +pub(crate) struct DecodeTestCase { + pub(crate) description: &'static str, + pub(crate) storage: &'static str, + pub(crate) expected_decoded: Result<&'static str, StorageDecodeError>, +} + +pub(crate) struct BatchTestCase { + pub(crate) description: &'static str, + pub(crate) definition_lengths: &'static [usize], + pub(crate) batch_size: i64, + pub(crate) batch_max_bytes: i64, + pub(crate) expected_lengths: Result<&'static [usize], FingerprintStorageError>, +} + +pub(crate) struct InsertTestCase { + pub(crate) description: &'static str, + pub(crate) definitions: &'static [&'static str], + pub(crate) expected_sql: Result<&'static str, FingerprintStorageError>, +} + +pub(crate) struct LatestIndexTestCase { + pub(crate) description: &'static str, + pub(crate) identities: &'static [(&'static str, &'static str)], + pub(crate) expected_index: LatestIndex, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index d7d585de55..46361823e6 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -11,6 +11,7 @@ mod constants; pub mod contracts; pub mod dag_artifact; pub mod explicit_references; +pub mod fingerprint_storage; pub mod graph; pub mod lineage; pub mod manifest; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs new file mode 100644 index 0000000000..676b2266c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -0,0 +1,241 @@ +//! Latest fingerprint rows decoded into `Fingerprint` models and indexed into a `FingerprintSet`. + +use std::ffi::CString; + +use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDateTime, PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::errors::StorageDecodeError; +use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage; +use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; +use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; +use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const DATETIME_MODULE: &str = "datetime"; +const BINASCII_MODULE: &str = "binascii"; +const TIMESTAMP_COLUMN: usize = 11; + +/// `FingerprintInputError(message)`. +pub(crate) fn fingerprint_input_error(py: Python<'_>, message: &str) -> PyErr { + match py + .import(EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("FingerprintInputError")) + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The `FingerprintSet` of fetched latest rows, a model winning a node-name collision. +#[pyfunction] +fn fingerprint_set_from_rows<'py>( + py: Python<'py>, + schema: &str, + qualified_name: &str, + rows: &Bound<'py, PyAny>, +) -> PyResult> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let fingerprint_class: Bound<'py, PyAny> = models.getattr("Fingerprint")?; + let from_iso: Bound<'py, PyAny> = py + .import(DATETIME_MODULE)? + .getattr("datetime")? + .getattr("fromisoformat")?; + let mut fingerprints: Vec> = Vec::new(); + for row in rows.try_iter()? { + fingerprints.push(row_fingerprint( + &fingerprint_class, + &from_iso, + &row?, + qualified_name, + )?); + } + indexed_set(&models, schema, &fingerprints) +} + +/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. +#[pyfunction] +fn select_latest_fingerprints<'py>( + py: Python<'py>, + fingerprint_set: &Bound<'py, PyAny>, + node_names: Vec, + filtered_node_types: Vec, +) -> PyResult> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let schema: String = fingerprint_set.getattr("schema")?.extract()?; + if node_names.is_empty() && filtered_node_types.is_empty() { + return indexed_set(&models, &schema, &[]); + } + let by_identity: Bound<'py, PyAny> = fingerprint_set.getattr("fingerprints_by_identity")?; + let latest: Vec> = if by_identity.is_truthy()? { + by_identity + .call_method0("values")? + .try_iter()? + .collect::>()? + } else { + Vec::new() + }; + let identities: Vec<(String, String)> = + latest.iter().map(identity_of).collect::>()?; + let positions: Vec = + selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); + let selected: Vec> = positions + .into_iter() + .map(|position| latest[position].clone()) + .collect(); + indexed_set(&models, &schema, &selected) +} + +fn indexed_set<'py>( + models: &Bound<'py, PyModule>, + schema: &str, + fingerprints: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = models.py(); + let identities: Vec<(String, String)> = fingerprints + .iter() + .map(identity_of) + .collect::>()?; + let index: LatestIndex = latest_index(&borrowed(&identities)); + let by_name: Bound<'py, PyDict> = PyDict::new(py); + for position in index.by_name { + by_name.set_item(&identities[position].1, &fingerprints[position])?; + } + let by_identity: Bound<'py, PyDict> = PyDict::new(py); + for position in index.by_identity { + by_identity.set_item(identities[position].clone(), &fingerprints[position])?; + } + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("schema", schema)?; + kwargs.set_item("fingerprints", by_name)?; + kwargs.set_item("fingerprints_by_identity", by_identity)?; + models.getattr("FingerprintSet")?.call((), Some(&kwargs)) +} + +fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + fingerprint.getattr("node_type")?.extract()?, + fingerprint.getattr("node_name")?.extract()?, + )) +} + +fn borrowed(identities: &[(String, String)]) -> Vec> { + identities + .iter() + .map(|(node_type, node_name)| FingerprintIdentity { + node_type, + node_name, + }) + .collect() +} + +fn row_fingerprint<'py>( + fingerprint_class: &Bound<'py, PyAny>, + from_iso: &Bound<'py, PyAny>, + row: &Bound<'py, PyAny>, + qualified_name: &str, +) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; + let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { + raw_ts + } else { + from_iso.call1((raw_ts.str()?,))? + }; + let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; + let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; + let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; + let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; + let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; + let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; + let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; + let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; + let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("node_type", node_type)?; + kwargs.set_item("node_name", node_name)?; + kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; + kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; + kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; + kwargs.set_item("run_id", row.get_item(5)?.str()?)?; + kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; + kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; + kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; + kwargs.set_item("definition", definition)?; + kwargs.set_item("metadata_json", metadata_json)?; + kwargs.set_item("ts", ts)?; + fingerprint_class.call((), Some(&kwargs)) +} + +fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + if value.is_none() { + return Ok(None); + } + value.str().map(Some) +} + +/// Base64 storage as text, raising Python's errors for non-ASCII or invalid storage. +fn decoded( + storage: &Bound<'_, PyString>, + node_name: &Bound<'_, PyString>, + qualified_name: &str, +) -> PyResult { + let py: Python<'_> = storage.py(); + let text: String = storage.extract()?; + if !text.is_ascii() { + let _ = storage.call_method1("encode", ("ascii",))?; + return Err(PyValueError::new_err( + "non-ASCII fingerprint storage encoded as ASCII", + )); + } + let failure: StorageDecodeError = match decode_storage(&text) { + Ok(decoded) => return Ok(decoded), + Err(failure) => failure, + }; + let cause: PyErr = decode_error(py, failure)?; + let name: String = node_name.extract()?; + let raised: PyErr = fingerprint_input_error( + py, + &format!( + "Invalid fingerprint definition storage for '{name}' in {qualified_name}: expected \ + base64-encoded UTF-8. This can happen after upgrading from an older sqlbuild \ + version; delete or rebuild {qualified_name} to regenerate fingerprints." + ), + ); + raised.value(py).setattr("__context__", cause.value(py))?; + raised.set_cause(py, Some(cause)); + Err(raised) +} + +fn decode_error(py: Python<'_>, failure: StorageDecodeError) -> PyResult { + match failure { + StorageDecodeError::Base64(message) => Ok(PyErr::from_value( + py.import(BINASCII_MODULE)? + .getattr("Error")? + .call1((message,))?, + )), + StorageDecodeError::Utf8 { + bytes, + start, + end, + reason, + } => { + let reason: CString = + CString::new(reason).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(PyErr::from_value( + PyUnicodeDecodeError::new(py, c"utf-8", &bytes, start..end, &reason)?.into_any(), + )) + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fingerprint_set_from_rows, module)?)?; + module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs new file mode 100644 index 0000000000..96da41548e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod latest_rows; +pub(crate) mod registration; +pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs new file mode 100644 index 0000000000..318ebc2d78 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs @@ -0,0 +1,10 @@ +//! Register the native fingerprint storage functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::fingerprints::{latest_rows, storage_sql}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + storage_sql::register(module)?; + latest_rows::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs new file mode 100644 index 0000000000..34ca1125c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs @@ -0,0 +1,116 @@ +//! Fingerprint table SQL: DDL, the latest-row read, its node filter, and batched inserts. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyTuple; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::errors::FingerprintStorageError; +use sqlbuild_analysis::fingerprint_storage::main::create_table_sql::create_table_sql; +use sqlbuild_analysis::fingerprint_storage::main::insert_batch_lengths::insert_batch_lengths; +use sqlbuild_analysis::fingerprint_storage::main::insert_sql::insert_sql; +use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; +use sqlbuild_analysis::fingerprint_storage::main::relevant_read_sql::relevant_read_sql; +use sqlbuild_analysis::fingerprint_storage::models::{RelevantRead, StoredFingerprint}; + +use crate::bindings::_helpers::fingerprints::latest_rows::fingerprint_input_error; + +/// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, given `(string_type, timestamp_type)`. +#[pyfunction] +fn fingerprint_create_table_sql(qualified_name: &str, column_types: (String, String)) -> String { + create_table_sql(qualified_name, &column_types.0, &column_types.1) +} + +/// The windowed latest-row read of the fingerprint table. +#[pyfunction] +fn fingerprint_read_latest_sql(qualified_name: &str) -> String { + read_latest_sql(qualified_name) +} + +/// The latest-row read narrowed to the relevant nodes, or `None` when nothing can match. +#[pyfunction] +#[pyo3(signature = (read_sql, node_names, filtered_node_types))] +fn fingerprint_relevant_read_sql( + read_sql: &str, + node_names: Option>, + filtered_node_types: Vec, +) -> Option { + match relevant_read_sql(read_sql, node_names.as_deref(), &filtered_node_types) { + RelevantRead::Sql(sql) => Some(sql), + RelevantRead::Empty => None, + } +} + +/// One `INSERT` of a single fingerprint. +#[pyfunction] +fn fingerprint_insert_sql( + py: Python<'_>, + qualified_name: &str, + fingerprint: &Bound<'_, PyAny>, +) -> PyResult { + insert_sql(qualified_name, &[stored_fingerprint(fingerprint)?]) + .map_err(|error| storage_error(py, error)) +} + +/// `(batch, insert_sql)` per bounded batch, given `(batch_size, batch_max_bytes)`. +#[pyfunction] +fn fingerprint_insert_batches<'py>( + py: Python<'py>, + qualified_name: &str, + fingerprints: Vec>, + limits: (i64, i64), +) -> PyResult, String)>> { + let stored: Vec = fingerprints + .iter() + .map(stored_fingerprint) + .collect::>()?; + let lengths: Vec = insert_batch_lengths(qualified_name, &stored, limits.0, limits.1) + .map_err(|error| storage_error(py, error))?; + let mut batches: Vec<(Bound<'py, PyTuple>, String)> = Vec::with_capacity(lengths.len()); + let mut start: usize = 0; + for length in lengths { + let end: usize = start + length; + let sql: String = insert_sql(qualified_name, &stored[start..end]) + .map_err(|error| storage_error(py, error))?; + batches.push((PyTuple::new(py, &fingerprints[start..end])?, sql)); + start = end; + } + Ok(batches) +} + +fn stored_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult { + Ok(StoredFingerprint { + node_type: fingerprint.getattr("node_type")?.extract()?, + node_name: fingerprint.getattr("node_name")?.extract()?, + target_database: fingerprint.getattr("target_database")?.extract()?, + target_schema: fingerprint.getattr("target_schema")?.extract()?, + target_name: fingerprint.getattr("target_name")?.extract()?, + run_id: formatted(fingerprint, "run_id")?, + definition_hash: formatted(fingerprint, "definition_hash")?, + version_hash: formatted(fingerprint, "version_hash")?, + schema_fingerprint: formatted(fingerprint, "schema_fingerprint")?, + definition: fingerprint.getattr("definition")?.extract()?, + metadata_json: fingerprint.getattr("metadata_json")?.extract()?, + ts: fingerprint + .getattr("ts")? + .call_method0("isoformat")? + .extract()?, + }) +} + +fn formatted(fingerprint: &Bound<'_, PyAny>, name: &str) -> PyResult { + fingerprint.getattr(name)?.str()?.extract() +} + +fn storage_error(py: Python<'_>, error: FingerprintStorageError) -> PyErr { + fingerprint_input_error(py, &error.to_string()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fingerprint_create_table_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_read_latest_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_relevant_read_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_insert_sql, module)?)?; + module.add_function(wrap_pyfunction!(fingerprint_insert_batches, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index ec29b33bb4..8e36750bba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod compiled_project; pub(crate) mod contracts; pub(crate) mod dag_artifact; pub(crate) mod discovery; +pub(crate) mod fingerprints; pub(crate) mod graph; pub(crate) mod lineage; pub(crate) mod manifest; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 2f3f2d11da..86bb63b8e9 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,9 +5,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, dag_artifact, discovery, graph, lineage, manifest, model_config, - project_assembly, refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, - type_system, + compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, lineage, manifest, + model_config, project_assembly, refactor, render, rules, scopes, semantic_checks, + sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -52,6 +52,7 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { semantic_checks::registration::register(module)?; contracts::registration::register(module)?; lineage::registration::register(module)?; + fingerprints::registration::register(module)?; sql_test_glue::registration::register(module)?; project_assembly::registration::register(module)?; compile_outputs::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ecbfb6acfc..5704e7e9cb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -79,6 +79,7 @@ from sqlbuild.compiler.discovery.models import ( ModelSchemaDeclaration, TolerantScopeDiscovery, ) +from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, @@ -1303,6 +1304,30 @@ def resource_sql_diagnostic_rows(catalog: object, request: object, /) -> list[ob def compiler_query_hash(sql: str, dialect: str | None, /) -> str: ... def function_definition_hash(sql: str, language: str, dialect: str | None, /) -> str: ... def normalize_query_text(sql: str, /) -> str: ... +def fingerprint_create_table_sql(qualified_name: str, column_types: tuple[str, str], /) -> str: ... +def fingerprint_read_latest_sql(qualified_name: str, /) -> str: ... +def fingerprint_relevant_read_sql( + read_sql: str, + node_names: Sequence[str] | None, + filtered_node_types: Sequence[str], + /, +) -> str | None: ... +def fingerprint_insert_sql(qualified_name: str, fingerprint: Fingerprint, /) -> str: ... +def fingerprint_insert_batches( + qualified_name: str, + fingerprints: Sequence[Fingerprint], + limits: tuple[int, int], + /, +) -> list[tuple[tuple[Fingerprint, ...], str]]: ... +def fingerprint_set_from_rows( + schema: str, qualified_name: str, rows: Sequence[Sequence[object]], / +) -> FingerprintSet: ... +def select_latest_fingerprints( + fingerprint_set: FingerprintSet, + node_names: Sequence[str], + filtered_node_types: Sequence[str], + /, +) -> FingerprintSet: ... def glob_name_match(name: str, pattern: str, ignore_case: bool, /) -> bool: ... def lone_surrogate_site(raw_values: dict[str, object], /) -> tuple[str, str, int] | None: ... def environment_template_names(values: list[str | None], /) -> list[str]: ... diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/latest.py b/src/sqlbuild/compiler/fingerprints/_helpers/latest.py deleted file mode 100644 index 7a360e9f9e..0000000000 --- a/src/sqlbuild/compiler/fingerprints/_helpers/latest.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Latest fingerprint set assembly shared by warehouse reads and in-memory selection.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet - - -def build_latest_fingerprint_set(*, schema: str, latest: Iterable[Fingerprint]) -> FingerprintSet: - """Index latest rows by identity and by node name, preferring models on name collisions.""" - - fingerprints: dict[str, Fingerprint] = {} - fingerprints_by_identity: dict[tuple[str, str], Fingerprint] = {} - fingerprint: Fingerprint - for fingerprint in latest: - fingerprints_by_identity[(fingerprint.node_type, fingerprint.node_name)] = fingerprint - if fingerprint.node_type == NODE_TYPE_MODEL or fingerprint.node_name not in fingerprints: - fingerprints[fingerprint.node_name] = fingerprint - return FingerprintSet( - schema=schema, - fingerprints=fingerprints, - fingerprints_by_identity=fingerprints_by_identity, - ) diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py index 2869b5325a..c87981494e 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py @@ -2,25 +2,11 @@ from __future__ import annotations -import base64 -from collections.abc import Callable, Sequence +from collections.abc import Callable +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.fingerprints.constants import ( - COLUMN_DEFINITION_B64, - COLUMN_DEFINITION_HASH, - COLUMN_METADATA_JSON_B64, - COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, - COLUMN_RUN_ID, - COLUMN_SCHEMA_FINGERPRINT, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_TIMESTAMP, - COLUMN_VERSION_HASH, - FINGERPRINT_TABLE_NAME, -) +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -57,23 +43,12 @@ def build_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, ) - string_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - return ( - f"CREATE TABLE IF NOT EXISTS {qualified_name} (" - f"{COLUMN_NODE_TYPE} {string_type} NOT NULL, " - f"{COLUMN_NODE_NAME} {string_type} NOT NULL, " - f"{COLUMN_TARGET_DATABASE} {string_type}, " - f"{COLUMN_TARGET_SCHEMA} {string_type}, " - f"{COLUMN_TARGET_NAME} {string_type}, " - f"{COLUMN_RUN_ID} {string_type} NOT NULL, " - f"{COLUMN_DEFINITION_HASH} {string_type} NOT NULL, " - f"{COLUMN_VERSION_HASH} {string_type} NOT NULL, " - f"{COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, " - f"{COLUMN_DEFINITION_B64} {string_type} NOT NULL, " - f"{COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, " - f"{COLUMN_TIMESTAMP} {timestamp_type} NOT NULL" - f")" + return _native.fingerprint_create_table_sql( + qualified_name, + ( + render_framework_type(FrameworkType.STRING), + render_framework_type(FrameworkType.TIMESTAMP), + ), ) @@ -82,23 +57,12 @@ def build_read_latest_sql( ) -> str: """Build a windowed SELECT for the latest fingerprint per node identity.""" - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - selected_columns: str = _fingerprint_select_columns() - return ( - f"SELECT {selected_columns} " - f"FROM (" - f"SELECT {selected_columns}, " - f"ROW_NUMBER() OVER (" - f"PARTITION BY {COLUMN_NODE_TYPE}, {COLUMN_NODE_NAME} " - f"ORDER BY {COLUMN_TIMESTAMP} DESC, {COLUMN_RUN_ID} DESC" - f") AS __sqlbuild_latest_rank " - f"FROM {qualified_name}" - f") AS __sqlbuild_latest_fingerprints " - f"WHERE __sqlbuild_latest_rank = 1" + return _native.fingerprint_read_latest_sql( + build_qualified_table_name( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + ) ) @@ -111,109 +75,11 @@ def build_insert_sql( ) -> str: """Build a complete INSERT statement for appending one fingerprint row.""" - return build_insert_many_sql( - database=database, - schema=schema, - fingerprints=(fingerprint,), - render_qualified_name=render_qualified_name, + return _native.fingerprint_insert_sql( + build_qualified_table_name( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + ), + fingerprint, ) - - -def build_insert_many_sql( - *, - database: str | None, - schema: str, - fingerprints: Sequence[Fingerprint], - render_qualified_name: Callable[..., str | None], -) -> str: - """Build one append-only INSERT for a non-empty fingerprint batch.""" - - if not fingerprints: - raise FingerprintInputError("fingerprint insert requires at least one row") - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - rows: str = ", ".join(_fingerprint_values(fingerprint) for fingerprint in fingerprints) - return ( - f"INSERT INTO {qualified_name} (" - f"{COLUMN_NODE_TYPE}, " - f"{COLUMN_NODE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_DEFINITION_HASH}, " - f"{COLUMN_VERSION_HASH}, " - f"{COLUMN_SCHEMA_FINGERPRINT}, " - f"{COLUMN_DEFINITION_B64}, " - f"{COLUMN_METADATA_JSON_B64}, " - f"{COLUMN_TIMESTAMP}" - f") VALUES {rows}" - ) - - -def _fingerprint_values(fingerprint: Fingerprint) -> str: - run_id: str = fingerprint.run_id - definition_hash: str = fingerprint.definition_hash - version_hash: str = fingerprint.version_hash - schema_fingerprint: str = fingerprint.schema_fingerprint - ts: str = fingerprint.ts.isoformat() - encoded_definition: str = _encode_definition_storage(fingerprint.definition).replace("'", "''") - encoded_metadata_json: str = _encode_definition_storage(fingerprint.metadata_json).replace( - "'", "''" - ) - node_type_literal: str = _required_string_literal(fingerprint.node_type) - node_name_literal: str = _required_string_literal(fingerprint.node_name) - target_database_literal: str = _optional_string_literal(fingerprint.target_database) - target_schema_literal: str = _optional_string_literal(fingerprint.target_schema) - target_name_literal: str = _optional_string_literal(fingerprint.target_name) - return ( - "(" - f"{node_type_literal}, " - f"{node_name_literal}, " - f"{target_database_literal}, " - f"{target_schema_literal}, " - f"{target_name_literal}, " - f"'{run_id}', " - f"'{definition_hash}', " - f"'{version_hash}', " - f"'{schema_fingerprint}', " - f"'{encoded_definition}', " - f"'{encoded_metadata_json}', " - f"'{ts}'" - f")" - ) - - -def _fingerprint_select_columns() -> str: - return ( - f"{COLUMN_NODE_TYPE}, " - f"{COLUMN_NODE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_DEFINITION_HASH}, " - f"{COLUMN_VERSION_HASH}, " - f"{COLUMN_SCHEMA_FINGERPRINT}, " - f"{COLUMN_DEFINITION_B64}, " - f"{COLUMN_METADATA_JSON_B64}, " - f"{COLUMN_TIMESTAMP}" - ) - - -def _optional_string_literal(value: str | None) -> str: - if value is None: - return "NULL" - return _required_string_literal(value) - - -def _required_string_literal(value: str) -> str: - escaped_value: str = value.replace("'", "''") - return f"'{escaped_value}'" - - -def _encode_definition_storage(value: str) -> str: - return base64.b64encode(value.encode("utf-8")).decode("ascii") diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py b/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py index 2fdd597deb..1db5155485 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/write_many.py @@ -7,11 +7,11 @@ from collections.abc import Callable, Sequence from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType from sqlbuild.compiler.fingerprints._helpers.sql import ( build_create_table_sql, - build_insert_many_sql, - build_insert_sql, + build_qualified_table_name, ) from sqlbuild.compiler.fingerprints.constants import ( FINGERPRINT_WRITE_ATTEMPTS, @@ -19,7 +19,6 @@ FINGERPRINT_WRITE_BATCH_SIZE, FINGERPRINT_WRITE_RETRY_BASE_SECONDS, ) -from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.fingerprints.types import FingerprintWriteProgress @@ -60,23 +59,19 @@ def write_fingerprints_impl( sqls=(create_sql, *index_sqls), fingerprints=fingerprints, ) - batches: tuple[tuple[Fingerprint, ...], ...] = _fingerprint_batches( - database=database, - schema=schema, - fingerprints=fingerprints, - render_qualified_name=render_qualified_name, - batch_size=FINGERPRINT_WRITE_BATCH_SIZE, - batch_max_bytes=FINGERPRINT_WRITE_BATCH_MAX_BYTES, - ) - completed_count: int = 0 - batch: tuple[Fingerprint, ...] - for batch in batches: - insert_sql: str = build_insert_many_sql( + batches: list[tuple[tuple[Fingerprint, ...], str]] = _native.fingerprint_insert_batches( + build_qualified_table_name( database=database, schema=schema, - fingerprints=batch, render_qualified_name=render_qualified_name, - ) + ), + list(fingerprints), + (FINGERPRINT_WRITE_BATCH_SIZE, FINGERPRINT_WRITE_BATCH_MAX_BYTES), + ) + completed_count: int = 0 + batch: tuple[Fingerprint, ...] + insert_sql: str + for batch, insert_sql in batches: _ = _execute_with_retry( connection=connection, execute=execute, @@ -88,45 +83,6 @@ def write_fingerprints_impl( on_progress(completed=completed_count, total=len(fingerprints)) -def _fingerprint_batches( - *, - database: str | None, - schema: str, - fingerprints: Sequence[Fingerprint], - render_qualified_name: Callable[..., str | None], - batch_size: int, - batch_max_bytes: int, -) -> tuple[tuple[Fingerprint, ...], ...]: - if batch_size < 1: - raise FingerprintInputError("fingerprint batch size must be positive") - if batch_max_bytes < 1: - raise FingerprintInputError("fingerprint batch byte limit must be positive") - batches: list[tuple[Fingerprint, ...]] = [] - current_batch: list[Fingerprint] = [] - current_bytes: int = 0 - fingerprint: Fingerprint - for fingerprint in fingerprints: - single_row_bytes: int = len( - build_insert_sql( - database=database, - schema=schema, - fingerprint=fingerprint, - render_qualified_name=render_qualified_name, - ).encode("utf-8") - ) - if current_batch and ( - len(current_batch) >= batch_size or current_bytes + single_row_bytes > batch_max_bytes - ): - batches.append(tuple(current_batch)) - current_batch = [] - current_bytes = 0 - current_batch.append(fingerprint) - current_bytes += single_row_bytes - if current_batch: - batches.append(tuple(current_batch)) - return tuple(batches) - - def _execute_with_retry( *, connection: Any, diff --git a/src/sqlbuild/compiler/fingerprints/main/_select_latest.py b/src/sqlbuild/compiler/fingerprints/main/_select_latest.py index 656c02090b..9e78817131 100644 --- a/src/sqlbuild/compiler/fingerprints/main/_select_latest.py +++ b/src/sqlbuild/compiler/fingerprints/main/_select_latest.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.fingerprints._helpers.latest import build_latest_fingerprint_set +import sqlbuild._native as _native from sqlbuild.compiler.fingerprints.models import FingerprintSet @@ -14,15 +14,4 @@ def select_latest_fingerprints( ) -> FingerprintSet: """Apply the ``read_latest_fingerprints`` node filter to an unfiltered latest read.""" - if not node_names and not filtered_node_types: - return FingerprintSet(schema=fingerprint_set.schema, fingerprints={}) - names: frozenset[str] = frozenset(node_names) - types: frozenset[str] = frozenset(filtered_node_types) - return build_latest_fingerprint_set( - schema=fingerprint_set.schema, - latest=( - fingerprint - for fingerprint in (fingerprint_set.fingerprints_by_identity or {}).values() - if fingerprint.node_name in names or (types and fingerprint.node_type not in types) - ), - ) + return _native.select_latest_fingerprints(fingerprint_set, node_names, filtered_node_types) diff --git a/src/sqlbuild/compiler/fingerprints/main/read.py b/src/sqlbuild/compiler/fingerprints/main/read.py index df1a170199..86cce8f3e9 100644 --- a/src/sqlbuild/compiler/fingerprints/main/read.py +++ b/src/sqlbuild/compiler/fingerprints/main/read.py @@ -2,19 +2,16 @@ from __future__ import annotations -import base64 -import binascii from collections.abc import Callable -from datetime import datetime from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.fingerprints._helpers.latest import build_latest_fingerprint_set from sqlbuild.compiler.fingerprints._helpers.sql import ( build_qualified_table_name, ) from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.fingerprints.models import FingerprintSet def read_latest_fingerprints( @@ -46,28 +43,13 @@ def read_latest_fingerprints( database=database, schema=schema, ) - if node_names is not None: - if not node_names and not filtered_node_types: - return FingerprintSet(schema=schema, fingerprints={}) - relevant_predicate: str = "" - if node_names: - literals: str = ", ".join( - "'" + node_name.replace("'", "''") + "'" for node_name in node_names - ) - relevant_predicate = f"node_name IN ({literals})" - if filtered_node_types: - type_literals: str = ", ".join( - "'" + node_type.replace("'", "''") + "'" for node_type in filtered_node_types - ) - python_identity_predicate: str = f"node_type NOT IN ({type_literals})" - relevant_predicate = ( - f"{relevant_predicate} OR {python_identity_predicate}" - if relevant_predicate - else python_identity_predicate - ) - read_sql = f"SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE {relevant_predicate}" + relevant_sql: str | None = _native.fingerprint_relevant_read_sql( + read_sql, node_names, filtered_node_types + ) + if relevant_sql is None: + return FingerprintSet(schema=schema, fingerprints={}) try: - result: Any = execute(connection=connection, sql=read_sql) + result: Any = execute(connection=connection, sql=relevant_sql) except Exception as error: raise FingerprintInputError( f"Unable to read fingerprints from {qualified_name}. This can happen after " @@ -75,46 +57,4 @@ def read_latest_fingerprints( "fingerprint table to regenerate fingerprints." ) from error rows: list[tuple[Any, ...]] = result.fetchall() - return build_latest_fingerprint_set( - schema=schema, - latest=(_row_to_fingerprint(row=row, qualified_name=qualified_name) for row in rows), - ) - - -def _row_to_fingerprint(*, row: tuple[Any, ...], qualified_name: str) -> Fingerprint: - raw_ts: Any = row[11] - ts: datetime = raw_ts if isinstance(raw_ts, datetime) else datetime.fromisoformat(str(raw_ts)) - node_type: str = str(row[0]) - node_name: str = str(row[1]) - raw_target_database: Any = row[2] - raw_target_schema: Any = row[3] - raw_target_name: Any = row[4] - definition_storage: str = str(row[9]) - metadata_json_storage: str = str(row[10]) - try: - definition: str = base64.b64decode( - definition_storage.encode("ascii"), validate=True - ).decode("utf-8") - metadata_json: str = base64.b64decode( - metadata_json_storage.encode("ascii"), validate=True - ).decode("utf-8") - except (binascii.Error, UnicodeDecodeError) as error: - raise FingerprintInputError( - f"Invalid fingerprint definition storage for '{node_name}' in {qualified_name}: " - "expected base64-encoded UTF-8. This can happen after upgrading from an older " - f"sqlbuild version; delete or rebuild {qualified_name} to regenerate fingerprints." - ) from error - return Fingerprint( - node_type=node_type, - node_name=node_name, - target_database=str(raw_target_database) if raw_target_database is not None else None, - target_schema=str(raw_target_schema) if raw_target_schema is not None else None, - target_name=str(raw_target_name) if raw_target_name is not None else None, - run_id=str(row[5]), - definition_hash=str(row[6]), - version_hash=str(row[7]), - schema_fingerprint=str(row[8]), - definition=definition, - metadata_json=metadata_json, - ts=ts, - ) + return _native.fingerprint_set_from_rows(schema, qualified_name, rows) From 8ff1aaf4e43bf15ce9700018c8dc7fde0882e7db Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:35:38 +0100 Subject: [PATCH 20/26] refactor: build project column lineage and its graph natively --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/lineage_graph/constants.rs | 17 + .../lineage_graph/main/analysis_dialect.rs | 14 + .../src/lineage_graph/main/edge_index.rs | 30 + .../src/lineage_graph/main/mod.rs | 2 + .../src/lineage_graph/mod.rs | 8 + .../src/lineage_graph/models.rs | 32 + .../src/lineage_graph/tests.rs | 6 + .../src/lineage_graph/tests/helpers.rs | 14 + .../src/lineage_graph/tests/test_graph.rs | 81 +++ .../src/lineage_graph/tests/test_types.rs | 18 + .../_helpers/lineage/column_lineage_graph.rs | 559 ++++++++++++++++++ .../bindings/_helpers/lineage/fast_lineage.rs | 119 ---- .../_helpers/lineage/lineage_objects.rs | 185 ++++++ .../src/bindings/_helpers/lineage/mod.rs | 6 +- .../_helpers/lineage/project_lineage.rs | 372 ++++++++++++ .../_helpers/lineage/project_schema.rs | 112 ++++ .../bindings/_helpers/lineage/registration.rs | 12 +- .../bindings/_helpers/lineage/rich_lineage.rs | 160 ----- src/sqlbuild/_native.pyi | 63 +- .../cli/commands/_helpers/compile/lineage.py | 16 +- .../cli/commands/_helpers/compile/output.py | 2 +- .../commands/_helpers/lineage/execution.py | 11 +- .../commands/_helpers/lineage/selection.py | 10 +- src/sqlbuild/cli/compile/models.py | 2 +- .../lineage/_helpers/native_fast_columns.py | 186 ------ .../lineage/_helpers/native_rich_columns.py | 133 ----- .../compiler/lineage/_helpers/rich_columns.py | 65 -- src/sqlbuild/compiler/lineage/constants.py | 8 - .../main/_build_native_column_lineage.py | 19 - .../lineage/main/build_semantic_uses.py | 41 -- src/sqlbuild/compiler/lineage/main/columns.py | 33 -- src/sqlbuild/compiler/lineage/models.py | 411 +------------ src/sqlbuild/compiler/lineage/types.py | 2 - .../fast_lineage/test_native_fast_lineage.py | 4 +- .../cli/commands/main/compile/helpers.py | 4 +- .../compile/test_native_analysis_seams.py | 6 +- .../sqlbuild/compiler/lineage/_test_types.py | 3 +- .../src/sqlbuild/compiler/lineage/helpers.py | 55 +- .../lineage/test_native_fast_lineage.py | 50 +- .../_helpers/lineage/test_selection.py | 18 +- .../main/compile/test_compile_output.py | 2 +- .../sqlbuild/compiler/lineage/_test_types.py | 3 +- .../src/sqlbuild/compiler/lineage/helpers.py | 15 + .../sqlbuild/compiler/lineage/test_columns.py | 52 +- 45 files changed, 1631 insertions(+), 1331 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/constants.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/mod.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/models.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/tests.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs delete mode 100644 src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py delete mode 100644 src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py delete mode 100644 src/sqlbuild/compiler/lineage/_helpers/rich_columns.py delete mode 100644 src/sqlbuild/compiler/lineage/constants.py delete mode 100644 src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py delete mode 100644 src/sqlbuild/compiler/lineage/main/build_semantic_uses.py delete mode 100644 src/sqlbuild/compiler/lineage/main/columns.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 46361823e6..36186f5894 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -14,6 +14,7 @@ pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; pub mod lineage; +pub mod lineage_graph; pub mod manifest; pub mod python_values; pub mod query_analysis; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/constants.rs b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs new file mode 100644 index 0000000000..9fd8a1cb57 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs @@ -0,0 +1,17 @@ +//! Rich lineage dialect aliases. + +/// Aliases Python maps before trying the name as a Polyglot dialect value. +pub const ANALYSIS_DIALECT_ALIASES: [(&str, &str); 12] = [ + ("", "generic"), + ("ansi", "generic"), + ("generic", "generic"), + ("bigquery", "bigquery"), + ("snowflake", "snowflake"), + ("duckdb", "duckdb"), + ("motherduck", "duckdb"), + ("databricks", "databricks"), + ("postgres", "postgresql"), + ("tsql", "tsql"), + ("sqlserver", "tsql"), + ("postgresql", "postgresql"), +]; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs new file mode 100644 index 0000000000..6cee9ed107 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/analysis_dialect.rs @@ -0,0 +1,14 @@ +//! The Polyglot analysis dialect for a stripped, lowercased adapter dialect name. + +use crate::lineage_graph::constants::ANALYSIS_DIALECT_ALIASES; +use crate::lineage_graph::models::AnalysisDialect; + +/// The aliased dialect value, or the name to look up as a dialect value. +pub fn analysis_dialect(normalized: &str) -> AnalysisDialect<'_> { + ANALYSIS_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == normalized) + .map_or(AnalysisDialect::Named(normalized), |(_, dialect)| { + AnalysisDialect::Alias(dialect) + }) +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs new file mode 100644 index 0000000000..8aff42a153 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/edge_index.rs @@ -0,0 +1,30 @@ +//! Edge indexes for the project column lineage graph. + +use std::collections::HashMap; + +use crate::lineage_graph::models::{EdgeIndex, EdgeKey}; + +/// Index `edges` by target model and source resource; equal edges share their first position. +pub fn edge_index(edges: &[EdgeKey]) -> EdgeIndex { + let mut index = EdgeIndex { + first_equal: Vec::with_capacity(edges.len()), + ..EdgeIndex::default() + }; + let mut first_positions: HashMap<&EdgeKey, usize> = HashMap::with_capacity(edges.len()); + for (position, edge) in edges.iter().enumerate() { + index + .by_target_model + .entry(edge.target_model.clone()) + .or_default() + .push(position); + index + .by_source_resource + .entry(edge.source_name.clone()) + .or_default() + .push(position); + index + .first_equal + .push(*first_positions.entry(edge).or_insert(position)); + } + index +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs new file mode 100644 index 0000000000..c6f3c18b65 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs @@ -0,0 +1,2 @@ +pub mod analysis_dialect; +pub mod edge_index; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs new file mode 100644 index 0000000000..f659fb0427 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs @@ -0,0 +1,8 @@ +//! The project column lineage graph: edge indexes and the rich analysis dialect. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/models.rs b/crates/sqlbuild-analysis/src/lineage_graph/models.rs new file mode 100644 index 0000000000..fbf845caf1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/models.rs @@ -0,0 +1,32 @@ +//! Column lineage graph edges and their indexes. + +use std::collections::HashMap; + +/// One collapsed edge's identity, every part as Python's `str()` renders it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EdgeKey { + pub source_type: String, + pub source_name: String, + pub source_column: String, + pub target_model: String, + pub target_column: String, + pub transform: String, + pub confidence: String, +} + +/// Edge positions by target model and source resource, and each edge's first equal edge. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EdgeIndex { + pub by_target_model: HashMap>, + pub by_source_resource: HashMap>, + pub first_equal: Vec, +} + +/// How an adapter dialect name maps onto a Polyglot analysis dialect. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnalysisDialect<'a> { + /// A known alias of this Polyglot dialect value. + Alias(&'static str), + /// Not an alias: the name itself must be a Polyglot dialect value. + Named(&'a str), +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests.rs new file mode 100644 index 0000000000..a9d1498b0f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_graph.rs"] +mod graph; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs new file mode 100644 index 0000000000..56d8bccd36 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs @@ -0,0 +1,14 @@ +use crate::lineage_graph::models::EdgeKey; + +/// A direct, high-confidence edge from `source.column` into `target.column`. +pub(crate) fn edge_key(source: (&str, &str), target: (&str, &str)) -> EdgeKey { + EdgeKey { + source_type: "model".to_owned(), + source_name: source.0.to_owned(), + source_column: source.1.to_owned(), + target_model: target.0.to_owned(), + target_column: target.1.to_owned(), + transform: "direct".to_owned(), + confidence: "high".to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs new file mode 100644 index 0000000000..32e30eec6f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs @@ -0,0 +1,81 @@ +use crate::lineage_graph::main::analysis_dialect::analysis_dialect; +use crate::lineage_graph::main::edge_index::edge_index; +use crate::lineage_graph::models::{AnalysisDialect, EdgeIndex, EdgeKey}; +use crate::lineage_graph::tests::helpers::edge_key; +use crate::lineage_graph::tests::test_types::{AnalysisDialectTestCase, EdgeIndexTestCase}; + +#[test] +fn given_edges_when_indexing_then_positions_keep_edge_order_and_equal_edges_share_one() { + let test_cases = [EdgeIndexTestCase { + description: "repeated and interleaved edges", + edges: &[ + (("orders", "id"), ("customers", "order_id")), + (("products", "id"), ("orders", "product_id")), + (("orders", "id"), ("customers", "order_id")), + (("orders", "total"), ("customers", "spend")), + ], + expected_targeting_orders: &[0, 2, 3], + expected_sourced_from_orders: &[0, 2, 3], + expected_first_equal: &[0, 1, 0, 3], + }]; + for test_case in test_cases { + let edges: Vec = test_case + .edges + .iter() + .map(|(source, target)| edge_key(*source, *target)) + .collect(); + let index: EdgeIndex = edge_index(&edges); + assert_eq!( + index.by_target_model.get("customers").map(Vec::as_slice), + Some(test_case.expected_targeting_orders), + "{}", + test_case.description + ); + assert_eq!( + index.by_source_resource.get("orders").map(Vec::as_slice), + Some(test_case.expected_sourced_from_orders), + "{}", + test_case.description + ); + assert_eq!( + index.first_equal.as_slice(), + test_case.expected_first_equal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dialect_names_when_resolving_then_aliases_map_and_others_stay_named() { + let test_cases = [ + AnalysisDialectTestCase { + description: "an empty name is generic", + normalized: "", + expected_dialect: AnalysisDialect::Alias("generic"), + }, + AnalysisDialectTestCase { + description: "MotherDuck analyzes as DuckDB", + normalized: "motherduck", + expected_dialect: AnalysisDialect::Alias("duckdb"), + }, + AnalysisDialectTestCase { + description: "SQL Server analyzes as T-SQL", + normalized: "sqlserver", + expected_dialect: AnalysisDialect::Alias("tsql"), + }, + AnalysisDialectTestCase { + description: "other names are looked up as given", + normalized: "mysql", + expected_dialect: AnalysisDialect::Named("mysql"), + }, + ]; + for test_case in test_cases { + assert_eq!( + analysis_dialect(test_case.normalized), + test_case.expected_dialect, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs new file mode 100644 index 0000000000..d664364af6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs @@ -0,0 +1,18 @@ +use crate::lineage_graph::models::AnalysisDialect; + +/// `(resource, column)`. +pub(crate) type ColumnRef = (&'static str, &'static str); + +pub(crate) struct EdgeIndexTestCase { + pub(crate) description: &'static str, + pub(crate) edges: &'static [(ColumnRef, ColumnRef)], + pub(crate) expected_targeting_orders: &'static [usize], + pub(crate) expected_sourced_from_orders: &'static [usize], + pub(crate) expected_first_equal: &'static [usize], +} + +pub(crate) struct AnalysisDialectTestCase { + pub(crate) description: &'static str, + pub(crate) normalized: &'static str, + pub(crate) expected_dialect: AnalysisDialect<'static>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs new file mode 100644 index 0000000000..53c7ff78cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -0,0 +1,559 @@ +//! `ProjectColumnLineage`: the project column lineage graph with lazily materialised views. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyType}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; +use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; + +use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; + +/// One compact model's facts and root-star flag. +struct CompactModel { + columns: Py, + has_star: bool, +} + +/// The Python objects one edge record materialises from. +enum EdgeParts { + /// An edge given to the eager constructor, returned as is. + Given(Py), + /// Fact values: source resource type, transform kind and confidence objects. + Facts { + resource_type: Py, + transform: Py, + confidence: Py, + }, +} + +/// Indexed edge records, materialised edges shared between equal records. +struct EdgeRecords { + keys: Vec, + parts: Vec, + index: EdgeIndex, + materialised: Vec>>, +} + +#[pyclass(module = "sqlbuild._native", name = "ProjectColumnLineage")] +pub(crate) struct ProjectColumnLineage { + models: Py, + compact: HashMap, + model_order: Vec, + edges: Option>, + records: Option, + edge_counts: HashMap, +} + +#[pymethods] +impl ProjectColumnLineage { + #[new] + #[pyo3(signature = (*, models, edges))] + fn new(models: &Bound<'_, PyAny>, edges: &Bound<'_, PyAny>) -> PyResult { + let mut keys: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); + for edge in edges.try_iter()? { + let edge: Bound<'_, PyAny> = edge?; + keys.push(EdgeKey { + source_name: edge + .getattr("source")? + .getattr("resource_name")? + .extract()?, + target_model: edge + .getattr("target")? + .getattr("resource_name")? + .extract()?, + ..empty_key() + }); + parts.push(EdgeParts::Given(edge.unbind())); + } + Ok(Self { + model_order: models + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + models: models.clone().unbind(), + compact: HashMap::new(), + edges: Some(edges.clone().unbind()), + records: Some(EdgeRecords::new(keys, parts)), + edge_counts: HashMap::new(), + }) + } + + /// Build indexed project lineage without expanding compact analysis facts. + #[classmethod] + #[pyo3(signature = (*, models, compact_models, model_order))] + fn from_fast_facts( + cls: &Bound<'_, PyType>, + models: &Bound<'_, PyAny>, + compact_models: &Bound<'_, PyDict>, + model_order: Vec, + ) -> PyResult { + let classes: LineageClasses<'_> = LineageClasses::new(cls.py())?; + let mut compact: HashMap = HashMap::new(); + for (name, entry) in compact_models.iter() { + let (columns, has_star): (Bound<'_, PyAny>, bool) = entry.extract()?; + let _ = compact.insert( + name.extract()?, + CompactModel { + columns: columns.unbind(), + has_star, + }, + ); + } + let mut edge_counts: HashMap = HashMap::new(); + for name in &model_order { + if let Some(model) = compact.get(name) { + let count: usize = fact_edge_count(&classes, model.columns.bind(cls.py()))?; + let _ = edge_counts.insert(name.clone(), count); + } + } + for item in models.call_method0("items")?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let _ = edge_counts.insert(name, upstream_count(&model.getattr("columns")?)?); + } + Ok(Self { + models: models.clone().unbind(), + compact, + model_order, + edges: None, + records: None, + edge_counts, + }) + } + + /// Return model lineage, expanding compact model facts on first access. + #[getter] + fn models(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !slf.compact.is_empty() { + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + let models: Bound<'_, PyAny> = slf.models.bind(py).clone(); + for name in &slf.model_order { + let Some(compact) = slf.compact.get(name) else { + continue; + }; + if models.contains(name)? { + continue; + } + let lineage: Bound<'_, PyAny> = + model_from_facts(&classes, name, compact.columns.bind(py), compact.has_star)?; + models.set_item(name, lineage)?; + } + slf.compact.clear(); + } + Ok(slf.models.clone_ref(py)) + } + + /// Return all graph edges, expanding compact edge records on first access. + #[getter] + fn edges(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if let Some(edges) = &slf.edges { + return Ok(edges.clone_ref(py)); + } + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + slf.ensure_records(&classes)?; + let count: usize = slf.records.as_ref().map_or(0, |records| records.keys.len()); + let edges: Vec> = slf.materialised(&classes, 0..count)?; + let edges: Py = PyTuple::new(py, edges)?.into_any().unbind(); + slf.edges = Some(edges.clone_ref(py)); + Ok(edges) + } + + /// Return whether model lineage is available without expanding its columns. + #[pyo3(name = "has_model")] + fn contains_model(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { + if self.models.bind(py).contains(model_name)? { + return Ok(true); + } + Ok(model_name + .extract::() + .is_ok_and(|name| self.compact.contains_key(&name))) + } + + /// Return whether model lineage retains unresolved root-star uncertainty. + fn model_has_star(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { + if let Ok(name) = model_name.extract::() + && let Some(compact) = self.compact.get(&name) + { + return Ok(compact.has_star); + } + let model: Bound<'_, PyAny> = self.models.bind(py).call_method1("get", (model_name,))?; + if model.is_none() { + return Ok(false); + } + model.getattr("has_star")?.extract() + } + + /// Return a target model's direct edge count without expanding edge objects. + fn edge_count_targeting(&self, model_name: &str) -> usize { + match &self.records { + Some(records) => records + .index + .by_target_model + .get(model_name) + .map_or(0, Vec::len), + None => self.edge_counts.get(model_name).copied().unwrap_or(0), + } + } + + /// Return edges whose target model is `model_name`. + fn edges_targeting<'py>( + mut slf: PyRefMut<'py, Self>, + model_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = slf.positions(|index| index.by_target_model.get(model_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return edges whose source resource is `resource_name`. + fn edges_sourced_from<'py>( + mut slf: PyRefMut<'py, Self>, + resource_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = + slf.positions(|index| index.by_source_resource.get(resource_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return direct downstream consumers of `resource_name.column_name`. + #[pyo3(signature = (*, resource_name, column_name))] + fn column_consumers<'py>( + slf: &Bound<'py, Self>, + resource_name: &str, + column_name: &Bound<'py, PyAny>, + ) -> PyResult> { + let sourced: Bound<'py, PyAny> = + slf.call_method1("edges_sourced_from", (resource_name,))?; + let mut consumers: Vec> = Vec::new(); + for edge in sourced.try_iter()? { + let edge: Bound<'py, PyAny> = edge?; + if edge + .getattr("source")? + .getattr("column_name")? + .eq(column_name)? + { + consumers.push(edge); + } + } + PyTuple::new(slf.py(), consumers) + } + + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let models: Bound<'_, PyString> = slf.getattr("models")?.repr()?; + let edges: Bound<'_, PyString> = slf.getattr("edges")?.repr()?; + Ok(format!( + "ProjectColumnLineage(models={models}, edges={edges})" + )) + } + + fn __eq__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !other.is_instance_of::() { + return Ok(py.NotImplemented()); + } + let equal: bool = slf.getattr("models")?.eq(other.getattr("models")?)? + && slf.getattr("edges")?.eq(other.getattr("edges")?)?; + Ok(pyo3::types::PyBool::new(py, equal) + .to_owned() + .into_any() + .unbind()) + } +} + +impl ProjectColumnLineage { + /// Build fast edge records from the compact facts, or the expanded models, in model order. + fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { + if self.records.is_some() { + return Ok(()); + } + let py: Python<'_> = classes.py(); + let mut builder = RecordBuilder::default(); + let models: Bound<'_, PyAny> = self.models.bind(py).clone(); + for name in &self.model_order { + if let Some(compact) = self.compact.get(name) { + builder.push_facts(classes, name, compact.columns.bind(py))?; + continue; + } + let model: Bound<'_, PyAny> = models.call_method1("get", (name,))?; + if model.is_none() { + continue; + } + builder.push_columns(name, &model.getattr("columns")?)?; + } + self.records = Some(EdgeRecords::new(builder.keys, builder.parts)); + Ok(()) + } + + fn positions(&self, select: impl Fn(&EdgeIndex) -> Option<&Vec>) -> Vec { + self.records + .as_ref() + .and_then(|records| select(&records.index)) + .cloned() + .unwrap_or_default() + } + + fn materialised( + &mut self, + classes: &LineageClasses<'_>, + positions: impl IntoIterator, + ) -> PyResult>> { + let Some(records) = self.records.as_mut() else { + return Ok(Vec::new()); + }; + positions + .into_iter() + .map(|position| records.edge(classes, position)) + .collect() + } +} + +impl EdgeRecords { + fn new(keys: Vec, parts: Vec) -> Self { + let index: EdgeIndex = edge_index(&keys); + let materialised: Vec>> = keys.iter().map(|_| None).collect(); + Self { + keys, + parts, + index, + materialised, + } + } + + /// The edge at `position`, built once per distinct fact edge. + fn edge(&mut self, classes: &LineageClasses<'_>, position: usize) -> PyResult> { + let py: Python<'_> = classes.py(); + let (resource_type, transform, confidence) = match &self.parts[position] { + EdgeParts::Given(edge) => return Ok(edge.clone_ref(py)), + EdgeParts::Facts { + resource_type, + transform, + confidence, + } => (resource_type, transform, confidence), + }; + let first: usize = self.index.first_equal[position]; + if let Some(edge) = &self.materialised[first] { + return Ok(edge.clone_ref(py)); + } + let key: &EdgeKey = &self.keys[position]; + let edge: Py = classes + .edge( + classes.qualified(resource_type.bind(py), &key.source_name, &key.source_column)?, + &classes.qualified( + &classes.model_resource, + &key.target_model, + &key.target_column, + )?, + transform.bind(py), + confidence.bind(py), + )? + .unbind(); + self.materialised[first] = Some(edge.clone_ref(py)); + Ok(edge) + } +} + +fn empty_key() -> EdgeKey { + EdgeKey { + source_type: String::new(), + source_name: String::new(), + source_column: String::new(), + target_model: String::new(), + target_column: String::new(), + transform: String::new(), + confidence: String::new(), + } +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +/// Edge records accumulated in model order. +#[derive(Default)] +struct RecordBuilder { + keys: Vec, + parts: Vec, +} + +impl RecordBuilder { + /// Records of compact facts, or of plain column facts, targeting `model_name`. + fn push_facts( + &mut self, + classes: &LineageClasses<'_>, + model_name: &str, + columns: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if !columns.is_instance(&classes.compact_facts)? { + return self.push_columns(model_name, columns); + } + let py: Python<'_> = columns.py(); + let pool: Bound<'_, PyAny> = columns.getattr("string_pool")?; + let remapped: bool = columns.getattr("resource_names")?.is_truthy()? + || columns.getattr("resource_name_indexes")?.is_truthy()?; + let mut members = CodeMembers::default(); + for row in columns.getattr("rows")?.try_iter()? { + let (name_index, transform_code, confidence_code, sources): ( + Bound<'_, PyAny>, + i64, + i64, + Bound<'_, PyAny>, + ) = row?.extract()?; + let output_column: String = pool.get_item(name_index)?.extract()?; + for source in sources.try_iter()? { + let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + source?.extract()?; + let resource_type: Bound<'_, PyAny> = pool.get_item(kind)?; + let resource_name: String = if remapped { + columns.call_method1("resource_name", (name,))?.extract()? + } else { + pool.get_item(name)?.extract()? + }; + let transform: Py = members.transform(columns, transform_code)?; + let confidence: Py = members.confidence(columns, confidence_code)?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: resource_name, + source_column: pool.get_item(column)?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(transform.bind(py))?, + confidence: text(confidence.bind(py))?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform, + confidence, + }); + } + } + Ok(()) + } + + /// Records of columns with `upstream_columns`, `transform_kind` and `confidence`. + fn push_columns(&mut self, model_name: &str, columns: &Bound<'_, PyAny>) -> PyResult<()> { + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let output_column: String = column.getattr("output_column")?.extract()?; + let transform: Bound<'_, PyAny> = column.getattr("transform_kind")?; + let confidence: Bound<'_, PyAny> = column.getattr("confidence")?; + for source in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let resource_type: Bound<'_, PyAny> = source.getattr("resource_type")?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: source.getattr("resource_name")?.extract()?, + source_column: source.getattr("column_name")?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(&transform)?, + confidence: text(&confidence)?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform: transform.clone().unbind(), + confidence: confidence.clone().unbind(), + }); + } + } + Ok(()) + } +} + +/// `facts.transform_kind(code)` and `facts.confidence(code)`, looked up once per code. +#[derive(Default)] +struct CodeMembers { + members: HashMap<(&'static str, i64), Py>, +} + +impl CodeMembers { + fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts, "transform_kind", code) + } + + fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts, "confidence", code) + } + + fn member( + &mut self, + facts: &Bound<'_, PyAny>, + method: &'static str, + code: i64, + ) -> PyResult> { + if let Some(member) = self.members.get(&(method, code)) { + return Ok(member.clone_ref(facts.py())); + } + let member: Py = facts.call_method1(method, (code,))?.unbind(); + let _ = self + .members + .insert((method, code), member.clone_ref(facts.py())); + Ok(member) + } +} + +/// The edge count of compact facts or plain column facts. +fn fact_edge_count(classes: &LineageClasses<'_>, columns: &Bound<'_, PyAny>) -> PyResult { + if !columns.is_instance(&classes.compact_facts)? { + return upstream_count(columns); + } + let mut count: usize = 0; + for row in columns.getattr("rows")?.try_iter()? { + count += row?.get_item(3)?.len()?; + } + Ok(count) +} + +fn upstream_count(columns: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = 0; + for column in columns.try_iter()? { + count += column?.getattr("upstream_columns")?.len()?; + } + Ok(count) +} + +/// `ModelColumnLineage` of compact or plain column facts. +fn model_from_facts<'py>( + classes: &LineageClasses<'py>, + model_name: &str, + columns: &Bound<'py, PyAny>, + has_star: bool, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut lineage_columns: Vec> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let mut sources: Vec> = Vec::new(); + for source in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + sources.push(classes.lineage_source.call1(( + source.getattr("resource_type")?, + source.getattr("resource_name")?, + source.getattr("column_name")?, + ))?); + } + lineage_columns.push(classes.column_lineage.call1(( + column.getattr("output_column")?, + column.getattr("transform_kind")?, + py.None(), + PyTuple::new(py, sources)?, + &classes.unknown_nullability, + column.getattr("confidence")?, + ))?); + } + classes.model(model_name, lineage_columns, has_star) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs deleted file mode 100644 index 469124bee8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/fast_lineage.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Fast column lineage. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::lineage::main::build_fast_lineage::build_fast_lineage; -use sqlbuild_analysis::lineage::models::{ - FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, LineageSource, -}; -use sqlbuild_analysis::semantic_validation::models as validation; - -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::models::ProjectCatalog; -use crate::bindings::types::CompilerDetach; - -type SourceRow = (&'static str, String, String); -type ColumnRow = (String, &'static str, &'static str, Vec); -/// `(dialect, [(resource_type, name, columns)], [(star_expansion, query_sql, column_names)])`. -type LineageRequestInput = ( - Option, - Vec<(String, String, Vec)>, - Vec<(bool, String, Vec)>, -); -/// `(status, columns, has_star, detail)`; see `FastLineageOutcome::into_parts`. -type OutcomeRow = (&'static str, Vec, bool, Option); - -/// Build fast lineage for the requested models on `catalog`'s analysis pool, if any. -#[pyfunction] -fn build_fast_column_lineage( - py: Python<'_>, - catalog: Option>, - request: LineageRequestInput, -) -> PyResult> { - let (dialect, schema, models) = request; - let schema: Vec = schema - .into_iter() - .map(schema_resource) - .collect::>()?; - let models: Vec = models.into_iter().map(lineage_model).collect(); - let request = FastLineageRequest { - dialect, - schema, - models, - }; - let catalog: Option<&validation::ProjectCatalog> = - catalog.as_ref().map(|catalog| &catalog.inner); - let outcomes: Vec = py - .compiler_detach(|| build_fast_lineage(&request, catalog)) - .map_err(compiler_error)?; - Ok(outcomes.into_iter().map(outcome_row).collect()) -} - -fn schema_resource( - (resource_type, name, columns): (String, String, Vec), -) -> PyResult { - let Some(kind) = LineageResourceType::from_value(&resource_type) else { - return Err(PyValueError::new_err(format!( - "unknown resource: {resource_type}" - ))); - }; - Ok(LineageSchemaResource { - resource_type: kind, - name, - columns, - }) -} - -fn lineage_model( - (star_expansion, query_sql, names): (bool, String, Vec), -) -> FastLineageModel { - if star_expansion { - FastLineageModel::StarExpansion { - query_sql, - existing_columns: names, - } - } else { - FastLineageModel::Parse { - query_sql, - inferred_columns: names, - } - } -} - -fn outcome_row(outcome: FastLineageOutcome) -> OutcomeRow { - let (status, columns, has_star, detail) = outcome.into_parts(); - (status, column_rows(columns), has_star, detail) -} - -fn column_rows(columns: Vec) -> Vec { - columns - .into_iter() - .map(|column| { - ( - column.output_column, - column.transform_kind.as_str(), - column.confidence.as_str(), - column - .upstream_columns - .into_iter() - .map(source_row) - .collect(), - ) - }) - .collect() -} - -fn source_row(source: LineageSource) -> SourceRow { - ( - source.resource_type.as_str(), - source.resource_name, - source.column_name, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_fast_column_lineage, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs new file mode 100644 index 0000000000..45a01e499d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -0,0 +1,185 @@ +//! The Python lineage models and enums native lineage builds. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; + +const LINEAGE_MODELS_MODULE: &str = "sqlbuild.compiler.lineage.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const LOGGING_MODULE: &str = "logging"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; + +/// The lineage model classes and enums, imported once per call. +pub(crate) struct LineageClasses<'py> { + pub(crate) qualified_column: Bound<'py, PyAny>, + pub(crate) lineage_source: Bound<'py, PyAny>, + pub(crate) lineage_edge: Bound<'py, PyAny>, + pub(crate) column_lineage: Bound<'py, PyAny>, + pub(crate) model_lineage: Bound<'py, PyAny>, + pub(crate) semantic_use: Bound<'py, PyAny>, + pub(crate) column_fact: Bound<'py, PyAny>, + pub(crate) source_fact: Bound<'py, PyAny>, + pub(crate) compact_facts: Bound<'py, PyAny>, + pub(crate) transform_kind: Bound<'py, PyAny>, + pub(crate) confidence: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) analysis_dialect: Bound<'py, PyAny>, + pub(crate) resource_type: Bound<'py, PyAny>, + pub(crate) model_resource: Bound<'py, PyAny>, + pub(crate) unknown_nullability: Bound<'py, PyAny>, +} + +impl<'py> LineageClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + let models: Bound<'py, PyModule> = py.import(LINEAGE_MODELS_MODULE)?; + let types: Bound<'py, PyModule> = py.import(LINEAGE_TYPES_MODULE)?; + let compile_models: Bound<'py, PyModule> = py.import(COMPILE_MODELS_MODULE)?; + let resource_type: Bound<'py, PyAny> = py + .import(COMPILE_TYPES_MODULE)? + .getattr("CompiledResourceType")?; + let nullability: Bound<'py, PyAny> = types.getattr("InferredNullability")?; + Ok(Self { + qualified_column: models.getattr("QualifiedLineageColumn")?, + lineage_source: models.getattr("ColumnLineageSource")?, + lineage_edge: models.getattr("ColumnLineageEdge")?, + column_lineage: models.getattr("ColumnLineage")?, + model_lineage: models.getattr("ModelColumnLineage")?, + semantic_use: models.getattr("DirectSemanticColumnUse")?, + column_fact: compile_models.getattr("CompiledLineageColumnFact")?, + source_fact: compile_models.getattr("CompiledLineageSourceFact")?, + compact_facts: compile_models.getattr("CompactLineageFacts")?, + transform_kind: types.getattr("ColumnTransformKind")?, + confidence: types.getattr("ColumnLineageConfidence")?, + analysis_dialect: types.getattr("PolyglotAnalysisDialect")?, + model_resource: resource_type.getattr("MODEL")?, + unknown_nullability: nullability.getattr("UNKNOWN")?, + nullability, + resource_type, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.qualified_column.py() + } + + /// `QualifiedLineageColumn(resource_type, resource_name, column_name)`. + pub(crate) fn qualified( + &self, + resource_type: &Bound<'py, PyAny>, + resource_name: &str, + column_name: &str, + ) -> PyResult> { + self.qualified_column + .call1((resource_type, resource_name, column_name)) + } + + /// `ColumnLineageEdge(source, target, transform_kind, confidence)`. + pub(crate) fn edge( + &self, + source: Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, + transform_kind: &Bound<'py, PyAny>, + confidence: &Bound<'py, PyAny>, + ) -> PyResult> { + self.lineage_edge + .call1((source, target, transform_kind, confidence)) + } + + /// `ModelColumnLineage(model_name, columns, has_star)`. + pub(crate) fn model( + &self, + model_name: &str, + columns: Vec>, + has_star: bool, + ) -> PyResult> { + self.model_lineage + .call1((model_name, PyTuple::new(self.py(), columns)?, has_star)) + } + + /// `ColumnLineage` of one native column; `nullability` is the enum member to record. + pub(crate) fn column( + &self, + column: &LineageColumn, + nullability: &Bound<'py, PyAny>, + resource_enum: bool, + ) -> PyResult> { + let mut sources: Vec> = Vec::with_capacity(column.upstream_columns.len()); + for source in &column.upstream_columns { + let resource_type: Bound<'py, PyAny> = if resource_enum { + self.resource_type.call1((source.resource_type.as_str(),))? + } else { + pyo3::types::PyString::new(self.py(), source.resource_type.as_str()).into_any() + }; + sources.push(self.lineage_source.call1(( + resource_type, + &source.resource_name, + &source.column_name, + ))?); + } + self.column_lineage.call1(( + &column.output_column, + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + self.py().None(), + PyTuple::new(self.py(), sources)?, + nullability, + self.confidence.call1((column.confidence.as_str(),))?, + )) + } + + /// `CompiledLineageColumnFact` of one native star-expansion column. + pub(crate) fn column_fact(&self, column: &LineageColumn) -> PyResult> { + let sources: Vec> = column + .upstream_columns + .iter() + .map(|source| self.source_fact(source)) + .collect::>()?; + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("output_column", &column.output_column)?; + fields.set_item("upstream_columns", PyTuple::new(self.py(), sources)?)?; + fields.set_item( + "transform_kind", + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + )?; + fields.set_item( + "confidence", + self.confidence.call1((column.confidence.as_str(),))?, + )?; + self.column_fact.call((), Some(&fields)) + } + + fn source_fact(&self, source: &LineageSource) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("resource_type", source.resource_type.as_str())?; + fields.set_item("resource_name", &source.resource_name)?; + fields.set_item("column_name", &source.column_name)?; + self.source_fact.call((), Some(&fields)) + } +} + +/// A structured debug event on the `sqlbuild.lineage` logger for one model. +pub(crate) fn log_lineage_event( + py: Python<'_>, + message: &str, + model_name: &str, + error: Option<&str>, +) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import(LOGGING_MODULE)? + .getattr("getLogger")? + .call1((LINEAGE_LOGGER,))?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("logger", logger)?; + fields.set_item("message", message)?; + fields.set_item("sqlbuild_model", model_name)?; + fields.set_item("sqlbuild_error", error)?; + let _ = py + .import(DEBUG_EVENT_MODULE)? + .getattr("log_debug_event")? + .call((), Some(&fields))?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index f0aec5490f..6ac630c2c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,4 +1,6 @@ -pub(crate) mod fast_lineage; +pub(crate) mod column_lineage_graph; +pub(crate) mod lineage_objects; +pub(crate) mod project_lineage; +pub(crate) mod project_schema; pub(crate) mod registration; pub(crate) mod relation_fingerprint; -pub(crate) mod rich_lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs new file mode 100644 index 0000000000..93d1b1f820 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -0,0 +1,372 @@ +//! Project column lineage in FAST or RICH mode, and direct semantic uses, from a project view. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage::main::build_fast_lineage::build_fast_lineage; +use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; +use sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses; +use sqlbuild_analysis::lineage::models::{ + DirectSemanticUse, FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, + RichLineageOutcome, RichLineageRequest, +}; +use sqlbuild_analysis::lineage_graph::main::analysis_dialect::analysis_dialect; +use sqlbuild_analysis::lineage_graph::models::AnalysisDialect; +use sqlbuild_analysis::semantic_validation::models as validation; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; +use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_lineage_event}; +use crate::bindings::_helpers::lineage::project_schema::{ + fast_schema, inferred_names, rich_schema, +}; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.lineage.exceptions"; +const FAST_MODE: &str = "fast"; +const RICH_MODE: &str = "rich"; +const GENERIC_DIALECT: &str = "generic"; + +/// The project's column lineage graph in `mode`, or `None` without SQL analysis. +#[pyfunction] +fn build_project_column_lineage<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + mode: &Bound<'py, PyAny>, + model_names: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let mode: String = mode.str()?.extract()?; + if mode != FAST_MODE && mode != RICH_MODE { + return Ok(None); + } + let classes: LineageClasses<'py> = LineageClasses::new(project.py())?; + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(None); + } + let models: Vec> = selected_models(project, model_names)?; + let lineage: Bound<'py, PyAny> = if mode == FAST_MODE { + fast_lineage(&classes, project, dialect, &models)? + } else { + rich_lineage(&classes, project, dialect, &models)? + }; + Ok(Some(lineage.unbind())) +} + +/// Direct non-projection semantic column uses of the requested models. +#[pyfunction] +fn build_direct_semantic_uses<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&str>, + model_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let classes: LineageClasses<'py> = LineageClasses::new(py)?; + let schema = rich_schema(project)?; + let mut models: Vec<(String, String, String)> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + if model_names.contains(&name)? { + models.push(( + name.extract()?, + model.getattr("query_sql")?.extract()?, + model.getattr("authored_sql")?.extract()?, + )); + } + } + let dialect: &str = dialect + .filter(|name| !name.is_empty()) + .unwrap_or(GENERIC_DIALECT); + let uses: Vec = py + .compiler_detach(|| direct_semantic_uses(&schema, &models, dialect)) + .map_err(compiler_error)?; + let objects: Vec> = uses + .into_iter() + .map(|usage| semantic_use(&classes, usage)) + .collect::>()?; + PyTuple::new(py, objects) +} + +fn semantic_use<'py>( + classes: &LineageClasses<'py>, + usage: DirectSemanticUse, +) -> PyResult> { + let source: Bound<'py, PyAny> = classes.qualified( + &classes + .resource_type + .call1((usage.resource_type.as_str(),))?, + &usage.resource_name, + &usage.column, + )?; + classes.semantic_use.call1(( + usage.model, + usage.context, + usage.expression, + source, + classes.confidence.call1((usage.confidence,))?, + usage.line, + usage.offset, + )) +} + +fn selected_models<'py>( + project: &Bound<'py, PyAny>, + model_names: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let mut models: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let selected: bool = match model_names { + None => true, + Some(names) => names.contains(model.getattr("name")?)?, + }; + if selected { + models.push(model); + } + } + Ok(models) +} + +/// One model's FAST lineage work: its compact facts and whether its root star expands. +struct FastModel<'py> { + model: Bound<'py, PyAny>, + name: String, + facts: Option>, + expands_star: bool, +} + +fn fast_lineage<'py>( + classes: &LineageClasses<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut work: Vec> = Vec::with_capacity(models.len()); + let mut requests: Vec = Vec::new(); + for model in models { + let facts: Bound<'py, PyAny> = model.getattr("fast_lineage_columns")?; + let expands_star: bool = model.getattr("fast_lineage_has_star")?.is_truthy()? + && !model.getattr("fast_lineage_star_resolved")?.is_truthy()?; + if facts.is_none() { + requests.push(FastLineageModel::Parse { + query_sql: model.getattr("query_sql")?.extract()?, + inferred_columns: inferred_names(model)?, + }); + } else if expands_star { + requests.push(FastLineageModel::StarExpansion { + query_sql: model.getattr("query_sql")?.extract()?, + existing_columns: output_columns(classes, &facts)?, + }); + } + work.push(FastModel { + model: model.clone(), + name: model.getattr("name")?.extract()?, + facts: (!facts.is_none()).then_some(facts), + expands_star, + }); + } + let outcomes: Vec = if requests.is_empty() { + Vec::new() + } else { + let binding_catalog: Bound<'py, PyAny> = project.getattr("binding_catalog")?; + let catalog: Option> = if binding_catalog.is_none() { + None + } else { + binding_catalog.getattr("native")?.extract()? + }; + let request = FastLineageRequest { + dialect: dialect.map(|name| name.extract()).transpose()?, + schema: fast_schema(project)?, + models: requests, + }; + let native: Option<&validation::ProjectCatalog> = + catalog.as_ref().map(|catalog| &catalog.inner); + py.compiler_detach(|| build_fast_lineage(&request, native)) + .map_err(compiler_error)? + }; + let mut outcomes = outcomes.into_iter(); + let built: Bound<'py, PyDict> = PyDict::new(py); + let compact: Bound<'py, PyDict> = PyDict::new(py); + let mut model_order: Vec = Vec::new(); + for model in work { + if let Some(mut columns) = model.facts { + if model.expands_star { + let star_columns: Vec = outcomes + .next() + .map(|outcome| outcome.into_parts().1) + .unwrap_or_default(); + let mut expanded: Vec> = + columns.try_iter()?.collect::>()?; + for column in &star_columns { + expanded.push(classes.column_fact(column)?); + } + columns = PyTuple::new(py, expanded)?.into_any(); + } + compact.set_item( + &model.name, + (columns, model.model.getattr("fast_lineage_has_star")?), + )?; + model_order.push(model.name); + continue; + } + match outcomes.next() { + Some(FastLineageOutcome::Built { columns, has_star }) => { + let lineage_columns: Vec> = columns + .iter() + .map(|column| classes.column(column, &classes.unknown_nullability, false)) + .collect::>()?; + built.set_item( + &model.name, + classes.model(&model.name, lineage_columns, has_star)?, + )?; + model_order.push(model.name); + } + Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( + py, + "fast column lineage parse failed; falling back", + &model.name, + Some(&detail), + )?, + Some(FastLineageOutcome::UnknownDialect(detail)) => { + return Err(unknown_dialect_error(py, &detail)); + } + _ => {} + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("models", built)?; + fields.set_item("compact_models", compact)?; + fields.set_item("model_order", PyTuple::new(py, model_order)?)?; + py.get_type::() + .call_method("from_fast_facts", (), Some(&fields)) +} + +/// The output column names of compact facts or plain column facts. +fn output_columns(classes: &LineageClasses<'_>, facts: &Bound<'_, PyAny>) -> PyResult> { + if facts.is_instance(&classes.compact_facts)? { + let pool: Bound<'_, PyAny> = facts.getattr("string_pool")?; + return facts + .getattr("rows")? + .try_iter()? + .map(|row| pool.get_item(row?.get_item(0)?)?.extract()) + .collect(); + } + facts + .try_iter()? + .map(|column| column?.getattr("output_column")?.extract()) + .collect() +} + +fn unknown_dialect_error(py: Python<'_>, dialect: &str) -> PyErr { + match py + .import(EXCEPTIONS_MODULE) + .and_then(|module| module.getattr("UnknownLineageDialectError")) + .and_then(|class| class.call1((format!("Unknown dialect: {dialect}"),))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn rich_lineage<'py>( + classes: &LineageClasses<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = classes.py(); + let outcomes: Vec = if models.is_empty() { + Vec::new() + } else { + let request = RichLineageRequest { + dialect: polyglot_dialect(classes, dialect)?, + schema: rich_schema(project)?, + models: models + .iter() + .map(|model| model.getattr("query_sql")?.extract()) + .collect::>()?, + }; + py.compiler_detach(|| build_rich_lineage(&request)) + .map_err(compiler_error)? + }; + let built: Bound<'py, PyDict> = PyDict::new(py); + let mut edges: Vec> = Vec::new(); + for (model, outcome) in models.iter().zip(outcomes) { + let name: String = model.getattr("name")?.extract()?; + match outcome { + RichLineageOutcome::Built { columns, has_star } => { + let mut lineage_columns: Vec> = Vec::with_capacity(columns.len()); + for column in &columns { + let nullability: Bound<'py, PyAny> = + classes.nullability.call1((column.nullability.as_str(),))?; + lineage_columns.push(classes.column(&column.column, &nullability, true)?); + } + built.set_item(&name, classes.model(&name, lineage_columns, has_star)?)?; + } + RichLineageOutcome::Skipped(detail) => log_lineage_event( + py, + "rich column lineage analysis failed; skipping model", + &name, + Some(&detail), + )?, + } + } + for item in built.iter() { + let (target_name, result): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item; + for column in result.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let target: Bound<'py, PyAny> = classes.qualified_column.call1(( + &classes.model_resource, + &target_name, + column.getattr("output_column")?, + ))?; + let transform: Bound<'py, PyAny> = column.getattr("transform_kind")?; + let confidence: Bound<'py, PyAny> = column.getattr("confidence")?; + for upstream in column.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'py, PyAny> = upstream?.call_method0("as_qualified_column")?; + edges.push(classes.edge(source, &target, &transform, &confidence)?); + } + } + } + let lineage: Bound<'py, PyAny> = py.get_type::().into_any(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("models", built)?; + fields.set_item("edges", PyTuple::new(py, edges)?)?; + lineage.call((), Some(&fields)) +} + +/// Python's `_polyglot_dialect(dialect).value`. +fn polyglot_dialect( + classes: &LineageClasses<'_>, + dialect: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let Some(dialect) = dialect else { + return Ok(GENERIC_DIALECT.to_owned()); + }; + let normalized: String = dialect + .call_method0("strip")? + .call_method0("lower")? + .extract()?; + match analysis_dialect(&normalized) { + AnalysisDialect::Alias(value) => Ok(value.to_owned()), + AnalysisDialect::Named(name) => classes + .analysis_dialect + .call1((name,))? + .getattr("value")? + .extract(), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_project_column_lineage, module)?)?; + module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs new file mode 100644 index 0000000000..43085f139d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs @@ -0,0 +1,112 @@ +//! Project resources and their columns, read from a compiled project view for lineage. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::lineage::models::{ + LineageResourceType, LineageSchemaResource, RichSchemaResource, +}; + +type TypedColumns = Vec<(String, Option)>; + +/// The project's models, then sources, then seeds, each with its typed and schema columns. +struct ProjectColumns { + resource_type: LineageResourceType, + name: String, + typed: TypedColumns, + schema: TypedColumns, +} + +/// Fast lineage's schema: model inferred then schema column names; source and seed columns. +pub(crate) fn fast_schema(project: &Bound<'_, PyAny>) -> PyResult> { + let mut schema: Vec = Vec::new(); + for resource in project_columns(project)? { + let mut columns: Vec = Vec::with_capacity(resource.typed.len()); + for (name, _) in resource.typed.into_iter().chain(resource.schema) { + columns.push(name); + } + schema.push(LineageSchemaResource { + resource_type: resource.resource_type, + name: resource.name, + columns, + }); + } + Ok(schema) +} + +/// Rich lineage's typed schema: inferred or declared types, then model schema columns. +pub(crate) fn rich_schema(project: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_columns(project)? + .into_iter() + .map(|resource| RichSchemaResource { + resource_type: resource.resource_type, + name: resource.name, + assigned: resource.typed, + defaulted: resource.schema, + }) + .collect()) +} + +/// `[name for name, _, _ in model.inferred_rows or ()]`. +pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult> { + Ok(inferred_columns(model)? + .into_iter() + .map(|(name, _)| name) + .collect()) +} + +fn project_columns(project: &Bound<'_, PyAny>) -> PyResult> { + let mut resources: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + resources.push(ProjectColumns { + resource_type: LineageResourceType::Model, + name: model.getattr("name")?.extract()?, + typed: inferred_columns(&model)?, + schema: schema_columns(&model)?, + }); + } + for (attribute, entry, resource_type) in [ + ("sources", "source_entry", LineageResourceType::Source), + ("seeds", "schema_entry", LineageResourceType::Seed), + ] { + for resource in project.getattr(attribute)?.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let mut typed: TypedColumns = Vec::new(); + for column in resource.getattr(entry)?.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + typed.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + resources.push(ProjectColumns { + resource_type, + name: resource.getattr("name")?.extract()?, + typed, + schema: Vec::new(), + }); + } + } + Ok(resources) +} + +fn inferred_columns(model: &Bound<'_, PyAny>) -> PyResult { + let rows: Bound<'_, PyAny> = model.getattr("inferred_rows")?; + if !rows.is_truthy()? { + return Ok(Vec::new()); + } + rows.try_iter()? + .map(|row| { + let (name, column_type, _nullability): (String, Option, Bound<'_, PyAny>) = + row?.extract()?; + Ok((name, column_type)) + }) + .collect() +} + +fn schema_columns(model: &Bound<'_, PyAny>) -> PyResult { + let rows: Bound<'_, PyAny> = model.getattr("schema_column_rows")?; + if !rows.is_truthy()? { + return Ok(Vec::new()); + } + rows.try_iter()?.map(|row| row?.extract()).collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index 806db27529..58ac10cb33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -1,11 +1,13 @@ -//! Register the native column lineage functions. +//! Register the native column lineage functions and classes. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lineage::{fast_lineage, relation_fingerprint, rich_lineage}; +use crate::bindings::_helpers::lineage::{ + column_lineage_graph, project_lineage, relation_fingerprint, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - fast_lineage::register(module)?; - relation_fingerprint::register(module)?; - rich_lineage::register(module) + column_lineage_graph::register(module)?; + project_lineage::register(module)?; + relation_fingerprint::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs deleted file mode 100644 index 2a9c17084f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Rich column lineage. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; -use sqlbuild_analysis::lineage::models::{ - LineageResourceType, LineageSource, RichLineageColumn, RichLineageOutcome, RichLineageRequest, - RichSchemaResource, -}; - -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::types::CompilerDetach; - -type TypedColumns = Vec<(String, Option)>; -type SourceRow = (&'static str, String, String); -type ColumnRow = ( - String, - &'static str, - &'static str, - &'static str, - Vec, -); -/// `(dialect, [(resource_type, name, assigned, defaulted)], [query_sql])`. -type RichRequestInput = ( - String, - Vec<(String, String, TypedColumns, TypedColumns)>, - Vec, -); -/// `(status, columns, has_star, detail)`: detail is the polyglot error or the deferral kind. -type OutcomeRow = (&'static str, Vec, bool, Option); - -/// Build rich lineage for the requested models' SQL, in request order. -#[pyfunction] -fn build_rich_column_lineage( - py: Python<'_>, - request: RichRequestInput, -) -> PyResult> { - let (dialect, schema, models) = request; - let schema: Vec = schema - .into_iter() - .map(schema_resource) - .collect::>()?; - let request = RichLineageRequest { - dialect, - schema, - models, - }; - let outcomes: Vec = py - .compiler_detach(|| build_rich_lineage(&request)) - .map_err(compiler_error)?; - Ok(outcomes.into_iter().map(outcome_row).collect()) -} - -fn schema_resource( - (resource_type, name, assigned, defaulted): (String, String, TypedColumns, TypedColumns), -) -> PyResult { - let Some(kind) = LineageResourceType::from_value(&resource_type) else { - return Err(PyValueError::new_err(format!( - "unknown resource: {resource_type}" - ))); - }; - Ok(RichSchemaResource { - resource_type: kind, - name, - assigned, - defaulted, - }) -} - -fn outcome_row(outcome: RichLineageOutcome) -> OutcomeRow { - let (status, columns, has_star, detail) = outcome.into_parts(); - ( - status, - columns.into_iter().map(column_row).collect(), - has_star, - detail, - ) -} - -fn column_row(column: RichLineageColumn) -> ColumnRow { - let RichLineageColumn { - column, - nullability, - } = column; - ( - column.output_column, - column.transform_kind.as_str(), - column.confidence.as_str(), - nullability.as_str(), - column - .upstream_columns - .into_iter() - .map(source_row) - .collect(), - ) -} - -fn source_row(source: LineageSource) -> SourceRow { - ( - source.resource_type.as_str(), - source.resource_name, - source.column_name, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_rich_column_lineage, module)?)?; - module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; - Ok(()) -} - -type SemanticUseRow = ( - String, - String, - String, - &'static str, - String, - String, - String, - Option, - Option, -); - -/// Project semantic uses onto authored resources without Python analysis reconstruction. -#[pyfunction] -fn build_direct_semantic_uses( - py: Python<'_>, - schema: Vec<(String, String, TypedColumns, TypedColumns)>, - models: Vec<(String, String, String)>, - dialect: &str, -) -> PyResult> { - let schema = schema - .into_iter() - .map(schema_resource) - .collect::>>()?; - let uses = py - .compiler_detach(|| { - sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses( - &schema, &models, dialect, - ) - }) - .map_err(compiler_error)?; - Ok(uses - .into_iter() - .map(|usage| { - ( - usage.model, - usage.context, - usage.expression, - usage.resource_type.as_str(), - usage.resource_name, - usage.column, - usage.confidence, - usage.line, - usage.offset, - ) - }) - .collect()) -} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5704e7e9cb..a75cb6bcb7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -80,6 +80,11 @@ from sqlbuild.compiler.discovery.models import ( TolerantScopeDiscovery, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.lineage.models import ( + ColumnLineageEdge, + DirectSemanticColumnUse, + ModelColumnLineage, +) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, @@ -1766,26 +1771,50 @@ def evaluate_project_model_contracts( def project_promotion_conflicts( settings: tuple[str | None, str, str], project: NativeCompiledProject ) -> list[tuple[str, str, str, str, str]]: ... -def build_fast_column_lineage( - catalog: object | None, - request: tuple[str | None, list[tuple[str, str, list[str]]], list[tuple[bool, str, list[str]]]], +def build_project_column_lineage( + project: CompiledProjectView, + dialect: str | None, + mode: ColumnLineageMode, + model_names: frozenset[str] | None, /, -) -> list[tuple[str, list[tuple[str, str, str, list[tuple[str, str, str]]]], bool, str | None]]: ... +) -> ProjectColumnLineage | None: ... def build_direct_semantic_uses( - schema: list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], - models: list[tuple[str, str, str]], - dialect: str, -) -> list[tuple[str, str, str, str, str, str, str, int | None, int | None]]: ... -def build_rich_column_lineage( - request: tuple[ - str, - list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], - list[str], - ], + project: CompiledProjectView, + dialect: str | None, + model_names: frozenset[str], /, -) -> list[ - tuple[str, list[tuple[str, str, str, str, list[tuple[str, str, str]]]], bool, str | None] -]: ... +) -> tuple[DirectSemanticColumnUse, ...]: ... + +class ProjectColumnLineage: + """Project-level column lineage graph with lazily materialized public views.""" + + def __init__( + self, + *, + models: dict[str, ModelColumnLineage], + edges: tuple[ColumnLineageEdge, ...], + ) -> None: ... + @classmethod + def from_fast_facts( + cls, + *, + models: dict[str, ModelColumnLineage], + compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]], + model_order: tuple[str, ...], + ) -> ProjectColumnLineage: ... + @property + def models(self) -> dict[str, ModelColumnLineage]: ... + @property + def edges(self) -> tuple[ColumnLineageEdge, ...]: ... + def has_model(self, model_name: str, /) -> bool: ... + def model_has_star(self, model_name: str, /) -> bool: ... + def edge_count_targeting(self, model_name: str, /) -> int: ... + def edges_targeting(self, model_name: str, /) -> tuple[ColumnLineageEdge, ...]: ... + def edges_sourced_from(self, resource_name: str, /) -> tuple[ColumnLineageEdge, ...]: ... + def column_consumers( + self, *, resource_name: str, column_name: str + ) -> tuple[ColumnLineageEdge, ...]: ... + def relation_lineage_fingerprint( project_dir: Path, prefix: bytes, interrupted_listing_uncacheable: bool, / ) -> str | None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py b/src/sqlbuild/cli/commands/_helpers/compile/lineage.py index fc1150b4e8..0aaad2bd5b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/lineage.py @@ -2,9 +2,9 @@ from __future__ import annotations +import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -19,16 +19,12 @@ def build_compile_lineage( case CompileLineageMode.NONE: return None case CompileLineageMode.FAST: - return build_project_column_lineage( - project=graph.project, - dialect=dialect, - mode=ColumnLineageMode.FAST, + return _native.build_project_column_lineage( + graph.project, dialect, ColumnLineageMode.FAST, None ) case CompileLineageMode.RICH: - return build_project_column_lineage( - project=graph.project, - dialect=dialect, - mode=ColumnLineageMode.RICH, + return _native.build_project_column_lineage( + graph.project, dialect, ColumnLineageMode.RICH, None ) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 97ef017e1c..08b08badab 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -7,6 +7,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.compile.semantic_notice import ( selected_semantic_coverage, semantic_coverage_notice, @@ -36,7 +37,6 @@ ) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py index e44ccc91ae..5a25c8072b 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py @@ -5,6 +5,7 @@ import json from pathlib import Path +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.lineage.cache import ( read_relation_lineage_cache, @@ -158,17 +159,15 @@ def _semantic_uses( ) -> tuple[DirectSemanticColumnUse, ...]: if not request.include_uses: return () - from sqlbuild.compiler.lineage.main.build_semantic_uses import build_direct_semantic_uses - if not isinstance(graph, ProjectGraph) or adapter is None: raise CliUserError( "semantic uses require a compiled project graph", code="C307", ) - return build_direct_semantic_uses( - project=graph.project, - dialect=adapter.sql_analysis_dialect(), - model_names=_semantic_use_model_names(request=request, graph=graph), + return _native.build_direct_semantic_uses( + graph.project, + adapter.sql_analysis_dialect(), + _semantic_use_model_names(request=request, graph=graph), ) diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py index 61f8dfe0bb..4e6a948f77 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py @@ -5,6 +5,8 @@ from collections.abc import Iterable from dataclasses import dataclass +import sqlbuild._native as _native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands.constants import ( BOTH_DIRECTIONS, COLUMN_TARGET_KIND, @@ -31,10 +33,8 @@ from sqlbuild.compiler.graph.main.path_nodes import path_nodes from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, - ProjectColumnLineage, QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -225,10 +225,8 @@ def select_column_target_lineage( and len(candidate_selection.model_names) >= RICH_LINEAGE_STATUS_MODEL_THRESHOLD ), ): - column_lineage: ProjectColumnLineage | None = build_project_column_lineage( - project=graph.project, - mode=mode, - model_names=candidate_selection.model_names, + column_lineage: ProjectColumnLineage | None = _native.build_project_column_lineage( + graph.project, None, mode, candidate_selection.model_names ) if column_lineage is None: raise CliUserError( diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index 4fde249c68..fd437097ee 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path +from sqlbuild._native import ProjectColumnLineage from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.compile.models import ( @@ -13,7 +14,6 @@ CompilerDiagnostic, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.models import NativeSqlTestArtifact diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py deleted file mode 100644 index 48e0931995..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Fast column lineage built by the native engine, assembled exactly as Python assembles it.""" - -from __future__ import annotations - -import logging -from collections.abc import Sequence - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage.constants import ( - NATIVE_LINEAGE_BUILT, - NATIVE_LINEAGE_UNKNOWN_DIALECT, - NATIVE_LINEAGE_UNPARSED, -) -from sqlbuild.compiler.lineage.exceptions import UnknownLineageDialectError -from sqlbuild.compiler.lineage.models import ( - ColumnLineage, - ColumnLineageSource, - ModelColumnLineage, - ProjectColumnLineage, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") - -type _NativeColumn = tuple[str, str, str, list[tuple[str, str, str]]] -type _NativeOutcome = tuple[str, list[_NativeColumn], bool, str | None] - - -def build_native_fast_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None, - model_names: frozenset[str] | None, -) -> ProjectColumnLineage | None: - """Build the fast lineage graph with native star expansion and parsing.""" - - if not project.settings.sql_analysis: - return None - models: tuple[CompiledModelView, ...] = tuple( - model for model in project.models if model_names is None or model.name in model_names - ) - requests: list[tuple[bool, str, list[str]]] = [] - for model in models: - request: tuple[bool, str, list[str]] | None = _native_request(model) - if request is not None: - requests.append(request) - catalog: object | None = ( - project.binding_catalog.native if project.binding_catalog is not None else None - ) - outcomes: list[_NativeOutcome] = ( - _native.build_fast_column_lineage(catalog, (dialect, _schema_resources(project), requests)) - if requests - else [] - ) - outcome_index: int = 0 - model_results: dict[str, ModelColumnLineage] = {} - compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]] = {} - model_order: list[str] = [] - for model in models: - if model.fast_lineage_columns is not None: - columns: Sequence[CompiledLineageColumnFact] = model.fast_lineage_columns - if _needs_star_expansion(model): - _, star_columns, _, _ = outcomes[outcome_index] - outcome_index += 1 - columns = (*columns, *(_column_fact(column) for column in star_columns)) - compact_models[model.name] = (columns, model.fast_lineage_has_star) - model_order.append(model.name) - continue - status, built_columns, has_star, detail = outcomes[outcome_index] - outcome_index += 1 - result: ModelColumnLineage | None = None - if status == NATIVE_LINEAGE_BUILT: - result = ModelColumnLineage( - model_name=model.name, - columns=tuple(_column_lineage(column) for column in built_columns), - has_star=has_star, - ) - elif status == NATIVE_LINEAGE_UNPARSED: - log_debug_event( - logger=_DEBUG_LOGGER, - message="fast column lineage parse failed; falling back", - sqlbuild_model=model.name, - sqlbuild_error=detail, - ) - elif status == NATIVE_LINEAGE_UNKNOWN_DIALECT: - raise UnknownLineageDialectError(f"Unknown dialect: {detail}") - if result is None: - continue - model_results[model.name] = result - model_order.append(model.name) - return ProjectColumnLineage.from_fast_facts( - models=model_results, - compact_models=compact_models, - model_order=tuple(model_order), - ) - - -def _needs_star_expansion(model: CompiledModelView) -> bool: - return model.fast_lineage_has_star and not model.fast_lineage_star_resolved - - -def _native_request(model: CompiledModelView) -> tuple[bool, str, list[str]] | None: - columns: Sequence[CompiledLineageColumnFact] | None = model.fast_lineage_columns - if columns is None: - return (False, model.query_sql, [name for name, _, _ in model.inferred_rows or ()]) - if not _needs_star_expansion(model): - return None - if isinstance(columns, CompactLineageFacts): - return (True, model.query_sql, [columns.string_pool[row[0]] for row in columns.rows]) - return (True, model.query_sql, [column.output_column for column in columns]) - - -def _schema_resources(project: CompiledProjectView) -> list[tuple[str, str, list[str]]]: - resources: list[tuple[str, str, list[str]]] = [] - for model in project.models: - columns: list[str] = [name for name, _, _ in model.inferred_rows or ()] - columns.extend(name for name, _ in model.schema_column_rows or ()) - resources.append((CompiledResourceType.MODEL.value, model.name, columns)) - for source in project.sources: - resources.append( - ( - CompiledResourceType.SOURCE.value, - source.name, - [column.name for column in source.source_entry.columns], - ) - ) - for seed in project.seeds: - resources.append( - ( - CompiledResourceType.SEED.value, - seed.name, - [column.name for column in seed.schema_entry.columns], - ) - ) - return resources - - -def _column_fact(column: _NativeColumn) -> CompiledLineageColumnFact: - output_column, transform_kind, confidence, sources = column - return CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=tuple( - CompiledLineageSourceFact( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - transform_kind=ColumnTransformKind(transform_kind), - confidence=ColumnLineageConfidence(confidence), - ) - - -def _column_lineage(column: _NativeColumn) -> ColumnLineage: - output_column, transform_kind, confidence, sources = column - return ColumnLineage( - output_column=output_column, - transform_kind=ColumnTransformKind(transform_kind), - expression_sql=None, - upstream_columns=tuple( - ColumnLineageSource( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - nullability=InferredNullability.UNKNOWN, - confidence=ColumnLineageConfidence(confidence), - ) diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py deleted file mode 100644 index 84f3bdd6cc..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Rich column lineage built by the native engine, assembled exactly as Python assembles it.""" - -from __future__ import annotations - -import logging - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage._helpers.rich_columns import ( - _polyglot_dialect, - assemble_rich_project_column_lineage, -) -from sqlbuild.compiler.lineage.constants import ( - NATIVE_LINEAGE_BUILT, - NATIVE_RICH_LINEAGE_SKIPPED, -) -from sqlbuild.compiler.lineage.models import ( - ColumnLineage, - ColumnLineageSource, - ModelColumnLineage, - ProjectColumnLineage, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") - -type _TypedColumns = list[tuple[str, str | None]] -type _NativeColumn = tuple[str, str, str, str, list[tuple[str, str, str]]] -type _NativeOutcome = tuple[str, list[_NativeColumn], bool, str | None] - - -def build_native_rich_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None, - model_names: frozenset[str] | None, -) -> ProjectColumnLineage | None: - """Build the rich project column lineage graph with native query analysis.""" - - if not project.settings.sql_analysis: - return None - models: tuple[CompiledModelView, ...] = tuple( - model for model in project.models if model_names is None or model.name in model_names - ) - outcomes: list[_NativeOutcome] = ( - _native.build_rich_column_lineage( - ( - _polyglot_dialect(dialect).value, - _schema_resources(project), - [model.query_sql for model in models], - ) - ) - if models - else [] - ) - model_results: dict[str, ModelColumnLineage] = {} - for model, (status, columns, has_star, detail) in zip(models, outcomes, strict=True): - if status == NATIVE_LINEAGE_BUILT: - model_results[model.name] = ModelColumnLineage( - model_name=model.name, - columns=tuple(_column_lineage(column) for column in columns), - has_star=has_star, - ) - elif status == NATIVE_RICH_LINEAGE_SKIPPED: - log_debug_event( - logger=_DEBUG_LOGGER, - message="rich column lineage analysis failed; skipping model", - sqlbuild_model=model.name, - sqlbuild_error=detail, - ) - return assemble_rich_project_column_lineage(model_results) - - -def _schema_resources( - project: CompiledProjectView, -) -> list[tuple[str, str, _TypedColumns, _TypedColumns]]: - resources: list[tuple[str, str, _TypedColumns, _TypedColumns]] = [] - for model in project.models: - resources.append( - ( - CompiledResourceType.MODEL.value, - model.name, - [(name, column_type) for name, column_type, _ in model.inferred_rows or ()], - list(model.schema_column_rows or ()), - ) - ) - for source in project.sources: - resources.append( - ( - CompiledResourceType.SOURCE.value, - source.name, - [(column.name, column.type) for column in source.source_entry.columns], - [], - ) - ) - for seed in project.seeds: - resources.append( - ( - CompiledResourceType.SEED.value, - seed.name, - [(column.name, column.type) for column in seed.schema_entry.columns], - [], - ) - ) - return resources - - -def _column_lineage(column: _NativeColumn) -> ColumnLineage: - output_column, transform_kind, confidence, nullability, sources = column - return ColumnLineage( - output_column=output_column, - transform_kind=ColumnTransformKind(transform_kind), - expression_sql=None, - upstream_columns=tuple( - ColumnLineageSource( - resource_type=CompiledResourceType(resource_type), - resource_name=resource_name, - column_name=column_name, - ) - for resource_type, resource_name, column_name in sources - ), - nullability=InferredNullability(nullability), - confidence=ColumnLineageConfidence(confidence), - ) diff --git a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py deleted file mode 100644 index 42e4c58586..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Rich column lineage glue: the analysis dialect name and the project graph assembly.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.types import BuiltinAdapter, TypeDialect -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - ModelColumnLineage, - ProjectColumnLineage, - QualifiedLineageColumn, -) -from sqlbuild.compiler.lineage.types import PolyglotAnalysisDialect - -_POLYGLOT_DIALECT_ALIASES: dict[str, PolyglotAnalysisDialect] = { - "": PolyglotAnalysisDialect.GENERIC, - "ansi": PolyglotAnalysisDialect.GENERIC, - "generic": PolyglotAnalysisDialect.GENERIC, - TypeDialect.GENERIC.value: PolyglotAnalysisDialect.GENERIC, - TypeDialect.BIGQUERY.value: PolyglotAnalysisDialect.BIGQUERY, - TypeDialect.SNOWFLAKE.value: PolyglotAnalysisDialect.SNOWFLAKE, - TypeDialect.DUCKDB.value: PolyglotAnalysisDialect.DUCKDB, - TypeDialect.MOTHERDUCK.value: PolyglotAnalysisDialect.DUCKDB, - TypeDialect.DATABRICKS.value: PolyglotAnalysisDialect.DATABRICKS, - TypeDialect.POSTGRES.value: PolyglotAnalysisDialect.POSTGRESQL, - TypeDialect.TSQL.value: PolyglotAnalysisDialect.TSQL, - BuiltinAdapter.SQLSERVER.value: PolyglotAnalysisDialect.TSQL, - "postgresql": PolyglotAnalysisDialect.POSTGRESQL, -} - - -def assemble_rich_project_column_lineage( - model_results: dict[str, ModelColumnLineage], -) -> ProjectColumnLineage: - """Collapse per-model rich lineage, in model order, into the project graph and its edges.""" - - collapsed_edges: list[ColumnLineageEdge] = [] - for target_resource_name, result in model_results.items(): - for column in result.columns: - target: QualifiedLineageColumn = QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name=target_resource_name, - column_name=column.output_column, - ) - for upstream in column.upstream_columns: - collapsed_edges.append( - ColumnLineageEdge( - source=upstream.as_qualified_column(), - target=target, - transform_kind=column.transform_kind, - confidence=column.confidence, - ) - ) - - return ProjectColumnLineage(models=model_results, edges=tuple(collapsed_edges)) - - -def _polyglot_dialect(dialect: str | TypeDialect | None) -> PolyglotAnalysisDialect: - if dialect is None: - return PolyglotAnalysisDialect.GENERIC - normalized: str = dialect.strip().lower() - mapped: PolyglotAnalysisDialect | None = _POLYGLOT_DIALECT_ALIASES.get(normalized) - if mapped is not None: - return mapped - return PolyglotAnalysisDialect(normalized) diff --git a/src/sqlbuild/compiler/lineage/constants.py b/src/sqlbuild/compiler/lineage/constants.py deleted file mode 100644 index 8f5dc7f196..0000000000 --- a/src/sqlbuild/compiler/lineage/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Column lineage analysis constants.""" - -STAR_COLUMN_NAME: str = "*" -INDEXED_EDGE_RECORD_LENGTH: int = 6 -NATIVE_LINEAGE_BUILT: str = "built" -NATIVE_LINEAGE_UNPARSED: str = "unparsed" -NATIVE_LINEAGE_UNKNOWN_DIALECT: str = "unknown_dialect" -NATIVE_RICH_LINEAGE_SKIPPED: str = "skipped" diff --git a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py deleted file mode 100644 index c558388a77..0000000000 --- a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Build fast column lineage facts natively.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage._helpers.native_fast_columns import ( - build_native_fast_project_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage - - -def build_native_column_lineage( - *, project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] | None -) -> ProjectColumnLineage | None: - """Return the fast lineage graph, or None without SQL analysis.""" - - return build_native_fast_project_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) diff --git a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py b/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py deleted file mode 100644 index 07da567097..0000000000 --- a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Build on-demand direct non-projection semantic column uses.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.lineage._helpers.native_rich_columns import _schema_resources -from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse, QualifiedLineageColumn -from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, NativeSemanticUseRow - - -def build_direct_semantic_uses( - *, project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] -) -> tuple[DirectSemanticColumnUse, ...]: - """Extract semantic uses only for explicitly requested models.""" - - rows: list[NativeSemanticUseRow] = _native.build_direct_semantic_uses( - _schema_resources(project), - [ - (model.name, model.query_sql, model.authored_sql) - for model in project.models - if model.name in model_names - ], - dialect or "generic", - ) - return tuple( - DirectSemanticColumnUse( - consumer_model=model, - context=context, - expression_sql=expression, - source=QualifiedLineageColumn( - resource_type=CompiledResourceType(kind), - resource_name=resource, - column_name=column, - ), - confidence=ColumnLineageConfidence(confidence), - line=line, - column=offset, - ) - for model, context, expression, kind, resource, column, confidence, line, offset in rows - ) diff --git a/src/sqlbuild/compiler/lineage/main/columns.py b/src/sqlbuild/compiler/lineage/main/columns.py deleted file mode 100644 index 1de9498177..0000000000 --- a/src/sqlbuild/compiler/lineage/main/columns.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Column lineage analyzer entry point.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage._helpers.native_rich_columns import ( - build_native_rich_project_column_lineage, -) -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage -from sqlbuild.compiler.lineage.types import ColumnLineageMode - - -def build_project_column_lineage( - *, - project: CompiledProjectView, - dialect: str | None = None, - mode: ColumnLineageMode = ColumnLineageMode.RICH, - model_names: frozenset[str] | None = None, -) -> ProjectColumnLineage | None: - """Build a sidecar project column lineage graph for compiled models.""" - - match mode: - case ColumnLineageMode.FAST: - return build_native_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) - case ColumnLineageMode.RICH: - return build_native_rich_project_column_lineage( - project=project, dialect=dialect, model_names=model_names - ) diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py index 23b65bd473..a56889257c 100644 --- a/src/sqlbuild/compiler/lineage/models.py +++ b/src/sqlbuild/compiler/lineage/models.py @@ -1,19 +1,10 @@ -"""Column lineage result models and project graph.""" +"""Column lineage result models.""" from __future__ import annotations -from collections import defaultdict -from collections.abc import Sequence -from dataclasses import dataclass, field -from typing import cast +from dataclasses import dataclass -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.constants import INDEXED_EDGE_RECORD_LENGTH from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -103,401 +94,3 @@ class ModelColumnLineage: model_name: str columns: tuple[ColumnLineage, ...] has_star: bool = False - - -@dataclass(frozen=True, init=False) -class ProjectColumnLineage: - """Project-level column lineage graph with lazily materialized public views.""" - - _models: dict[str, ModelColumnLineage] = field(init=False, repr=False) - _compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]] = field( - init=False, repr=False - ) - _model_order: tuple[str, ...] = field(init=False, repr=False) - _edges: tuple[ColumnLineageEdge, ...] | None = field(init=False, repr=False) - _edge_records: tuple[object, ...] = field(init=False, repr=False) - _edge_cache: dict[ - tuple[str, str, str, str, str, ColumnTransformKind, ColumnLineageConfidence], - ColumnLineageEdge, - ] = field(init=False, repr=False) - _edge_counts_by_target_model: dict[str, int] = field(init=False, repr=False) - _indexes_initialized: bool = field(init=False, repr=False) - _edge_records_by_target_model: dict[str, tuple[object, ...]] = field(init=False, repr=False) - _edge_records_by_source_resource: dict[str, tuple[object, ...]] = field(init=False, repr=False) - _edge_record_by_target_column: dict[tuple[str, str], object] = field(init=False, repr=False) - - def __init__( - self, - *, - models: dict[str, ModelColumnLineage], - edges: tuple[ColumnLineageEdge, ...], - ) -> None: - object.__setattr__(self, "_models", models) - object.__setattr__(self, "_compact_models", {}) - object.__setattr__(self, "_model_order", tuple(models)) - object.__setattr__(self, "_edges", edges) - object.__setattr__(self, "_edge_records", edges) - object.__setattr__(self, "_edge_cache", {}) - object.__setattr__(self, "_edge_counts_by_target_model", {}) - object.__setattr__(self, "_indexes_initialized", False) - self._initialize_indexes() - - @classmethod - def from_fast_facts( - cls, - *, - models: dict[str, ModelColumnLineage], - compact_models: dict[str, tuple[Sequence[CompiledLineageColumnFact], bool]], - model_order: tuple[str, ...], - ) -> ProjectColumnLineage: - """Build indexed project lineage without expanding compact analysis facts.""" - - result: ProjectColumnLineage = cls(models=models, edges=()) - object.__setattr__(result, "_compact_models", compact_models) - object.__setattr__(result, "_model_order", model_order) - object.__setattr__(result, "_edges", None) - object.__setattr__(result, "_edge_records", ()) - object.__setattr__(result, "_indexes_initialized", False) - edge_counts: dict[str, int] = {} - for model_name in model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = compact_models.get( - model_name - ) - if compact is None: - continue - compact_columns: Sequence[CompiledLineageColumnFact] = compact[0] - if isinstance(compact_columns, CompactLineageFacts): - edge_counts[model_name] = sum(len(row[3]) for row in compact_columns.rows) - else: - edge_counts[model_name] = sum( - len(column.upstream_columns) for column in compact_columns - ) - for model_name, model in models.items(): - edge_counts[model_name] = sum(len(column.upstream_columns) for column in model.columns) - object.__setattr__(result, "_edge_counts_by_target_model", edge_counts) - return result - - def _build_fast_edge_records( - self, - ) -> tuple[object, ...]: - records: list[object] = [] - for model_name in self._model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = ( - self._compact_models.get(model_name) - ) - if compact is not None: - columns: Sequence[CompiledLineageColumnFact] = compact[0] - if isinstance(columns, CompactLineageFacts): - for name_index, transform_code, confidence_code, sources in columns.rows: - output_column: str = columns.string_pool[name_index] - for source in sources: - records.append( - ( - columns, - source, - model_name, - output_column, - transform_code, - confidence_code, - ) - ) - else: - for column in columns: - for source in column.upstream_columns: - records.append( - ( - source, - model_name, - column.output_column, - column.transform_kind, - column.confidence, - ) - ) - continue - model: ModelColumnLineage | None = self._models.get(model_name) - if model is None: - continue - for column in model.columns: - for source in column.upstream_columns: - records.append( - ( - source, - model_name, - column.output_column, - column.transform_kind, - column.confidence, - ) - ) - return tuple(records) - - @property - def models(self) -> dict[str, ModelColumnLineage]: - """Return model lineage, expanding compact model facts on first access.""" - - if self._compact_models: - for model_name in self._model_order: - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = ( - self._compact_models.get(model_name) - ) - if compact is not None and model_name not in self._models: - self._models[model_name] = self._model_lineage_from_compact_facts( - model_name=model_name, - columns=compact[0], - has_star=compact[1], - ) - object.__setattr__(self, "_compact_models", {}) - return self._models - - @property - def edges(self) -> tuple[ColumnLineageEdge, ...]: - """Return all graph edges, expanding compact edge records on first access.""" - - if self._edges is None: - self._ensure_indexes() - object.__setattr__( - self, - "_edges", - tuple(self._materialize_edge(record) for record in self._edge_records), - ) - edges: tuple[ColumnLineageEdge, ...] | None = self._edges - if edges is None: - return () - return edges - - def has_model(self, model_name: str) -> bool: - """Return whether model lineage is available without expanding its columns.""" - - return model_name in self._models or model_name in self._compact_models - - def model_has_star(self, model_name: str) -> bool: - """Return whether model lineage retains unresolved root-star uncertainty.""" - - compact: tuple[Sequence[CompiledLineageColumnFact], bool] | None = self._compact_models.get( - model_name - ) - if compact is not None: - return compact[1] - model: ModelColumnLineage | None = self._models.get(model_name) - return model.has_star if model is not None else False - - def edge_count_targeting(self, model_name: str) -> int: - """Return a target model's direct edge count without expanding edge objects.""" - - return self._edge_counts_by_target_model.get( - model_name, - len(self._edge_records_by_target_model.get(model_name, ())), - ) - - def _initialize_indexes(self) -> None: - by_target_model: dict[str, list[object]] = defaultdict(list) - by_source_resource: dict[str, list[object]] = defaultdict(list) - by_target_column: dict[tuple[str, str], object] = {} - for record in self._edge_records: - source_name, target_name, target_column = self._edge_record_identity(record) - by_target_model[target_name].append(record) - by_source_resource[source_name].append(record) - by_target_column.setdefault((target_name, target_column), record) - object.__setattr__( - self, - "_edge_records_by_target_model", - {key: tuple(value) for key, value in by_target_model.items()}, - ) - object.__setattr__( - self, - "_edge_records_by_source_resource", - {key: tuple(value) for key, value in by_source_resource.items()}, - ) - object.__setattr__(self, "_edge_record_by_target_column", by_target_column) - object.__setattr__( - self, - "_edge_counts_by_target_model", - {key: len(value) for key, value in self._edge_records_by_target_model.items()}, - ) - object.__setattr__(self, "_indexes_initialized", True) - - def _ensure_indexes(self) -> None: - if self._indexes_initialized: - return - object.__setattr__(self, "_edge_records", self._build_fast_edge_records()) - self._initialize_indexes() - - def _materialize_edge(self, record: object) -> ColumnLineageEdge: - if isinstance(record, ColumnLineageEdge): - return record - ( - resource_type, - resource_name, - source_column, - target_model, - target_column, - transform_kind, - confidence, - ) = self._edge_record_values(record) - key: tuple[ - str, - str, - str, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ] = ( - str(resource_type), - resource_name, - source_column, - target_model, - target_column, - transform_kind, - confidence, - ) - cached: ColumnLineageEdge | None = self._edge_cache.get(key) - if cached is not None: - return cached - edge: ColumnLineageEdge = ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=resource_type, - resource_name=resource_name, - column_name=source_column, - ), - target=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name=target_model, - column_name=target_column, - ), - transform_kind=transform_kind, - confidence=confidence, - ) - self._edge_cache[key] = edge - return edge - - def edges_targeting(self, model_name: str) -> tuple[ColumnLineageEdge, ...]: - """Return edges whose target model is `model_name`.""" - - self._ensure_indexes() - return tuple( - self._materialize_edge(record) - for record in self._edge_records_by_target_model.get(model_name, ()) - ) - - def edges_sourced_from(self, resource_name: str) -> tuple[ColumnLineageEdge, ...]: - """Return edges whose source resource is `resource_name`.""" - - self._ensure_indexes() - return tuple( - self._materialize_edge(record) - for record in self._edge_records_by_source_resource.get(resource_name, ()) - ) - - def column_consumers( - self, - *, - resource_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Return direct downstream consumers of `resource_name.column_name`.""" - - return tuple( - edge - for edge in self.edges_sourced_from(resource_name) - if edge.source.column_name == column_name - ) - - @staticmethod - def _edge_record_identity(record: object) -> tuple[str, str, str]: - if isinstance(record, ColumnLineageEdge): - return ( - record.source.resource_name, - record.target.resource_name, - record.target.column_name, - ) - _, resource_name, _, target_model, target_column, _, _ = ( - ProjectColumnLineage._edge_record_values(record) - ) - return resource_name, target_model, target_column - - @staticmethod - def _edge_record_values( - record: object, - ) -> tuple[ - CompiledResourceType | str, - str, - str, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ]: - if isinstance(record, tuple) and len(record) == INDEXED_EDGE_RECORD_LENGTH: - facts, source, target_model, target_column, transform_code, confidence_code = cast( - tuple[CompactLineageFacts, tuple[int, int, int], str, str, int, int], record - ) - return ( - facts.string_pool[source[0]], - facts.resource_name(source[1]), - facts.string_pool[source[2]], - target_model, - target_column, - facts.transform_kind(transform_code), - facts.confidence(confidence_code), - ) - source, target_model, target_column, transform_kind, confidence = cast( - tuple[ - CompiledLineageSourceFact | ColumnLineageSource, - str, - str, - ColumnTransformKind, - ColumnLineageConfidence, - ], - record, - ) - return ( - source.resource_type, - source.resource_name, - source.column_name, - target_model, - target_column, - transform_kind, - confidence, - ) - - @staticmethod - def _model_lineage_from_compact_facts( - *, - model_name: str, - columns: Sequence[CompiledLineageColumnFact], - has_star: bool, - ) -> ModelColumnLineage: - lineage_columns: list[ColumnLineage] = [] - for column in columns: - upstream_columns: list[ColumnLineageSource] = [] - for source in column.upstream_columns: - upstream_columns.append( - ColumnLineageSource( - resource_type=source.resource_type, - resource_name=source.resource_name, - column_name=source.column_name, - ) - ) - lineage_columns.append( - ColumnLineage( - output_column=column.output_column, - transform_kind=column.transform_kind, - expression_sql=None, - upstream_columns=tuple(upstream_columns), - nullability=InferredNullability.UNKNOWN, - confidence=column.confidence, - ) - ) - return ModelColumnLineage( - model_name=model_name, - columns=tuple(lineage_columns), - has_star=has_star, - ) - - def __repr__(self) -> str: - return f"ProjectColumnLineage(models={self.models!r}, edges={self.edges!r})" - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ProjectColumnLineage): - return NotImplemented - return self.models == other.models and self.edges == other.edges diff --git a/src/sqlbuild/compiler/lineage/types.py b/src/sqlbuild/compiler/lineage/types.py index db075949c4..1a7ba6da30 100644 --- a/src/sqlbuild/compiler/lineage/types.py +++ b/src/sqlbuild/compiler/lineage/types.py @@ -4,8 +4,6 @@ from enum import StrEnum -type NativeSemanticUseRow = tuple[str, str, str, str, str, str, str, int | None, int | None] - class ColumnTransformKind(StrEnum): """High-level transform classification for one output column.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 6e246632b2..1161eb5b16 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -110,11 +110,11 @@ import sqlbuild._native as native -def _panicked(request): +def _panicked(*arguments): raise native.NativeCompilerError({_NATIVE_PANIC_ERROR!r}) -native.build_rich_column_lineage = _panicked +native.build_project_column_lineage = _panicked """ _RICH_COMPILE: tuple[str, ...] = ( "compile", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index fca7d74599..49f6b58631 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,6 +31,7 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization +import sqlbuild.cli.commands._helpers.compile.lineage as compile_lineage import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse.main._write_reusable_compile as write_reusable_module @@ -42,7 +43,6 @@ import sqlbuild.compiler.contracts.main.promotion_conflicts as promotion_conflicts import sqlbuild.compiler.contracts.main.validate as contract_validation import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module -import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( @@ -4843,7 +4843,7 @@ def full_macro_call_runs( (diagnostic_recovery, "complete_native_semantic_diagnostics"), (contract_validation, "evaluate_native_model_contracts"), (promotion_conflicts, "native_promotion_conflict_diagnostics"), - (column_lineage, "build_native_column_lineage"), + (compile_lineage, "build_compile_lineage"), (target_writer, "plan_native_sql_test_artifacts"), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index ff8ae2d638..e8f977cece 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,9 +8,9 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly @@ -36,7 +36,7 @@ "complete_native_semantic_diagnostics": [ANY], "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], - "build_native_column_lineage": [ANY], + "build_compile_lineage": [ANY], "plan_native_sql_test_artifacts": [ANY], }, ) @@ -62,7 +62,7 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_seams == test_case.expected_native_returns planned_counts: list[int] = [] for seams in (native_seams,): - assert isinstance(seams["build_native_column_lineage"][0], ProjectColumnLineage) + assert isinstance(seams["build_compile_lineage"][0], ProjectColumnLineage) assert isinstance(seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) assert seams["analyze_native_model_sql"][0].session is not None diff --git a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py index 5abec348b2..b8ae9adfd3 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py @@ -23,7 +23,7 @@ class FormerlyDeferredLineageTestCase: description: str dialect: str keeps_catalog: bool - expected_native_statuses: dict[str, int] + expected_models_with_lineage: int expected_error: str | None @@ -33,7 +33,6 @@ class UnparsedLineageTestCase: description: str query_sql: str - expected_status: str expected_messages: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index d9dbdbe177..df42624aac 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -2,23 +2,20 @@ import random from collections import Counter -from collections.abc import Callable, Sequence +from collections.abc import Callable from dataclasses import replace from pathlib import Path -from typing import Any, cast +from typing import cast import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ProjectColumnLineage, build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage +from sqlbuild.compiler.lineage.types import ColumnLineageMode _PROJECT_TOML: str = ( 'name = "orders_lineage"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' @@ -243,11 +240,24 @@ def lineage_views( lineage: ProjectColumnLineage = cast( ProjectColumnLineage, - build_native_column_lineage(project=project, dialect=dialect, model_names=model_names), + fast_column_lineage(project=project, dialect=dialect, model_names=model_names), ) return _public_views(project=project, lineage=lineage) +def fast_column_lineage( + *, + project: CompiledProjectView, + dialect: str | None, + model_names: frozenset[str] | None, +) -> ProjectColumnLineage | None: + """The native fast lineage graph, or None without SQL analysis.""" + + return native_module.build_project_column_lineage( + project, dialect, ColumnLineageMode.FAST, model_names + ) + + def _public_views( *, project: CompiledProjectView, lineage: ProjectColumnLineage ) -> tuple[list[object], list[object]]: @@ -279,18 +289,21 @@ def lineage_log_records(caplog: pytest.LogCaptureFixture) -> list[tuple[str, str ] -def record_native_outcomes(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count the native engine's per-model outcome statuses for the rest of the test.""" +def fast_lineage_work( + *, project: CompiledProjectView, model_names: frozenset[str] | None +) -> Counter[str]: + """Count the selected models fast lineage parses and those whose root star it expands.""" - statuses: Counter[str] = Counter() - build: Callable[..., Sequence[tuple[str, object, bool, str | None]]] = ( - native_module.build_fast_column_lineage + names: frozenset[str] = model_names or frozenset(model.name for model in project.models) + return Counter( + parse=sum( + model.name in names and model.fast_lineage_columns is None for model in project.models + ), + star=sum( + model.name in names + and model.fast_lineage_columns is not None + and model.fast_lineage_has_star + and not model.fast_lineage_star_resolved + for model in project.models + ), ) - - def counted(*arguments: Any) -> Sequence[tuple[str, object, bool, str | None]]: - outcomes: Sequence[tuple[str, object, bool, str | None]] = build(*arguments) - statuses.update(outcome[0] for outcome in outcomes) - return outcomes - - monkeypatch.setattr(native_module, "build_fast_column_lineage", counted) - return statuses diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index a71200243c..44fddb43c8 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -7,17 +7,15 @@ from collections import Counter from dataclasses import replace from pathlib import Path +from typing import cast import pytest import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( - build_native_column_lineage, -) -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -34,10 +32,11 @@ ) from tests.integration.src.sqlbuild.compiler.lineage.helpers import ( compiled_project, + fast_column_lineage, + fast_lineage_work, generated_lineage_files, lineage_log_records, lineage_views, - record_native_outcomes, without_compact_facts, ) @@ -62,10 +61,9 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_projects_agree( test_case: GeneratedLineageParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) + work: Counter[str] = Counter() differences: list[tuple[object, object, object]] = [] golden: list[GoldenEntry] = [] for index in range(test_case.count): @@ -79,6 +77,8 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje for variant in (project, without_compact_facts(project)): for dialect in test_case.dialects: for model_names in (None, selected): + for _ in ("with catalog", "restored"): + work.update(fast_lineage_work(project=variant, model_names=model_names)) names, with_catalog = lineage_views( project=variant, dialect=dialect, model_names=model_names ) @@ -109,10 +109,9 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje == [] ) assert differences == [] - assert statuses["deferred"] == 0 - assert sum(statuses.values()) >= test_case.expected_minimum_native - assert statuses["built"] + statuses["omitted"] >= test_case.expected_minimum_parsed - assert statuses["star"] >= test_case.expected_minimum_star_expansions + assert sum(work.values()) >= test_case.expected_minimum_native + assert work["parse"] >= test_case.expected_minimum_parsed + assert work["star"] >= test_case.expected_minimum_star_expansions @pytest.mark.parametrize( @@ -121,13 +120,11 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje UnparsedLineageTestCase( description="an unterminated subquery", query_sql='SELECT order_id FROM (SELECT * FROM __ref("orders_model_0")', - expected_status="unparsed", expected_messages=("fast column lineage parse failed; falling back",), ), UnparsedLineageTestCase( description="two statements", query_sql="SELECT 1; SELECT 2", - expected_status="unparsed", expected_messages=("fast column lineage parse failed; falling back",), ), ], @@ -136,10 +133,8 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_it( test_case: UnparsedLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(11), model_count=_MODEL_COUNT), @@ -157,14 +152,13 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i ) with caplog.at_level(logging.DEBUG, logger="sqlbuild.lineage"): - native: ProjectColumnLineage | None = build_native_column_lineage( + native: ProjectColumnLineage | None = fast_column_lineage( project=broken, dialect=None, model_names=None ) native_records: list[tuple[str, str, object]] = lineage_log_records(caplog) assert native is not None assert not native.has_model(project.models[0].name) - assert statuses[test_case.expected_status] == 1 assert tuple(message for _, message, _ in native_records) == test_case.expected_messages @@ -175,14 +169,14 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i description="a dialect outside the old native parser build", dialect="mysql", keeps_catalog=True, - expected_native_statuses={"built": _MODEL_COUNT}, + expected_models_with_lineage=_MODEL_COUNT, expected_error=None, ), FormerlyDeferredLineageTestCase( description="a project restored without its analysis catalog", dialect="duckdb", keeps_catalog=False, - expected_native_statuses={"built": _MODEL_COUNT}, + expected_models_with_lineage=_MODEL_COUNT, expected_error=None, ), ], @@ -191,9 +185,7 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_answers( test_case: FormerlyDeferredLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", @@ -204,9 +196,13 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a project=compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] ) - _ = lineage_views(project=project, dialect=test_case.dialect, model_names=None) + lineage: ProjectColumnLineage = cast( + ProjectColumnLineage, + fast_column_lineage(project=project, dialect=test_case.dialect, model_names=None), + ) + models_with_lineage: int = sum(lineage.has_model(model.name) for model in project.models) - assert statuses == Counter(test_case.expected_native_statuses), test_case.description + assert models_with_lineage == test_case.expected_models_with_lineage, test_case.description @pytest.mark.parametrize( @@ -216,7 +212,7 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a description="a dialect name Polyglot does not know raises Python's error", dialect="nonsense", keeps_catalog=True, - expected_native_statuses={"unknown_dialect": _MODEL_COUNT}, + expected_models_with_lineage=0, expected_error="Unknown dialect: nonsense", ), ], @@ -225,9 +221,7 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_raises( test_case: FormerlyDeferredLineageTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", @@ -241,8 +235,6 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais with pytest.raises(ValueError, match=test_case.expected_error): _ = lineage_views(project=project, dialect=test_case.dialect, model_names=None) - assert statuses == Counter(test_case.expected_native_statuses), test_case.description - @pytest.mark.parametrize( "test_case", @@ -279,7 +271,7 @@ def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compil ) with pytest.raises(native_module.NativeCompilerError) as raised: - _ = build_native_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) + _ = fast_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) assert str(raised.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py index c2b51307d9..5094b0e206 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py @@ -4,6 +4,8 @@ import pytest +import sqlbuild._native as native_module +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.lineage.selection import ( normalize_lineage_target, select_column_target_lineage, @@ -16,7 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, - ProjectColumnLineage, QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -136,10 +137,12 @@ def test_given_column_target_when_selecting_then_returns_column_trace( received_model_names: list[frozenset[str]] = [] received_modes: list[ColumnLineageMode] = [] - def build_column_lineage_spy(*args: object, **kwargs: object) -> ProjectColumnLineage: - del args - received_model_names.append(cast(frozenset[str], kwargs["model_names"])) - received_modes.append(cast(ColumnLineageMode, kwargs["mode"])) + def build_column_lineage_spy( + project: object, dialect: object, mode: object, model_names: object + ) -> ProjectColumnLineage: + del project, dialect + received_model_names.append(cast(frozenset[str], model_names)) + received_modes.append(cast(ColumnLineageMode, mode)) return ProjectColumnLineage( models={}, edges=( @@ -170,10 +173,7 @@ def build_column_lineage_spy(*args: object, **kwargs: object) -> ProjectColumnLi ), ) - monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.lineage.selection.build_project_column_lineage", - build_column_lineage_spy, - ) + monkeypatch.setattr(native_module, "build_project_column_lineage", build_column_lineage_spy) result: ColumnLineageTrace | None = select_column_target_lineage( graph=graph, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index ac3fe4e037..bfc8b5a41f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import ProjectColumnLineage from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, @@ -19,7 +20,6 @@ from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 2206c7981f..b83ea020b9 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -36,8 +36,7 @@ class RichLineageAnalysisScalingTestCase: description: str chain_length: int unrelated_model_counts: tuple[int, ...] - expected_schema_builds: int - expected_analysis_calls: int + expected_model_names: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 14d2b573af..086dbddaff 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -2,6 +2,8 @@ from pathlib import Path +import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -18,6 +20,7 @@ CompiledSeedView, ) from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSeedFile +from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig @@ -122,3 +125,15 @@ def make_chain_models( for index in range(unrelated_model_count) ) return (root, *chain, *unrelated) + + +def project_column_lineage( + *, + project: CompiledProjectView, + dialect: str | None = None, + mode: ColumnLineageMode = ColumnLineageMode.RICH, + model_names: frozenset[str] | None = None, +) -> ProjectColumnLineage | None: + """The project's native column lineage graph, rich by default.""" + + return native.build_project_column_lineage(project, dialect, mode, model_names) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 6a2d2784f8..76def5dffc 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -1,11 +1,10 @@ from __future__ import annotations from dataclasses import replace -from typing import Any import pytest -import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -16,9 +15,7 @@ CompiledResourceType, ) from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.lineage._helpers import native_rich_columns -from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage -from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage, ProjectColumnLineage +from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnLineageMode, @@ -37,6 +34,7 @@ make_compiled_model, make_compiled_project, make_compiled_seed, + project_column_lineage, ) @@ -177,7 +175,7 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte ), ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None model_lineage: ModelColumnLineage = result.models[test_case.model_name] @@ -309,7 +307,7 @@ def test_given_compiled_project_when_building_fast_column_lineage_then_infers_ex ), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, mode=ColumnLineageMode.FAST, ) @@ -366,7 +364,7 @@ def test_given_select_star_when_building_column_lineage_then_expands_known_schem ) ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None model_lineage: ModelColumnLineage = result.models[test_case.model_name] @@ -405,7 +403,7 @@ def test_given_linear_project_when_listing_column_consumers_then_returns_expecte ) ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert result is not None consumers: tuple[str, ...] = tuple( @@ -457,7 +455,7 @@ def test_given_compact_analysis_facts_when_building_fast_lineage_then_preserves_ ), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=make_compiled_project(models=(upstream, target)), mode=ColumnLineageMode.FAST, ) @@ -553,7 +551,7 @@ def test_given_cte_sourced_column_when_building_rich_column_lineage_then_resolve models=upstream_models + (target_model,), ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, mode=ColumnLineageMode.RICH, ) @@ -612,7 +610,7 @@ def test_given_postgres_dialect_when_building_rich_column_lineage_then_uses_poly ) project: CompiledProjectView = make_compiled_project(models=upstream_models + (target_model,)) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, dialect="postgres", mode=ColumnLineageMode.RICH, @@ -660,7 +658,7 @@ def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_n sql_analysis_enabled=test_case.sql_analysis_enabled, ) - result: ProjectColumnLineage | None = build_project_column_lineage(project=project) + result: ProjectColumnLineage | None = project_column_lineage(project=project) assert (result is None) is test_case.expected_result_is_none @@ -669,41 +667,22 @@ def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_n "test_case", [ RichLineageAnalysisScalingTestCase( - description="builds the schema once and analyses each traced model against its refs", + description="analyses only the traced models, whatever else the project holds", chain_length=4, unrelated_model_counts=(0, 250), - expected_schema_builds=1, - expected_analysis_calls=4, + expected_model_names=("orders_0", "orders_1", "orders_2", "orders_3"), ) ], ids=lambda case: case.description, ) def test_given_unrelated_models_when_tracing_rich_lineage_then_analysis_work_stays_linear( test_case: RichLineageAnalysisScalingTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: - build_lineage: Any = native.build_rich_column_lineage - schema_resources: Any = native_rich_columns._schema_resources - schema_builds: list[CompiledProjectView] = [] - analysed_queries: list[str] = [] - - def counting_schema_mapping(project: CompiledProjectView) -> Any: - schema_builds.append(project) - return schema_resources(project) - - def recording_native_lineage(request: Any) -> Any: - analysed_queries.extend(request[2]) - return build_lineage(request) - - monkeypatch.setattr(native_rich_columns, "_schema_resources", counting_schema_mapping) - monkeypatch.setattr(native, "build_rich_column_lineage", recording_native_lineage) chain_names: frozenset[str] = frozenset( f"orders_{index}" for index in range(test_case.chain_length) ) edges_by_project_size: list[tuple[str, ...]] = [] for unrelated_model_count in test_case.unrelated_model_counts: - schema_builds.clear() - analysed_queries.clear() project: CompiledProjectView = make_compiled_project( models=make_chain_models( chain_length=test_case.chain_length, @@ -711,13 +690,12 @@ def recording_native_lineage(request: Any) -> Any: ) ) - result: ProjectColumnLineage | None = build_project_column_lineage( + result: ProjectColumnLineage | None = project_column_lineage( project=project, model_names=chain_names ) assert result is not None - assert len(schema_builds) == test_case.expected_schema_builds - assert len(analysed_queries) == test_case.expected_analysis_calls + assert tuple(result.models) == test_case.expected_model_names, test_case.description edges_by_project_size.append( tuple( edge_label( From 2aa95bd480bdfda647a0ba145f55bbbdd6e463b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:01:40 +0100 Subject: [PATCH 21/26] refactor: render lineage trees and column payloads natively --- .../src/lineage_graph/constants.rs | 9 + .../src/lineage_graph/main/mod.rs | 2 + .../lineage_graph/main/trace_limit_note.rs | 13 + .../src/lineage_graph/main/tree_lines.rs | 63 ++++ .../src/lineage_graph/mod.rs | 1 + .../src/lineage_graph/models.rs | 9 + .../src/lineage_graph/tests/helpers.rs | 25 +- .../src/lineage_graph/tests/test_graph.rs | 48 ++- .../src/lineage_graph/tests/test_types.rs | 16 + .../src/lineage_graph/types.rs | 13 + .../src/bindings/_helpers/lineage/mod.rs | 1 + .../bindings/_helpers/lineage/registration.rs | 5 +- .../bindings/_helpers/lineage/tree_render.rs | 306 ++++++++++++++++++ fensu.toml | 5 + src/sqlbuild/_native.pyi | 41 ++- .../cli/commands/_helpers/lineage/output.py | 14 +- .../compiler/lineage/_helpers/__init__.py | 1 - .../compiler/lineage/_helpers/tree_render.py | 138 -------- .../compiler/lineage/main/__init__.py | 1 - .../main/render_column_trace_limit_note.py | 20 -- .../lineage/main/render_column_trace_tree.py | 40 --- .../lineage/main/render_dependency_tree.py | 34 -- .../compiler/lineage/main/serialize_column.py | 21 -- .../lineage/main/serialize_column_edge.py | 23 -- 24 files changed, 559 insertions(+), 290 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs create mode 100644 crates/sqlbuild-analysis/src/lineage_graph/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs delete mode 100644 src/sqlbuild/compiler/lineage/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/lineage/_helpers/tree_render.py delete mode 100644 src/sqlbuild/compiler/lineage/main/__init__.py delete mode 100644 src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py delete mode 100644 src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py delete mode 100644 src/sqlbuild/compiler/lineage/main/render_dependency_tree.py delete mode 100644 src/sqlbuild/compiler/lineage/main/serialize_column.py delete mode 100644 src/sqlbuild/compiler/lineage/main/serialize_column_edge.py diff --git a/crates/sqlbuild-analysis/src/lineage_graph/constants.rs b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs index 9fd8a1cb57..8dad7393e9 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/constants.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/constants.rs @@ -15,3 +15,12 @@ pub const ANALYSIS_DIALECT_ALIASES: [(&str, &str); 12] = [ ("sqlserver", "tsql"), ("postgresql", "postgresql"), ]; + +/// The branch before a node's last child. +pub const TREE_BRANCH_LAST: &str = "└── "; +/// The branch before every other child. +pub const TREE_BRANCH_MID: &str = "├── "; +/// The indent under a last child. +pub const TREE_CONTINUATION_LAST: &str = " "; +/// The indent under every other child. +pub const TREE_CONTINUATION_MID: &str = "│ "; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs index c6f3c18b65..2fd9d3193b 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/mod.rs @@ -1,2 +1,4 @@ pub mod analysis_dialect; pub mod edge_index; +pub mod trace_limit_note; +pub mod tree_lines; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs new file mode 100644 index 0000000000..07a7d4af70 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/trace_limit_note.rs @@ -0,0 +1,13 @@ +//! The note shown when a column trace is truncated. + +/// The note's lines, before styling, when `total` columns exceed the `limit` shown. +pub fn trace_limit_note(total: i64, limit: i64) -> Vec { + if total <= limit { + return Vec::new(); + } + vec![ + format!("Showing {limit} of {total} columns."), + "Use --depth 1 to show direct column dependencies only.".to_owned(), + "Use --format json for the full trace.".to_owned(), + ] +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs b/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs new file mode 100644 index 0000000000..3e81952f41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/main/tree_lines.rs @@ -0,0 +1,63 @@ +//! Box-drawing tree traversal that expands each node once across the tree. + +use std::collections::HashSet; +use std::hash::Hash; + +use crate::lineage_graph::constants::{ + TREE_BRANCH_LAST, TREE_BRANCH_MID, TREE_CONTINUATION_LAST, TREE_CONTINUATION_MID, +}; +use crate::lineage_graph::models::TreeLine; +use crate::lineage_graph::types::TreeVisitor; + +/// Visit `children` depth first in order, emitting one line per node; repeats are not expanded. +pub fn tree_lines( + children: Vec, + seen: HashSet, + visitor: &mut impl TreeVisitor, +) -> Result<(), E> +where + K: Eq + Hash, +{ + let mut shown: HashSet = seen; + let mut pending: Vec<(N, String, bool)> = pending_children(children, ""); + while let Some((node, prefix, is_last)) = pending.pop() { + let key: K = visitor.key(&node)?; + let repeated: bool = shown.contains(&key); + visitor.line(TreeLine { + node: &node, + prefix: &prefix, + branch: if is_last { + TREE_BRANCH_LAST + } else { + TREE_BRANCH_MID + }, + repeated, + })?; + if repeated { + continue; + } + let _ = shown.insert(key); + let continuation: &str = if is_last { + TREE_CONTINUATION_LAST + } else { + TREE_CONTINUATION_MID + }; + let grandchildren: Vec = visitor.children(&node)?; + pending.extend(pending_children( + grandchildren, + &format!("{prefix}{continuation}"), + )); + } + Ok(()) +} + +fn pending_children(children: Vec, prefix: &str) -> Vec<(N, String, bool)> { + let last: usize = children.len().saturating_sub(1); + let mut pending: Vec<(N, String, bool)> = children + .into_iter() + .enumerate() + .map(|(index, child)| (child, prefix.to_owned(), index == last)) + .collect(); + pending.reverse(); + pending +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/mod.rs b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs index f659fb0427..55f895632d 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/mod.rs @@ -3,6 +3,7 @@ pub mod constants; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/lineage_graph/models.rs b/crates/sqlbuild-analysis/src/lineage_graph/models.rs index fbf845caf1..679d5c425e 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/models.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/models.rs @@ -30,3 +30,12 @@ pub enum AnalysisDialect<'a> { /// Not an alias: the name itself must be a Polyglot dialect value. Named(&'a str), } + +/// One tree line: its node, indent, branch, and whether the node was already shown. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TreeLine<'a, N> { + pub node: &'a N, + pub prefix: &'a str, + pub branch: &'static str, + pub repeated: bool, +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs index 56d8bccd36..2aafbda202 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/helpers.rs @@ -1,4 +1,8 @@ -use crate::lineage_graph::models::EdgeKey; +use std::convert::Infallible; + +use crate::lineage_graph::models::{EdgeKey, TreeLine}; +use crate::lineage_graph::tests::test_types::StringTree; +use crate::lineage_graph::types::TreeVisitor; /// A direct, high-confidence edge from `source.column` into `target.column`. pub(crate) fn edge_key(source: (&str, &str), target: (&str, &str)) -> EdgeKey { @@ -12,3 +16,22 @@ pub(crate) fn edge_key(source: (&str, &str), target: (&str, &str)) -> EdgeKey { confidence: "high".to_owned(), } } + +impl TreeVisitor<&'static str, &'static str, Infallible> for StringTree { + fn key(&mut self, node: &&'static str) -> Result<&'static str, Infallible> { + Ok(node) + } + + fn line(&mut self, line: TreeLine<'_, &'static str>) -> Result<(), Infallible> { + let suffix: &str = if line.repeated { " (shown)" } else { "" }; + self.lines.push(format!( + "{}{}{}{suffix}", + line.prefix, line.branch, line.node + )); + Ok(()) + } + + fn children(&mut self, node: &&'static str) -> Result, Infallible> { + Ok(self.children.get(node).cloned().unwrap_or_default()) + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs index 32e30eec6f..0000185abe 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_graph.rs @@ -1,8 +1,11 @@ use crate::lineage_graph::main::analysis_dialect::analysis_dialect; use crate::lineage_graph::main::edge_index::edge_index; +use crate::lineage_graph::main::tree_lines::tree_lines; use crate::lineage_graph::models::{AnalysisDialect, EdgeIndex, EdgeKey}; use crate::lineage_graph::tests::helpers::edge_key; -use crate::lineage_graph::tests::test_types::{AnalysisDialectTestCase, EdgeIndexTestCase}; +use crate::lineage_graph::tests::test_types::{ + AnalysisDialectTestCase, EdgeIndexTestCase, StringTree, TreeLinesTestCase, +}; #[test] fn given_edges_when_indexing_then_positions_keep_edge_order_and_equal_edges_share_one() { @@ -79,3 +82,46 @@ fn given_dialect_names_when_resolving_then_aliases_map_and_others_stay_named() { ); } } + +#[test] +fn given_a_graph_when_drawing_a_tree_then_each_node_expands_once() { + let test_cases = [TreeLinesTestCase { + description: "a diamond expands its shared node once and skips the focus", + children: &[ + ("orders", &["customers", "products"]), + ("customers", &["regions", "orders"]), + ("products", &["regions"]), + ("regions", &["countries"]), + ], + roots: &["customers", "products"], + seen: &["orders"], + expected_lines: &[ + "├── customers", + "│ ├── regions", + "│ │ └── countries", + "│ └── orders (shown)", + "└── products", + " └── regions (shown)", + ], + }]; + for test_case in test_cases { + let mut tree = StringTree { + children: test_case + .children + .iter() + .map(|(node, children)| (*node, children.to_vec())) + .collect(), + lines: Vec::new(), + }; + let Ok(()) = tree_lines( + test_case.roots.to_vec(), + test_case.seen.iter().copied().collect(), + &mut tree, + ); + assert_eq!( + tree.lines, test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs index d664364af6..91192c2d77 100644 --- a/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/lineage_graph/tests/test_types.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::lineage_graph::models::AnalysisDialect; /// `(resource, column)`. @@ -16,3 +18,17 @@ pub(crate) struct AnalysisDialectTestCase { pub(crate) normalized: &'static str, pub(crate) expected_dialect: AnalysisDialect<'static>, } + +pub(crate) struct TreeLinesTestCase { + pub(crate) description: &'static str, + pub(crate) children: &'static [(&'static str, &'static [&'static str])], + pub(crate) roots: &'static [&'static str], + pub(crate) seen: &'static [&'static str], + pub(crate) expected_lines: &'static [&'static str], +} + +/// A string tree whose lines render as `prefix + branch + node`, repeats marked `(shown)`. +pub(crate) struct StringTree { + pub(crate) children: HashMap<&'static str, Vec<&'static str>>, + pub(crate) lines: Vec, +} diff --git a/crates/sqlbuild-analysis/src/lineage_graph/types.rs b/crates/sqlbuild-analysis/src/lineage_graph/types.rs new file mode 100644 index 0000000000..c85a12a94c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_graph/types.rs @@ -0,0 +1,13 @@ +//! Caller-supplied tree behaviour. + +use crate::lineage_graph::models::TreeLine; + +/// The caller's view of a tree: node identity, child order and line output. +pub trait TreeVisitor { + /// The identity a node is shown once by. + fn key(&mut self, node: &N) -> Result; + /// Record one line. + fn line(&mut self, line: TreeLine<'_, N>) -> Result<(), E>; + /// A node's children, in display order. + fn children(&mut self, node: &N) -> Result, E>; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index 6ac630c2c6..cb9ea47c07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod project_lineage; pub(crate) mod project_schema; pub(crate) mod registration; pub(crate) mod relation_fingerprint; +pub(crate) mod tree_render; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index 58ac10cb33..cb84372d9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::lineage::{ - column_lineage_graph, project_lineage, relation_fingerprint, + column_lineage_graph, project_lineage, relation_fingerprint, tree_render, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_lineage_graph::register(module)?; project_lineage::register(module)?; - relation_fingerprint::register(module) + relation_fingerprint::register(module)?; + tree_render::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs new file mode 100644 index 0000000000..6007889188 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs @@ -0,0 +1,306 @@ +//! Lineage output: box-drawing dependency and column-trace trees, and column JSON payloads. + +use std::collections::HashSet; + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage_graph::main::trace_limit_note::trace_limit_note; +use sqlbuild_analysis::lineage_graph::main::tree_lines::tree_lines; +use sqlbuild_analysis::lineage_graph::models::TreeLine; +use sqlbuild_analysis::lineage_graph::types::TreeVisitor; + +const BUILTINS_MODULE: &str = "builtins"; + +/// Box-drawing dependency tree lines for one focus node, expanding each node once. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_dependency_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_dependency_tree", request)?; + let py: Python<'_> = request.py(); + let mut tree = DependencyTree { + deps: request.get("deps")?, + sort_key: request.get("sort_key")?, + format_node: request.get("format_node")?, + branch_style: request.get("branch_style")?, + already_shown: request.get("already_shown")?, + sorted: py.import(BUILTINS_MODULE)?.getattr("sorted")?, + identities: PyDict::new(py), + lines: Vec::new(), + }; + let focus: Bound<'_, PyAny> = request.get("focus")?; + let seen: Bound<'_, PyAny> = request.get("seen")?; + let roots: Vec> = tree.children(&focus)?; + let mut shown: HashSet = HashSet::new(); + for node in seen.try_iter()? { + let _ = shown.insert(tree.key(&node?)?); + } + tree_lines(roots, shown, &mut tree)?; + Ok(tree.lines) +} + +/// Box-drawing column-trace tree lines plus any truncation note. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_column_trace_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_column_trace_tree", request)?; + let target: Bound<'_, PyAny> = request.get("target")?; + let mut tree = ColumnTraceTree { + deps: request.get("deps")?, + column_id: request.get("column_id")?, + related_column: request.get("related_column")?, + format_related: request.get("format_related")?, + branch_style: request.get("branch_style")?, + already_shown: request.get("already_shown")?, + last_key: String::new(), + lines: Vec::new(), + }; + let seen: HashSet = HashSet::from([text(&tree.column_id.call1((&target,))?)?]); + let roots: Vec> = + tree.sorted_edges(&text(&tree.column_id.call1((&target,))?)?)?; + tree_lines(roots, seen, &mut tree)?; + let mut lines: Vec = tree.lines; + lines.extend(limit_note( + &request.get("total")?, + &request.get("limit")?, + &request.get("note_style")?, + )?); + Ok(lines) +} + +/// The shared truncation note when a column trace is limited. +#[pyfunction] +#[pyo3(signature = (**request))] +fn render_column_trace_limit_note(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("render_column_trace_limit_note", request)?; + limit_note( + &request.get("total")?, + &request.get("limit")?, + &request.get("note_style")?, + ) +} + +/// A qualified lineage column's JSON payload, its resource type rendered by the caller. +#[pyfunction] +#[pyo3(signature = (**request))] +fn serialize_column<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("serialize_column", request)?; + column_payload( + &request.get("column")?, + &request.get("render_resource_type")?, + ) +} + +/// A column lineage edge's JSON payload, resource types rendered by the caller. +#[pyfunction] +#[pyo3(signature = (**request))] +fn serialize_column_edge<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("serialize_column_edge", request)?; + let edge: Bound<'py, PyAny> = request.get("edge")?; + let render: Bound<'py, PyAny> = request.get("render_resource_type")?; + let payload: Bound<'py, PyDict> = PyDict::new(request.py()); + payload.set_item("source", column_payload(&edge.getattr("source")?, &render)?)?; + payload.set_item("target", column_payload(&edge.getattr("target")?, &render)?)?; + payload.set_item("transform", edge.getattr("transform_kind")?.str()?)?; + payload.set_item("confidence", edge.getattr("confidence")?.str()?)?; + Ok(payload) +} + +/// Keyword arguments of one call, each required. +struct Request<'a, 'py> { + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + py: Python<'py>, +} + +impl<'a, 'py> Request<'a, 'py> { + fn new(function: &'static str, arguments: Option<&'a Bound<'py, PyDict>>) -> PyResult { + let py: Python<'py> = match arguments { + Some(arguments) => arguments.py(), + None => { + return Err(PyTypeError::new_err(format!( + "{function}() missing required keyword-only arguments" + ))); + } + }; + Ok(Self { + function, + arguments, + py, + }) + } + + fn py(&self) -> Python<'py> { + self.py + } + + fn get(&self, name: &str) -> PyResult> { + let value: Option> = match self.arguments { + Some(arguments) => arguments.get_item(name)?, + None => None, + }; + value.ok_or_else(|| { + PyTypeError::new_err(format!( + "{}() missing 1 required keyword-only argument: '{name}'", + self.function + )) + }) + } +} + +/// `str(value)`, as an f-string renders a callback's text. +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn limit_note( + total: &Bound<'_, PyAny>, + limit: &Bound<'_, PyAny>, + note_style: &Bound<'_, PyAny>, +) -> PyResult> { + let notes: Vec = trace_limit_note(total.extract()?, limit.extract()?); + if notes.is_empty() { + return Ok(notes); + } + let mut lines: Vec = vec![String::new()]; + for note in notes { + lines.push(note_style.call1((note,))?.extract()?); + } + Ok(lines) +} + +fn column_payload<'py>( + column: &Bound<'py, PyAny>, + render_resource_type: &Bound<'py, PyAny>, +) -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(column.py()); + payload.set_item("resource_type", render_resource_type.call1((column,))?)?; + payload.set_item("resource_name", column.getattr("resource_name")?)?; + payload.set_item("column_name", column.getattr("column_name")?)?; + Ok(payload) +} + +/// Dependency nodes shown once by Python equality, children sorted by the caller's key. +struct DependencyTree<'py> { + deps: Bound<'py, PyAny>, + sort_key: Bound<'py, PyAny>, + format_node: Bound<'py, PyAny>, + branch_style: Bound<'py, PyAny>, + already_shown: Bound<'py, PyAny>, + sorted: Bound<'py, PyAny>, + identities: Bound<'py, PyDict>, + lines: Vec, +} + +impl<'py> TreeVisitor, usize, PyErr> for DependencyTree<'py> { + fn key(&mut self, node: &Bound<'py, PyAny>) -> PyResult { + if let Some(identity) = self.identities.get_item(node)? { + return identity.extract(); + } + let identity: usize = self.identities.len(); + self.identities.set_item(node, identity)?; + Ok(identity) + } + + fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { + let suffix: String = if line.repeated { + text(&self.already_shown.call0()?)? + } else { + String::new() + }; + let branch: String = text( + &self + .branch_style + .call1((format!("{}{}", line.prefix, line.branch),))?, + )?; + let node: String = text(&self.format_node.call1((line.node,))?)?; + self.lines.push(format!("{branch}{node}{suffix}")); + Ok(()) + } + + fn children(&mut self, node: &Bound<'py, PyAny>) -> PyResult>> { + let children: Bound<'py, PyAny> = self + .deps + .call_method1("get", (node, PyTuple::empty(node.py())))?; + let kwargs: Bound<'py, PyDict> = PyDict::new(node.py()); + kwargs.set_item("key", &self.sort_key)?; + let sorted: Bound<'py, PyAny> = self.sorted.call((children,), Some(&kwargs))?; + Ok(sorted.downcast::()?.iter().collect()) + } +} + +/// Column-trace edges shown once per related column id, sorted by that id. +struct ColumnTraceTree<'py> { + deps: Bound<'py, PyAny>, + column_id: Bound<'py, PyAny>, + related_column: Bound<'py, PyAny>, + format_related: Bound<'py, PyAny>, + branch_style: Bound<'py, PyAny>, + already_shown: Bound<'py, PyAny>, + last_key: String, + lines: Vec, +} + +impl<'py> ColumnTraceTree<'py> { + fn related_id(&self, edge: &Bound<'py, PyAny>) -> PyResult { + text( + &self + .column_id + .call1((self.related_column.call1((edge,))?,))?, + ) + } + + /// `deps.get(column_id, ())` sorted by each edge's related column id, stably. + fn sorted_edges(&self, column_id: &str) -> PyResult>> { + let py: Python<'py> = self.deps.py(); + let edges: Bound<'py, PyAny> = self + .deps + .call_method1("get", (PyString::new(py, column_id), PyTuple::empty(py)))?; + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for edge in edges.try_iter()? { + let edge: Bound<'py, PyAny> = edge?; + keyed.push((self.related_id(&edge)?, edge)); + } + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, edge)| edge).collect()) + } +} + +impl<'py> TreeVisitor, String, PyErr> for ColumnTraceTree<'py> { + fn key(&mut self, edge: &Bound<'py, PyAny>) -> PyResult { + self.last_key = self.related_id(edge)?; + Ok(self.last_key.clone()) + } + + fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { + let suffix: String = if line.repeated { + text(&self.already_shown.call0()?)? + } else { + String::new() + }; + let branch: String = text(&self.branch_style.call1((line.branch,))?)?; + let related: String = text(&self.format_related.call1((line.node,))?)?; + self.lines + .push(format!("{}{branch}{related}{suffix}", line.prefix)); + Ok(()) + } + + fn children(&mut self, _edge: &Bound<'py, PyAny>) -> PyResult>> { + let key: String = self.last_key.clone(); + self.sorted_edges(&key) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_dependency_tree, module)?)?; + module.add_function(wrap_pyfunction!(render_column_trace_tree, module)?)?; + module.add_function(wrap_pyfunction!(render_column_trace_limit_note, module)?)?; + module.add_function(wrap_pyfunction!(serialize_column, module)?)?; + module.add_function(wrap_pyfunction!(serialize_column_edge, module)?)?; + Ok(()) +} diff --git a/fensu.toml b/fensu.toml index 3bab925188..a4d7b2e642 100644 --- a/fensu.toml +++ b/fensu.toml @@ -73,6 +73,11 @@ rule = "FFR309" path = "src/sqlbuild/adapters/sqlserver/__init__.py" reason = "The SQL Server backend intentionally owns behavior through its adapter class rather than function entrypoints." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/lineage/__init__.py" +reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a75cb6bcb7..6b8d8c4dd1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,9 +1,11 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Mapping, Sequence, Set +from collections.abc import Callable, Hashable, Mapping, Sequence, Set from pathlib import Path, PurePath from typing import Any, Literal, TypedDict, overload +from _typeshed import SupportsRichComparison + from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts @@ -84,6 +86,7 @@ from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, DirectSemanticColumnUse, ModelColumnLineage, + QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.pipeline.models import ( @@ -1784,6 +1787,42 @@ def build_direct_semantic_uses( model_names: frozenset[str], /, ) -> tuple[DirectSemanticColumnUse, ...]: ... +def render_dependency_tree[Node: Hashable]( + *, + focus: Node, + deps: dict[Node, list[Node]], + seen: set[Node], + format_node: Callable[[Node], str], + sort_key: Callable[[Node], SupportsRichComparison], + branch_style: Callable[[str], str], + already_shown: Callable[[], str], +) -> list[str]: ... +def render_column_trace_tree[Column, Edge]( + *, + target: Column, + deps: dict[str, list[Edge]], + total: int, + limit: int, + column_id: Callable[[Column], str], + related_column: Callable[[Edge], Column], + format_related: Callable[[Edge], str], + branch_style: Callable[[str], str], + already_shown: Callable[[], str], + note_style: Callable[[str], str], +) -> list[str]: ... +def render_column_trace_limit_note( + *, total: int, limit: int, note_style: Callable[[str], str] +) -> list[str]: ... +def serialize_column( + *, + column: QualifiedLineageColumn, + render_resource_type: Callable[[QualifiedLineageColumn], str], +) -> dict[str, object]: ... +def serialize_column_edge( + *, + edge: ColumnLineageEdge, + render_resource_type: Callable[[QualifiedLineageColumn], str], +) -> dict[str, object]: ... class ProjectColumnLineage: """Project-level column lineage graph with lazily materialized public views.""" diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/output.py b/src/sqlbuild/cli/commands/_helpers/lineage/output.py index 1984153b55..b962ec46e1 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/output.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/output.py @@ -4,6 +4,13 @@ import json +from sqlbuild._native import ( + render_column_trace_limit_note, + render_column_trace_tree, + render_dependency_tree, + serialize_column, + serialize_column_edge, +) from sqlbuild.cli.commands.constants import ( BOTH_DIRECTIONS, DOWNSTREAM_DIRECTION, @@ -16,13 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.main.render_column_trace_limit_note import ( - render_column_trace_limit_note, -) -from sqlbuild.compiler.lineage.main.render_column_trace_tree import render_column_trace_tree -from sqlbuild.compiler.lineage.main.render_dependency_tree import render_dependency_tree -from sqlbuild.compiler.lineage.main.serialize_column import serialize_column -from sqlbuild.compiler.lineage.main.serialize_column_edge import serialize_column_edge from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn from sqlbuild.compiler.lineage.types import ColumnTransformKind from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/src/sqlbuild/compiler/lineage/_helpers/__init__.py b/src/sqlbuild/compiler/lineage/_helpers/__init__.py deleted file mode 100644 index efd4cc4374..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Column lineage helper utilities.""" diff --git a/src/sqlbuild/compiler/lineage/_helpers/tree_render.py b/src/sqlbuild/compiler/lineage/_helpers/tree_render.py deleted file mode 100644 index 1873b21e45..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/tree_render.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Generic box-drawing tree renderers shared by lineage output commands.""" - -from __future__ import annotations - -from collections.abc import Callable, Hashable -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from _typeshed import SupportsRichComparison as SupportsSortKey -else: - SupportsSortKey = object - -_BRANCH_LAST: str = "└── " -_BRANCH_MID: str = "├── " -_CONTINUATION_LAST: str = " " -_CONTINUATION_MID: str = "│ " - - -def render_dependency_branch[Node: Hashable]( - *, - node: Node, - deps: dict[Node, list[Node]], - prefix: str, - seen: set[Node], - format_node: Callable[[Node], str], - sort_key: Callable[[Node], SupportsSortKey], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render a box-drawing dependency branch, expanding each node once across the tree.""" - - lines: list[str] = [] - shown: set[Node] = set(seen) - pending: list[tuple[Node, str, bool]] = _pending_children( - children=sorted(deps.get(node, ()), key=sort_key), prefix=prefix - ) - while pending: - child: Node - child_prefix: str - is_last: bool - child, child_prefix, is_last = pending.pop() - branch: str = _BRANCH_LAST if is_last else _BRANCH_MID - suffix: str = already_shown() if child in shown else "" - lines.append(f"{branch_style(child_prefix + branch)}{format_node(child)}{suffix}") - if child in shown: - continue - shown.add(child) - continuation: str = _CONTINUATION_LAST if is_last else _CONTINUATION_MID - pending.extend( - _pending_children( - children=sorted(deps.get(child, ()), key=sort_key), - prefix=child_prefix + continuation, - ) - ) - return lines - - -def render_column_trace_branch[Column, Edge]( - *, - column: Column, - deps: dict[str, list[Edge]], - prefix: str, - seen: set[str], - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], - format_related: Callable[[Edge], str], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render a box-drawing column-trace branch, expanding each column once across the tree.""" - - lines: list[str] = [] - shown: set[str] = set(seen) - pending: list[tuple[Edge, str, bool]] = _pending_children( - children=_sorted_edges( - edges=deps.get(column_id(column), ()), - column_id=column_id, - related_column=related_column, - ), - prefix=prefix, - ) - while pending: - edge: Edge - edge_prefix: str - is_last: bool - edge, edge_prefix, is_last = pending.pop() - branch: str = _BRANCH_LAST if is_last else _BRANCH_MID - related: Column = related_column(edge) - related_id: str = column_id(related) - suffix: str = already_shown() if related_id in shown else "" - lines.append(f"{edge_prefix}{branch_style(branch)}{format_related(edge)}{suffix}") - if related_id in shown: - continue - shown.add(related_id) - continuation: str = _CONTINUATION_LAST if is_last else _CONTINUATION_MID - pending.extend( - _pending_children( - children=_sorted_edges( - edges=deps.get(related_id, ()), - column_id=column_id, - related_column=related_column, - ), - prefix=edge_prefix + continuation, - ) - ) - return lines - - -def _pending_children[Item](*, children: list[Item], prefix: str) -> list[tuple[Item, str, bool]]: - last: int = len(children) - 1 - return [(child, prefix, index == last) for index, child in reversed(list(enumerate(children)))] - - -def _sorted_edges[Column, Edge]( - *, - edges: list[Edge] | tuple[()], - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], -) -> list[Edge]: - return sorted(edges, key=lambda edge: column_id(related_column(edge))) - - -def render_column_trace_limit_note( - *, - total: int, - limit: int, - note_style: Callable[[str], str], -) -> list[str]: - """Render the shared truncation note when a column trace is limited.""" - - if total <= limit: - return [] - return [ - "", - note_style(f"Showing {limit} of {total} columns."), - note_style("Use --depth 1 to show direct column dependencies only."), - note_style("Use --format json for the full trace."), - ] diff --git a/src/sqlbuild/compiler/lineage/main/__init__.py b/src/sqlbuild/compiler/lineage/main/__init__.py deleted file mode 100644 index 800fd62391..0000000000 --- a/src/sqlbuild/compiler/lineage/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Column lineage analyzer entry points.""" diff --git a/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py b/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py deleted file mode 100644 index fff2e4b4af..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_column_trace_limit_note.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public entry for rendering the column-trace truncation note.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - render_column_trace_limit_note as _render_column_trace_limit_note, -) - - -def render_column_trace_limit_note( - *, - total: int, - limit: int, - note_style: Callable[[str], str], -) -> list[str]: - """Render the shared column-trace truncation note when the trace is limited.""" - - return _render_column_trace_limit_note(total=total, limit=limit, note_style=note_style) diff --git a/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py b/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py deleted file mode 100644 index d2e1ffcecf..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_column_trace_tree.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Public entry for rendering a box-drawing column-trace tree with limit note.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - render_column_trace_branch, - render_column_trace_limit_note, -) - - -def render_column_trace_tree[Column, Edge]( - *, - target: Column, - deps: dict[str, list[Edge]], - total: int, - limit: int, - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], - format_related: Callable[[Edge], str], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], - note_style: Callable[[str], str], -) -> list[str]: - """Render the box-drawing column-trace tree lines plus any truncation note.""" - - lines: list[str] = render_column_trace_branch( - column=target, - deps=deps, - prefix="", - seen={column_id(target)}, - column_id=column_id, - related_column=related_column, - format_related=format_related, - branch_style=branch_style, - already_shown=already_shown, - ) - lines.extend(render_column_trace_limit_note(total=total, limit=limit, note_style=note_style)) - return lines diff --git a/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py b/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py deleted file mode 100644 index 840c0e3899..0000000000 --- a/src/sqlbuild/compiler/lineage/main/render_dependency_tree.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public entry for rendering a box-drawing lineage dependency tree.""" - -from __future__ import annotations - -from collections.abc import Callable, Hashable - -from sqlbuild.compiler.lineage._helpers.tree_render import ( - SupportsSortKey, - render_dependency_branch, -) - - -def render_dependency_tree[Node: Hashable]( - *, - focus: Node, - deps: dict[Node, list[Node]], - seen: set[Node], - format_node: Callable[[Node], str], - sort_key: Callable[[Node], SupportsSortKey], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: - """Render the box-drawing dependency tree lines for one focus node.""" - - return render_dependency_branch( - node=focus, - deps=deps, - prefix="", - seen=seen, - format_node=format_node, - sort_key=sort_key, - branch_style=branch_style, - already_shown=already_shown, - ) diff --git a/src/sqlbuild/compiler/lineage/main/serialize_column.py b/src/sqlbuild/compiler/lineage/main/serialize_column.py deleted file mode 100644 index 59239af8e4..0000000000 --- a/src/sqlbuild/compiler/lineage/main/serialize_column.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public entry for serializing a qualified lineage column to a JSON payload.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage.models import QualifiedLineageColumn - - -def serialize_column( - *, - column: QualifiedLineageColumn, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: - """Serialize a qualified lineage column using a caller-provided resource renderer.""" - - return { - "resource_type": render_resource_type(column), - "resource_name": column.resource_name, - "column_name": column.column_name, - } diff --git a/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py b/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py deleted file mode 100644 index 9a914e1c26..0000000000 --- a/src/sqlbuild/compiler/lineage/main/serialize_column_edge.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entry for serializing a column lineage edge to a stable JSON payload.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.lineage.main.serialize_column import serialize_column -from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn - - -def serialize_column_edge( - *, - edge: ColumnLineageEdge, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: - """Serialize a column lineage edge using a caller-provided resource renderer.""" - - return { - "source": serialize_column(column=edge.source, render_resource_type=render_resource_type), - "target": serialize_column(column=edge.target, render_resource_type=render_resource_type), - "transform": str(edge.transform_kind), - "confidence": str(edge.confidence), - } From ae895b386eea8f5098eef79637397d26407a65fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:13:17 +0100 Subject: [PATCH 22/26] fix: raise the interpreter's own decoder error for invalid fingerprint storage --- .../_helpers/fingerprints/latest_rows.rs | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index 676b2266c6..895fbdc8d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -6,7 +6,7 @@ use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDateTime, PyDict, PyString}; +use pyo3::types::{PyBytes, PyDateTime, PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::errors::StorageDecodeError; use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage; @@ -18,6 +18,7 @@ const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; +const BASE64_MODULE: &str = "base64"; const TIMESTAMP_COLUMN: usize = 11; /// `FingerprintInputError(message)`. @@ -197,7 +198,7 @@ fn decoded( Ok(decoded) => return Ok(decoded), Err(failure) => failure, }; - let cause: PyErr = decode_error(py, failure)?; + let cause: PyErr = decode_error(storage, failure)?; let name: String = node_name.extract()?; let raised: PyErr = fingerprint_input_error( py, @@ -212,19 +213,35 @@ fn decoded( Err(raised) } -fn decode_error(py: Python<'_>, failure: StorageDecodeError) -> PyResult { +/// The interpreter's own decoder error, whose message varies by Python version, else ours. +fn decode_error(storage: &Bound<'_, PyString>, failure: StorageDecodeError) -> PyResult { + let py: Python<'_> = storage.py(); match failure { - StorageDecodeError::Base64(message) => Ok(PyErr::from_value( - py.import(BINASCII_MODULE)? - .getattr("Error")? - .call1((message,))?, - )), + StorageDecodeError::Base64(message) => { + let kwargs: Bound<'_, PyDict> = PyDict::new(py); + kwargs.set_item("validate", true)?; + let decoded = py.import(BASE64_MODULE)?.getattr("b64decode")?.call( + (storage.call_method1("encode", ("ascii",))?,), + Some(&kwargs), + ); + match decoded { + Err(error) => Ok(error), + Ok(_) => Ok(PyErr::from_value( + py.import(BINASCII_MODULE)? + .getattr("Error")? + .call1((message,))?, + )), + } + } StorageDecodeError::Utf8 { bytes, start, end, reason, } => { + if let Err(error) = PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",)) { + return Ok(error); + } let reason: CString = CString::new(reason).map_err(|error| PyValueError::new_err(error.to_string()))?; Ok(PyErr::from_value( From 5c20c5331775e32d7ffe8dce359a5f29deaa8263 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:13:21 +0100 Subject: [PATCH 23/26] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index fc6b5173d8..058318c195 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 134588 +remaining_lines = 130215 [[stays]] path = "executor" From 0998608e9196f0705af16c5367f6765e5d75d7a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:21:57 +0100 Subject: [PATCH 24/26] test: ignore phase durations when comparing launcher streams --- tests/integration/src/sqlbuild/cli/entry/helpers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index cfb0c8ed1b..dc5bba1d5b 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -12,6 +12,7 @@ type LauncherRun = tuple[int, bytes, bytes] _ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") +_PHASE_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d+s\)") NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") PROJECT_FILES: dict[str, str] = { @@ -43,7 +44,7 @@ def write_project(project_dir: Path) -> Path: def run(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: - """Run one CLI process with reuse enabled; return its exit code, stdout and stderr.""" + """Run one CLI process with reuse enabled; return its exit code, stdout and timeless stderr.""" completed: subprocess.CompletedProcess[bytes] = subprocess.run( [str(program), *args], @@ -53,7 +54,11 @@ def run(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: env={**os.environ, "NO_COLOR": "1", REUSE_DISABLE_ENV_VAR: "0"}, timeout=600, ) - return completed.returncode, completed.stdout, completed.stderr + return ( + completed.returncode, + completed.stdout, + _PHASE_SECONDS.sub(b"(s)", completed.stderr), + ) def launch_records(project_dir: Path) -> dict[Path, int]: From 294f137df0cbbd2936989013764c65024aa3cdb5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:29:24 +0100 Subject: [PATCH 25/26] chore: drop duplicate boundary entries and a stale import check --- python-boundary.toml | 8 -------- .../main/evaluate_rule/test_authoring_imports.py | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index 058318c195..3bef235801 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -247,11 +247,3 @@ note = "A baseline that does not compile yields no relation keys, as before the [[handled_native_errors]] location = "lint/_helpers/fixes.py::_FixSession._verdict" note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." - -[[handled_native_errors]] -location = "compiler/discovery/_helpers/filesystem/aggregation.py::_discover_declarations" -note = "Tolerant scope discovery records a failing declaration category as a fault; no Python implementation runs." - -[[handled_native_errors]] -location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" -note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py index 7981434951..e2e0395869 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py @@ -19,7 +19,7 @@ imported_module="sqlbuild.rules", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.pipeline.main.project", + "sqlbuild.compiler.discovery._helpers.filesystem.core", "sqlbuild.adapters.duckdb.classes.duckdb_adapter", ), expected_loaded_modules=(), @@ -29,7 +29,7 @@ imported_module="sqlbuild.rule_engine._helpers.host.custom_host", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.pipeline.main.project", + "sqlbuild.compiler.discovery._helpers.filesystem.core", ), expected_loaded_modules=(), ), From 8042fa4aa07e7904ceb130f55a5c77e1892d56b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:01:53 +0100 Subject: [PATCH 26/26] build: lint Rust with the CI toolchain in make check-ci --- AGENTS.md | 5 +++-- Makefile | 11 +++++++---- .../src/bindings/_helpers/discovery/snapshot.rs | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6d5c006c03..7776917bb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -215,8 +215,9 @@ - An incremental rebuild after a focused change takes minutes. A build that is unexpectedly cold or much slower than that is a signal to stop and investigate, not to wait or start over. - Get fast feedback first: `cargo check -p ` while editing, `cargo test -p ` for - the crate you touched, then one maturin build and the focused Python tests. Run clippy with the - CI toolchain for the touched crates only. + the crate you touched, then one maturin build and the focused Python tests. CI lints with the + latest stable toolchain, so keep the local `stable` toolchain current (`rustup update stable`); + `make check-ci` includes `make rust-lint` (workspace clippy with `-D warnings`) before a push. - Fix compiler errors by reading them and making the smallest correct change. Never delete and rewrite a working module, or restart a port from scratch, because a build failed. - Do not run two native builds against the same target directory at once, and do not run diff --git a/Makefile b/Makefile index 1b9ccf6c88..52be76b8d4 100644 --- a/Makefile +++ b/Makefile @@ -33,12 +33,15 @@ type: dev-native: uv run --no-sync maturin develop --profile dev --skip-install -rust-check: - cargo fmt --all --check - cargo clippy --workspace --all-targets --all-features -- -D warnings +rust-check: rust-lint uv run fensu check cargo test --workspace --all-features +.PHONY: rust-lint +rust-lint: + cargo fmt --all --check + cargo clippy --workspace --all-targets --all-features -- -D warnings + test: @mkdir -p /tmp/opencode @@ -604,7 +607,7 @@ verify-quick: exit $$status -check-ci: check-e2e-shards check-native-layering check-python-boundary +check-ci: check-e2e-shards check-native-layering check-python-boundary rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs index 77019fa064..33d57d6a5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs @@ -136,8 +136,10 @@ pub(crate) fn escaped(text: &Bound<'_, PyString>) -> PyResult { .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? .extract()?; let units = encoded - .chunks_exact(2) - .map(|pair| u16::from_le_bytes([pair[0], pair[1]])); + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)); Ok(char::decode_utf16(units) .map(|decoded| match decoded { Ok(character) => character.to_string(),